Read Zarr Mesh Files

app/controllers/WKRemoteWorkerController.scala

@@ -79,16 +79,18 @@ class WKRemoteWorkerController @Inject()(jobDAO: JobDAO,
       for {
         _ <- workerDAO.findOneByKey(key) ?~> "job.worker.notFound"
         jobBeforeChange <- jobDAO.findOne(id)(GlobalAccessContext)
-        _ <- jobDAO.updateStatus(id, request.body)
+        _ <- jobDAO.updateStatus(id, request.body) ?~> "job.updateStatus.failed"
         jobAfterChange <- jobDAO.findOne(id)(GlobalAccessContext) ?~> "job.notFound"
         _ = jobService.trackStatusChange(jobBeforeChange, jobAfterChange)
         _ <- jobService.cleanUpIfFailed(jobAfterChange) ?~> "job.cleanup.failed"
         _ <- Fox.runIf(request.body.state == JobState.SUCCESS) {
-          creditTransactionService.completeTransactionOfJob(jobAfterChange._id)(GlobalAccessContext)
+          creditTransactionService
+            .completeTransactionOfJob(jobAfterChange._id)(GlobalAccessContext) ?~> "job.creditTransaction.failed"
         }
         _ <- Fox.runIf(
           jobAfterChange.state != request.body.state && (request.body.state == JobState.FAILURE || request.body.state == JobState.CANCELLED)) {
-          creditTransactionService.refundTransactionForJob(jobAfterChange._id)(GlobalAccessContext)
+          creditTransactionService
+            .refundTransactionForJob(jobAfterChange._id)(GlobalAccessContext) ?~> "job.creditTransaction.refund.failed"
         }
       } yield Ok
   }

app/models/organization/CreditTransactionService.scala

@@ -2,7 +2,7 @@ package models.organization
 
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.objectid.ObjectId
-import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.util.tools.{Empty, Failure, Fox, FoxImplicits, Full}
 import com.typesafe.scalalogging.LazyLogging
 import play.api.libs.json.{JsObject, Json}
 
@@ -42,15 +42,30 @@ class CreditTransactionService @Inject()(creditTransactionDAO: CreditTransaction
 
   def completeTransactionOfJob(jobId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] =
     for {
-      transaction <- creditTransactionDAO.findTransactionForJob(jobId)
-      _ <- organizationService.assertOrganizationHasPaidPlan(transaction._organization)
-      _ <- creditTransactionDAO.commitTransaction(transaction._id)
+      transactionBox <- creditTransactionDAO.findTransactionForJob(jobId).shiftBox
+      _ <- transactionBox match {
+        case Full(transaction) =>
+          for {
+            _ <- organizationService.assertOrganizationHasPaidPlan(transaction._organization)
+            _ <- creditTransactionDAO.commitTransaction(transaction._id)
+          } yield ()
+        case Empty      => Fox.successful(()) // Assume transaction-less Job
+        case f: Failure => f.toFox
+      }
+
     } yield ()
 
   def refundTransactionForJob(jobId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] =
     for {
-      transaction <- creditTransactionDAO.findTransactionForJob(jobId)
-      _ <- refundTransaction(transaction)
+      transactionBox <- creditTransactionDAO.findTransactionForJob(jobId).shiftBox
+      _ <- transactionBox match {
+        case Full(transaction) =>
+          for {
+            _ <- refundTransaction(transaction)
+          } yield ()
+        case Empty      => Fox.successful(()) // Assume transaction-less Job
+        case f: Failure => f.toFox
+      }
     } yield ()
 
   private def refundTransaction(creditTransaction: CreditTransaction)(implicit ctx: DBAccessContext): Fox[Unit] =

conf/evolutions/135-neuroglancer-attachment.sql

@@ -0,0 +1,9 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 134, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT ADD VALUE 'neuroglancerPrecomputed';
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 135;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/135-neuroglancer-attachment.sql

@@ -0,0 +1 @@
+-- Removing enum types directly is not possible so no reversion is available for this.

conf/messages

@@ -265,6 +265,10 @@ mesh.file.listChunks.failed=Failed to load chunk list for segment {0} from mesh
 mesh.file.loadChunk.failed=Failed to load mesh chunk for segment
 mesh.file.open.failed=Failed to open mesh file for reading
 mesh.file.readEncoding.failed=Failed to read encoding from mesh file
+mesh.file.lookup.failed=Failed to look up mesh file “{0}”
+mesh.file.readVersion.failed=Failed to read format version from file “{0}”
+mesh.file.readMappingName.failed=Failed to read mapping name from mesh file “{0}”
+mesh.meshFileName.required=Trying to load mesh from mesh file, but mesh file name was not supplied.
 
 task.create.noTasks=Zero tasks were requested
 task.create.failed=Failed to create Task
@@ -347,7 +351,9 @@ job.trainModel.notAllowed.organization = Training AI models is only allowed for
 job.runInference.notAllowed.organization = Running inference is only allowed for datasets of your own organization.
 job.paidJob.notAllowed.noPaidPlan = You are not allowed to run this job because your organization does not have a paid plan.
 job.notEnoughCredits = Your organization does not have enough WEBKNOSSOS credits to run this job.
-creditTransaction.notPaidPlan = Your organization does not have a paid plan.
+job.updateStatus.failed = Failed to update long-running job’s status
+job.creditTransaction.failed = Failed to perform credit transaction
+job.creditTransaction.refund.failed = Failed to perform credit transaction refund
 
 voxelytics.disabled = Voxelytics workflow reporting and logging are not enabled for this WEBKNOSSOS instance.
 voxelytics.runNotFound = Workflow runs not found
@@ -368,10 +374,10 @@ folder.notFound=Could not find the requested folder
 folder.delete.root=Cannot delete the organization’s root folder
 folder.move.root=Cannot move the organization’s root folder
 folder.update.notAllowed=No write access on this folder
+folder.noWriteAccess=No write access in this folder
 folder.update.name.failed=Failed to update the folder’s name
 folder.update.teams.failed=Failed to update the folder’s allowed teams
 folder.create.failed.teams.failed=Failed to create folder in this location
-folder.noWriteAccess=No write access in this folder
 folder.nameMustNotContainSlash=Folder names cannot contain forward slashes
 
 segmentAnything.notEnabled=AI based quick select is not enabled for this WEBKNOSSOS instance.

frontend/javascripts/admin/api/mesh.ts

@@ -22,7 +22,7 @@ type MeshSegmentInfo = {
 };
 
 type ListMeshChunksRequest = {
-  meshFile: APIMeshFileInfo;
+  meshFileName: string;
   segmentId: number;
 };
 
@@ -52,7 +52,7 @@ export function getMeshfileChunksForSegment(
       params.append("editableMappingTracingId", editableMappingTracingId);
     }
     const payload: ListMeshChunksRequest = {
-      meshFile,
+      meshFileName: meshFile.name,
       segmentId,
     };
     return Request.sendJSONReceiveJSON(
@@ -72,7 +72,7 @@ type MeshChunkDataRequest = {
 };
 
 type MeshChunkDataRequestList = {
-  meshFile: APIMeshFileInfo;
+  meshFileName: string;
   requests: MeshChunkDataRequest[];
 };
 

frontend/javascripts/types/api_types.ts

@@ -1005,12 +1005,10 @@ export type ServerEditableMapping = {
 
 export type APIMeshFileInfo = {
   name: string;
-  path: string | null | undefined;
-  fileType: string | null | undefined;
   mappingName?: string | null | undefined;
-  // 0   - is the first mesh file version
-  // 1-2 - the format should behave as v0 (refer to voxelytics for actual differences)
-  // 3   - is the newer version with draco encoding.
+  // 0   - unsupported (is the first mesh file version)
+  // 1-2 - unsupported (the format should behave as v0; refer to voxelytics for actual differences)
+  // 3+  - is the newer version with draco encoding.
   formatVersion: number;
 };
 export type APIConnectomeFile = {

frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts

@@ -365,7 +365,7 @@ function* loadPrecomputedMeshesInChunksForLod(
           dataset,
           getBaseSegmentationName(segmentationLayer),
           {
-            meshFile,
+            meshFileName: meshFile.name,
             // Only extract the relevant properties
             requests: chunks.map(({ byteOffset, byteSize }) => ({
               byteOffset,

frontend/javascripts/viewer/view/action-bar/create_animation_modal.tsx

@@ -258,7 +258,7 @@ function CreateAnimationModal(props: Props) {
       const adhocMagIndex = getMagInfo(layer.resolutions).getClosestExistingIndex(
         preferredQualityForMeshAdHocComputation,
       );
-      const adhocMag = layer.resolutions[adhocMagIndex];
+      const adhocMag = getMagInfo(layer.resolutions).getMagByIndexOrThrow(adhocMagIndex);
 
       return Object.values(meshInfos)
         .filter((meshInfo: MeshInformation) => meshInfo.isVisible)

tools/postgres/schema.sql

@@ -21,7 +21,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(134);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(135);
 COMMIT TRANSACTION;
 
 
@@ -163,7 +163,7 @@ CREATE TABLE webknossos.dataset_layer_additionalAxes(
 );
 
 CREATE TYPE webknossos.LAYER_ATTACHMENT_TYPE AS ENUM ('agglomerate', 'connectome', 'segmentIndex', 'mesh', 'cumsum');
-CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json');
+CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json', 'neuroglancerPrecomputed');
 CREATE TABLE webknossos.dataset_layer_attachments(
    _dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
    layerName TEXT NOT NULL,

unreleased_changes/8682.md

@@ -0,0 +1,5 @@
+### Added
+- Precomputed Meshes can now also be read from the new zarr3-based format, and from remote object storage.
+
+### Postgres Evolutions
+- [135-neuroglancer-attachment.sql](conf/evolutions/135-neuroglancer-attachment.sql)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala

@@ -6,8 +6,10 @@ import com.google.inject.name.Names
 import com.scalableminds.webknossos.datastore.services._
 import com.scalableminds.webknossos.datastore.services.mesh.{
   AdHocMeshServiceHolder,
+  Hdf5MeshFileService,
   MeshFileService,
-  NeuroglancerPrecomputedMeshFileService
+  NeuroglancerPrecomputedMeshFileService,
+  ZarrMeshFileService
 }
 import com.scalableminds.webknossos.datastore.services.uploading.UploadService
 import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptorService}
@@ -31,8 +33,14 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[ApplicationHealthService]).asEagerSingleton()
     bind(classOf[DSDatasetErrorLoggingService]).asEagerSingleton()
     bind(classOf[MeshFileService]).asEagerSingleton()
+    bind(classOf[ZarrMeshFileService]).asEagerSingleton()
+    bind(classOf[Hdf5MeshFileService]).asEagerSingleton()
+    bind(classOf[AgglomerateService]).asEagerSingleton()
+    bind(classOf[ZarrAgglomerateService]).asEagerSingleton()
+    bind(classOf[Hdf5AgglomerateService]).asEagerSingleton()
     bind(classOf[NeuroglancerPrecomputedMeshFileService]).asEagerSingleton()
     bind(classOf[RemoteSourceDescriptorService]).asEagerSingleton()
+    bind(classOf[ChunkCacheService]).asEagerSingleton()
     bind(classOf[DatasetCache]).asEagerSingleton()
   }
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala

@@ -1,7 +1,6 @@
 package com.scalableminds.webknossos.datastore.controllers
 
 import com.google.inject.Inject
-import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.services._
 import com.scalableminds.webknossos.datastore.services.mesh.{
@@ -10,8 +9,7 @@ import com.scalableminds.webknossos.datastore.services.mesh.{
   ListMeshChunksRequest,
   MeshChunkDataRequestList,
   MeshFileService,
-  MeshMappingHelper,
-  NeuroglancerPrecomputedMeshFileService
+  MeshMappingHelper
 }
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
@@ -21,7 +19,6 @@ import scala.concurrent.ExecutionContext
 class DSMeshController @Inject()(
     accessTokenService: DataStoreAccessTokenService,
     meshFileService: MeshFileService,
-    neuroglancerPrecomputedMeshService: NeuroglancerPrecomputedMeshFileService,
     fullMeshService: DSFullMeshService,
     dataSourceRepository: DataSourceRepository,
     val dsRemoteWebknossosClient: DSRemoteWebknossosClient,
@@ -38,22 +35,21 @@ class DSMeshController @Inject()(
       accessTokenService.validateAccessFromTokenContext(
         UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
         for {
-          meshFiles <- meshFileService.exploreMeshFiles(organizationId, datasetDirectoryName, dataLayerName)
-          neuroglancerMeshFiles <- neuroglancerPrecomputedMeshService.exploreMeshFiles(organizationId,
-                                                                                       datasetDirectoryName,
-                                                                                       dataLayerName)
-          allMeshFiles = meshFiles ++ neuroglancerMeshFiles
-        } yield Ok(Json.toJson(allMeshFiles))
+          (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
+                                                                                    datasetDirectoryName,
+                                                                                    dataLayerName)
+          meshFileInfos <- meshFileService.listMeshFiles(dataSource.id, dataLayer)
+        } yield Ok(Json.toJson(meshFileInfos))
       }
     }
 
   def listMeshChunksForSegment(organizationId: String,
                                datasetDirectoryName: String,
                                dataLayerName: String,
-                               /* If targetMappingName is set, assume that meshfile contains meshes for
+                               /* If targetMappingName is set, assume that meshFile contains meshes for
                                             the oversegmentation. Collect mesh chunks of all *unmapped* segment ids
                                             belonging to the supplied agglomerate id.
-                                            If it is not set, use meshfile as is, assume passed id is present in meshfile
+                                            If it is not set, use meshFile as is, assume passed id is present in meshFile
                                    Note: in case of an editable mapping, targetMappingName is its baseMapping name.
                                 */
                                targetMappingName: Option[String],
@@ -62,14 +58,11 @@ class DSMeshController @Inject()(
       accessTokenService.validateAccessFromTokenContext(
         UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
         for {
-          _ <- Fox.successful(())
-          mappingNameForMeshFile = meshFileService.mappingNameForMeshFile(organizationId,
-                                                                          datasetDirectoryName,
-                                                                          dataLayerName,
-                                                                          request.body.meshFile.name)
           (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
                                                                                     datasetDirectoryName,
                                                                                     dataLayerName)
+          meshFileKey <- meshFileService.lookUpMeshFileKey(dataSource.id, dataLayer, request.body.meshFileName)
+          mappingNameForMeshFile <- meshFileService.mappingNameForMeshFile(meshFileKey)
           segmentIds: Seq[Long] <- segmentIdsForAgglomerateIdIfNeeded(
             dataSource.id,
             dataLayer,
@@ -79,15 +72,7 @@ class DSMeshController @Inject()(
             mappingNameForMeshFile,
             omitMissing = false
           )
-          chunkInfos <- if (request.body.meshFile.isNeuroglancerPrecomputed) {
-            neuroglancerPrecomputedMeshService.listMeshChunksForMultipleSegments(request.body.meshFile.path, segmentIds)
-          } else {
-            meshFileService.listMeshChunksForSegmentsMerged(organizationId,
-                                                            datasetDirectoryName,
-                                                            dataLayerName,
-                                                            request.body.meshFile.name,
-                                                            segmentIds)
-          }
+          chunkInfos <- meshFileService.listMeshChunksForSegmentsMerged(meshFileKey, segmentIds)
         } yield Ok(Json.toJson(chunkInfos))
       }
     }
@@ -99,13 +84,11 @@ class DSMeshController @Inject()(
       accessTokenService.validateAccessFromTokenContext(
         UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
         for {
-          (data, encoding) <- if (request.body.meshFile.isNeuroglancerPrecomputed) {
-            neuroglancerPrecomputedMeshService.readMeshChunk(request.body.meshFile.path, request.body.requests)
-          } else {
-            meshFileService
-              .readMeshChunk(organizationId, datasetDirectoryName, dataLayerName, request.body)
-              .toFox ?~> "mesh.file.loadChunk.failed"
-          }
+          (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
+                                                                                    datasetDirectoryName,
+                                                                                    dataLayerName)
+          meshFileKey <- meshFileService.lookUpMeshFileKey(dataSource.id, dataLayer, request.body.meshFileName)
+          (data, encoding) <- meshFileService.readMeshChunk(meshFileKey, request.body.requests) ?~> "mesh.file.loadChunk.failed"
         } yield {
           if (encoding.contains("gzip")) {
             Ok(data).withHeaders("Content-Encoding" -> "gzip")

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -266,7 +266,7 @@ class DataSourceController @Inject()(
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
                                                                                   datasetDirectoryName,
                                                                                   dataLayerName)
-        agglomerateList = agglomerateService.listAgglomerates(dataSource.id, dataLayer)
+        agglomerateList = agglomerateService.listAgglomeratesFiles(dataSource.id, dataLayer)
       } yield Ok(Json.toJson(agglomerateList))
     }
   }
@@ -450,7 +450,8 @@ class DataSourceController @Inject()(
                                       layerName: Option[String]): InboxDataSource = {
     val (closedAgglomerateFileHandleCount, clearedBucketProviderCount, removedChunksCount) =
       binaryDataServiceHolder.binaryDataService.clearCache(organizationId, datasetDirectoryName, layerName)
-    val closedMeshFileHandleCount = meshFileService.clearCache(organizationId, datasetDirectoryName, layerName)
+    val closedMeshFileHandleCount =
+      meshFileService.clearCache(DataSourceId(organizationId, datasetDirectoryName), layerName)
     val reloadedDataSource: InboxDataSource = dataSourceService.dataSourceFromDir(
       dataSourceService.dataBaseDir.resolve(organizationId).resolve(datasetDirectoryName),
       organizationId)