Skip to content

Commit 9f89d38

Browse files
frcrothfrcroth
committed
Explore remote datasets as virtual datasets
1 parent 6096fe4 commit 9f89d38

File tree

18 files changed

+313
-65
lines changed

18 files changed

+313
-65
lines changed

‎app/controllers/DatasetController.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,10 @@ class DatasetController @Inject()(userService: UserService,
145145
_ <- Fox.fromBool(dataSource.dataLayers.nonEmpty) ?~> "dataset.explore.zeroLayers"
146146
folderIdOpt <- Fox.runOptional(request.body.folderPath)(folderPath =>
147147
folderService.getOrCreateFromPathLiteral(folderPath, request.identity._organization)) ?~> "dataset.explore.autoAdd.getFolder.failed"
148-
_ <- wkExploreRemoteLayerService.addRemoteDatasource(dataSource,
149-
request.body.datasetName,
150-
request.identity,
151-
folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
148+
_ <- wkExploreRemoteLayerService.addRemoteDatasourceToDatabase(dataSource,
149+
request.body.datasetName,
150+
request.identity,
151+
folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
152152
} yield Ok
153153
}
154154

‎app/controllers/WKRemoteDataStoreController.scala

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import com.scalableminds.webknossos.datastore.helpers.{LayerMagLinkInfo, MagLink
99
import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
1010
import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
1111
import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
12-
import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus}
12+
import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataSourceRegistrationInfo, DataStoreStatus}
1313
import com.scalableminds.webknossos.datastore.services.uploading.{
1414
LinkedLayerIdentifier,
1515
ReserveAdditionalInformation,
@@ -270,6 +270,34 @@ class WKRemoteDataStoreController @Inject()(
270270

271271
}
272272

273+
// Register a datasource from the datastore as a dataset in the database.
274+
// This is called when adding remote virtual datasets (that should only exist in the database)
275+
// by the data store after exploration.
276+
def registerDataSource(name: String,
277+
key: String,
278+
organizationId: String,
279+
directoryName: String,
280+
token: String): Action[DataSourceRegistrationInfo] =
281+
Action.async(validateJson[DataSourceRegistrationInfo]) { implicit request =>
282+
dataStoreService.validateAccess(name, key) { dataStore =>
283+
for {
284+
user <- bearerTokenService.userForToken(token)
285+
organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> Messages(
286+
"organization.notFound",
287+
organizationId) ~> NOT_FOUND
288+
_ <- Fox.fromBool(organization._id == user._organization) ?~> "notAllowed" ~> FORBIDDEN
289+
dataset <- datasetService.createVirtualDataset(
290+
directoryName,
291+
organizationId,
292+
dataStore,
293+
request.body.dataSource,
294+
request.body.folderId,
295+
user
296+
)
297+
} yield Ok(dataset._id.toString)
298+
}
299+
}
300+
273301
def jobExportProperties(name: String, key: String, jobId: ObjectId): Action[AnyContent] = Action.async {
274302
implicit request =>
275303
dataStoreService.validateAccess(name, key) { _ =>

‎app/models/dataset/DataStore.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@ class DataStoreService @Inject()(dataStoreDAO: DataStoreDAO, jobService: JobServ
7979

8080
def validateAccess(name: String, key: String)(block: DataStore => Future[Result])(
8181
implicit m: MessagesProvider): Fox[Result] =
82-
Fox.fromFuture((for {
83-
dataStore <- dataStoreDAO.findOneByName(name)(GlobalAccessContext)
82+
Fox.fromFuture((for {dataStore <- dataStoreDAO.findOneByName(name)(GlobalAccessContext)
8483
_ <- Fox.fromBool(key == dataStore.key)
8584
result <- Fox.fromFuture(block(dataStore))
8685
} yield result).getOrElse(Forbidden(Json.obj("granted" -> false, "msg" -> Messages("dataStore.notFound")))))

‎app/models/dataset/DatasetService.scala

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package models.dataset
22

3-
import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
3+
import com.scalableminds.util.accesscontext.{AuthorizedAccessContext, DBAccessContext, GlobalAccessContext}
44
import com.scalableminds.util.objectid.ObjectId
55
import com.scalableminds.util.time.Instant
66
import com.scalableminds.util.tools.{Fox, FoxImplicits}
@@ -23,6 +23,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
2323
AbstractDataLayer,
2424
AbstractSegmentationLayer,
2525
DataFormat,
26+
DataSource,
2627
DataSourceId,
2728
GenericDataSource,
2829
DataLayerLike => DataLayer
@@ -36,6 +37,7 @@ import models.team._
3637
import models.user.{User, UserService}
3738
import net.liftweb.common.Box.tryo
3839
import net.liftweb.common.{Empty, EmptyBox, Full}
40+
import play.api.i18n.Messages
3941
import play.api.libs.json.{JsObject, Json}
4042
import security.RandomIDGenerator
4143
import utils.WkConf
@@ -97,6 +99,34 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
9799
} yield newDataset
98100
}
99101

102+
private def virtualRemoteDatasetStatus = "Virtual remote dataset"
103+
104+
def createVirtualDataset(datasetName: String,
105+
organizationId: String,
106+
dataStore: DataStore,
107+
dataSource: DataSource,
108+
folderId: Option[String],
109+
user: User): Fox[Dataset] =
110+
for {
111+
_ <- assertValidDatasetName(datasetName)
112+
isDatasetNameAlreadyTaken <- datasetDAO.doesDatasetDirectoryExistInOrganization(datasetName, organizationId)(
113+
GlobalAccessContext)
114+
_ <- Fox.fromBool(!isDatasetNameAlreadyTaken) ?~> "dataset.name.alreadyTaken"
115+
organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> "organization.notFound"
116+
folderId <- ObjectId.fromString(folderId.getOrElse(organization._rootFolder.toString)) ?~> "dataset.upload.folderId.invalid"
117+
_ <- folderDAO.assertUpdateAccess(folderId)(AuthorizedAccessContext(user)) ?~> "folder.noWriteAccess"
118+
newDatasetId = ObjectId.generate
119+
abstractDataSource = dataSource.copy(dataLayers = dataSource.dataLayers.map(_.asAbstractLayer))
120+
dataset <- createDataset(dataStore,
121+
newDatasetId,
122+
datasetName,
123+
abstractDataSource,
124+
status = Some(virtualRemoteDatasetStatus))
125+
datasetId = dataset._id
126+
_ <- datasetDAO.updateFolder(datasetId, folderId)(GlobalAccessContext)
127+
_ <- addUploader(dataset, user._id)(GlobalAccessContext)
128+
} yield dataset
129+
100130
def getAllUnfinishedDatasetUploadsOfUser(userId: ObjectId, organizationId: String)(
101131
implicit ctx: DBAccessContext): Fox[List[DatasetCompactInfo]] =
102132
datasetDAO.findAllCompactWithSearch(
@@ -114,7 +144,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
114144
datasetId: ObjectId,
115145
datasetName: String,
116146
dataSource: InboxDataSource,
117-
publication: Option[ObjectId] = None
147+
publication: Option[ObjectId] = None,
148+
status: Option[String] = None
118149
): Fox[Dataset] = {
119150
implicit val ctx: DBAccessContext = GlobalAccessContext
120151
val metadata =
@@ -147,7 +178,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
147178
name = datasetName,
148179
voxelSize = dataSource.voxelSizeOpt,
149180
sharingToken = None,
150-
status = dataSource.statusOpt.getOrElse(""),
181+
status = status.orElse(dataSource.statusOpt).getOrElse(""),
151182
logoUrl = None,
152183
metadata = metadata
153184
)

‎app/models/dataset/WKRemoteDataStoreClient.scala

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -82,18 +82,6 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
8282
.silent
8383
.getWithJsonResponse[List[DirectoryStorageReport]]
8484

85-
def addDataSource(organizationId: String,
86-
datasetName: String,
87-
dataSource: GenericDataSource[DataLayer],
88-
folderId: Option[ObjectId],
89-
userToken: String): Fox[Unit] =
90-
for {
91-
_ <- rpc(s"${dataStore.url}/data/datasets/$organizationId/$datasetName")
92-
.addQueryString("token" -> userToken)
93-
.addQueryStringOptional("folderId", folderId.map(_.toString))
94-
.postJson(dataSource)
95-
} yield ()
96-
9785
def hasSegmentIndexFile(organizationId: String, datasetName: String, layerName: String)(
9886
implicit ec: ExecutionContext): Fox[Boolean] = {
9987
val cacheKey = (organizationId, datasetName, layerName)

‎app/models/dataset/explore/WKExploreRemoteLayerService.scala

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -105,17 +105,22 @@ class WKExploreRemoteLayerService @Inject()(credentialService: CredentialService
105105
credentialId <- Fox.runOptional(credentialOpt)(c => credentialService.insertOne(c)) ?~> "dataVault.credential.insert.failed"
106106
} yield credentialId
107107

108-
def addRemoteDatasource(dataSource: GenericDataSource[DataLayer],
109-
datasetName: String,
110-
user: User,
111-
folderId: Option[ObjectId])(implicit ctx: DBAccessContext): Fox[Unit] =
108+
def addRemoteDatasourceToDatabase(dataSource: GenericDataSource[DataLayer],
109+
datasetName: String,
110+
user: User,
111+
folderId: Option[ObjectId])(implicit ctx: DBAccessContext): Fox[Unit] =
112112
for {
113-
organization <- organizationDAO.findOne(user._organization)
114113
dataStore <- dataStoreDAO.findOneWithUploadsAllowed
114+
organizationId = user._organization
115115
_ <- datasetService.assertValidDatasetName(datasetName)
116-
client = new WKRemoteDataStoreClient(dataStore, rpc)
117-
userToken <- bearerTokenService.createAndInitDataStoreTokenForUser(user)
118-
_ <- client.addDataSource(organization._id, datasetName, dataSource, folderId, userToken)
119-
} yield ()
116+
datasetId <- datasetService.createVirtualDataset(
117+
dataSource.id.directoryName,
118+
organizationId,
119+
dataStore,
120+
dataSource,
121+
folderId.map(_.toString),
122+
user
123+
)
124+
} yield datasetId
120125

121126
}

‎conf/webknossos.latest.routes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ PUT /datastores/:name/datasources
111111
PUT /datastores/:name/datasources/paths controllers.WKRemoteDataStoreController.updatePaths(name: String, key: String)
112112
GET /datastores/:name/datasources/:organizationId/:directoryName/paths controllers.WKRemoteDataStoreController.getPaths(name: String, key: String, organizationId: String, directoryName: String)
113113
GET /datastores/:name/datasources/:datasetId controllers.WKRemoteDataStoreController.getDataSource(name: String, key: String, datasetId: ObjectId)
114+
POST /datastores/:name/datasources/:organizationId/:directoryName controllers.WKRemoteDataStoreController.registerDataSource(name: String, key: String, organizationId: String, directoryName: String, token: String)
114115
PATCH /datastores/:name/status controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String)
115116
POST /datastores/:name/reserveUpload controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String)
116117
GET /datastores/:name/getUnfinishedUploadsForUser controllers.WKRemoteDataStoreController.getUnfinishedUploadsForUser(name: String, key: String, token: String, organizationName: String)

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -387,32 +387,17 @@ class DataSourceController @Inject()(
387387
}
388388
}
389389

390-
// Stores a remote dataset in the database.
390+
// Called by the frontend after the user has set datasetName / FolderId of an explored dataSource
391+
// Add this data source to the WK database
391392
def add(organizationId: String, datasetName: String, folderId: Option[String]): Action[DataSource] =
392393
Action.async(validateJson[DataSource]) { implicit request =>
393394
accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources) {
394395
for {
395-
reservedAdditionalInfo <- dsRemoteWebknossosClient.reserveDataSourceUpload(
396-
ReserveUploadInformation(
397-
uploadId = "", // Set by core backend
398-
name = datasetName,
399-
organization = organizationId,
400-
totalFileCount = 1,
401-
filePaths = None,
402-
totalFileSizeInBytes = None,
403-
layersToLink = None,
404-
initialTeams = List.empty,
405-
folderId = folderId,
406-
requireUniqueName = Some(false),
407-
)
408-
) ?~> "dataset.upload.validation.failed"
409-
datasourceId = DataSourceId(reservedAdditionalInfo.directoryName, organizationId)
410-
_ <- dataSourceService.updateDataSource(request.body.copy(id = datasourceId), expectExisting = false)
411-
uploadedDatasetId <- dsRemoteWebknossosClient.reportUpload(datasourceId,
412-
0L,
413-
needsConversion = false,
414-
viaAddRoute = true) ?~> "reportUpload.failed"
415-
} yield Ok(Json.obj("newDatasetId" -> uploadedDatasetId))
396+
_ <- Fox.successful(())
397+
dataSourceId = DataSourceId(datasetName, organizationId)
398+
dataSource = request.body.copy(id = dataSourceId)
399+
datasetId <- dsRemoteWebknossosClient.registerDataSource(dataSource, dataSourceId, folderId) ?~> "dataset.add.failed"
400+
} yield Ok(Json.obj("newDatasetId" -> datasetId))
416401
}
417402
}
418403

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/N5DataLayers.scala

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,24 @@ case class N5DataLayer(
3737
override val numChannels: Option[Int] = Some(1),
3838
additionalAxes: Option[Seq[AdditionalAxis]] = None,
3939
attachments: Option[DatasetLayerAttachments] = None,
40-
) extends N5Layer
40+
) extends N5Layer {
41+
override def asAbstractLayer: DataLayerLike =
42+
AbstractDataLayer(
43+
name,
44+
category,
45+
boundingBox,
46+
resolutions,
47+
elementClass,
48+
defaultViewConfiguration,
49+
adminViewConfiguration,
50+
coordinateTransformations,
51+
additionalAxes,
52+
attachments,
53+
Some(mags),
54+
numChannels,
55+
Some(dataFormat)
56+
)
57+
}
4158

4259
object N5DataLayer {
4360
implicit val jsonFormat: OFormat[N5DataLayer] = Json.format[N5DataLayer]
@@ -57,7 +74,26 @@ case class N5SegmentationLayer(
5774
additionalAxes: Option[Seq[AdditionalAxis]] = None,
5875
attachments: Option[DatasetLayerAttachments] = None,
5976
) extends SegmentationLayer
60-
with N5Layer
77+
with N5Layer {
78+
override def asAbstractLayer: DataLayerLike =
79+
AbstractSegmentationLayer(
80+
name,
81+
category,
82+
boundingBox,
83+
resolutions,
84+
elementClass,
85+
largestSegmentId,
86+
mappings,
87+
defaultViewConfiguration,
88+
adminViewConfiguration,
89+
coordinateTransformations,
90+
additionalAxes,
91+
attachments,
92+
Some(mags),
93+
numChannels,
94+
Some(dataFormat)
95+
)
96+
}
6197

6298
object N5SegmentationLayer {
6399
implicit val jsonFormat: OFormat[N5SegmentationLayer] = Json.format[N5SegmentationLayer]

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/PrecomputedDataLayers.scala

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,24 @@ case class PrecomputedDataLayer(
3737
override val numChannels: Option[Int] = Some(1),
3838
additionalAxes: Option[Seq[AdditionalAxis]] = None,
3939
attachments: Option[DatasetLayerAttachments] = None,
40-
) extends PrecomputedLayer
40+
) extends PrecomputedLayer {
41+
override def asAbstractLayer: DataLayerLike =
42+
AbstractDataLayer(
43+
name,
44+
category,
45+
boundingBox,
46+
resolutions,
47+
elementClass,
48+
defaultViewConfiguration,
49+
adminViewConfiguration,
50+
coordinateTransformations,
51+
additionalAxes,
52+
attachments,
53+
Some(mags),
54+
numChannels,
55+
Some(dataFormat)
56+
)
57+
}
4158

4259
object PrecomputedDataLayer {
4360
implicit val jsonFormat: OFormat[PrecomputedDataLayer] = Json.format[PrecomputedDataLayer]
@@ -57,7 +74,26 @@ case class PrecomputedSegmentationLayer(
5774
additionalAxes: Option[Seq[AdditionalAxis]] = None,
5875
attachments: Option[DatasetLayerAttachments] = None,
5976
) extends SegmentationLayer
60-
with PrecomputedLayer
77+
with PrecomputedLayer {
78+
override def asAbstractLayer: DataLayerLike =
79+
AbstractSegmentationLayer(
80+
name,
81+
category,
82+
boundingBox,
83+
resolutions,
84+
elementClass,
85+
largestSegmentId,
86+
mappings,
87+
defaultViewConfiguration,
88+
adminViewConfiguration,
89+
coordinateTransformations,
90+
additionalAxes,
91+
attachments,
92+
Some(mags),
93+
numChannels,
94+
Some(dataFormat)
95+
)
96+
}
6197

6298
object PrecomputedSegmentationLayer {
6399
implicit val jsonFormat: OFormat[PrecomputedSegmentationLayer] = Json.format[PrecomputedSegmentationLayer]

0 commit comments

Comments
 (0)