diff --git a/app/controllers/DatasetController.scala b/app/controllers/DatasetController.scala index e48d60ea9d7..b2fb0dbd2ae 100755 --- a/app/controllers/DatasetController.scala +++ b/app/controllers/DatasetController.scala @@ -118,7 +118,8 @@ case class ReserveAttachmentUploadToPathRequest( attachmentName: String, attachmentType: LayerAttachmentType.Value, attachmentDataformat: LayerAttachmentDataformat.Value, - pathPrefix: Option[UPath] + pathPrefix: Option[UPath], + overwritePending: Option[Boolean] = None ) object ReserveAttachmentUploadToPathRequest { @@ -166,14 +167,14 @@ class DatasetController @Inject()(userService: UserService, wKRemoteSegmentAnythingClient: WKRemoteSegmentAnythingClient, teamService: TeamService, datasetDAO: DatasetDAO, - datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO, + datasetLayerAttachmentsDAO: DatasetLayerAttachmentDAO, datasetUploadToPathsService: UploadToPathsService, folderService: FolderService, thumbnailService: ThumbnailService, thumbnailCachingService: ThumbnailCachingService, usedStorageService: UsedStorageService, conf: WkConf, - datasetMagsDAO: DatasetMagsDAO, + datasetMagsDAO: DatasetMagDAO, slackNotificationService: SlackNotificationService, authenticationService: AccessibleBySwitchingService, analyticsService: AnalyticsService, @@ -727,6 +728,7 @@ class DatasetController @Inject()(userService: UserService, for { dataset <- datasetDAO.findOne(datasetId) ?~> notFoundMessage(datasetId.toString) ~> NOT_FOUND _ <- Fox.assertTrue(datasetService.isEditableBy(dataset, Some(request.identity))) ?~> "notAllowed" ~> FORBIDDEN + _ <- datasetMagsDAO.findOneWithPendingUploadToPath(datasetId, request.body.layerName, request.body.mag) ?~> "dataset.finishMagUploadToPath.notPending" _ <- datasetMagsDAO.finishUploadToPath(datasetId, request.body.layerName, request.body.mag) dataStoreClient <- datasetService.clientFor(dataset) _ <- Fox.runIf(!dataset.isVirtual) { @@ -755,10 +757,15 @@ class DatasetController @Inject()(userService: UserService, for { dataset <- datasetDAO.findOne(datasetId) ?~> notFoundMessage(datasetId.toString) ~> NOT_FOUND _ <- Fox.assertTrue(datasetService.isEditableBy(dataset, Some(request.identity))) ?~> "notAllowed" ~> FORBIDDEN + _ <- datasetLayerAttachmentsDAO.findOneWithPendingUploadToPath( + datasetId, + request.body.layerName, + request.body.attachmentType, + request.body.attachmentName) ?~> "dataset.finishAttachmentUploadToPath.notPending" _ <- datasetLayerAttachmentsDAO.finishUploadToPath(datasetId, request.body.layerName, - request.body.attachmentName, - request.body.attachmentType) + request.body.attachmentType, + request.body.attachmentName) dataStoreClient <- datasetService.clientFor(dataset) _ <- Fox.runIf(!dataset.isVirtual) { for { diff --git a/app/controllers/WKRemoteDataStoreController.scala b/app/controllers/WKRemoteDataStoreController.scala index 0cdd213d1b7..a046afa5cb6 100644 --- a/app/controllers/WKRemoteDataStoreController.scala +++ b/app/controllers/WKRemoteDataStoreController.scala @@ -5,18 +5,26 @@ import com.scalableminds.util.objectid.ObjectId import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.controllers.JobExportProperties +import com.scalableminds.webknossos.datastore.helpers.UPath import com.scalableminds.webknossos.datastore.models.UnfinishedUpload import com.scalableminds.webknossos.datastore.models.datasource.{ DataSource, DataSourceId, DataSourceStatus, + LayerAttachmentType, UnusableDataSource } import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus} import com.scalableminds.webknossos.datastore.services.uploading.{ + AttachmentUploadAdditionalInfo, + AttachmentUploadInfo, + DatasetUploadAdditionalInfo, + DatasetUploadInfo, + MagUploadAdditionalInfo, + MagUploadInfo, + ReportAttachmentUploadParameters, ReportDatasetUploadParameters, - ReserveAdditionalInformation, - ReserveUploadInformation + ReportMagUploadParameters } import com.typesafe.scalalogging.LazyLogging import models.dataset._ @@ -47,6 +55,9 @@ class WKRemoteDataStoreController @Inject()( userDAO: UserDAO, teamDAO: TeamDAO, jobDAO: JobDAO, + datasetMagDAO: DatasetMagDAO, + datasetAttachmentDAO: DatasetLayerAttachmentDAO, + uploadToPathsService: UploadToPathsService, jobService: JobService, credentialDAO: CredentialDAO, wkSilhouetteEnvironment: WkSilhouetteEnvironment)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) @@ -56,26 +67,28 @@ class WKRemoteDataStoreController @Inject()( val bearerTokenService: WebknossosBearerTokenAuthenticatorService = wkSilhouetteEnvironment.combinedAuthenticatorService.tokenAuthenticatorService - def reserveDatasetUpload(name: String, key: String, token: String): Action[ReserveUploadInformation] = - Action.async(validateJson[ReserveUploadInformation]) { implicit request => + def reserveDatasetUpload(name: String, key: String, token: String): Action[DatasetUploadInfo] = + Action.async(validateJson[DatasetUploadInfo]) { implicit request => dataStoreService.validateAccess(name, key) { dataStore => val uploadInfo = request.body for { user <- bearerTokenService.userForToken(token) ~> FORBIDDEN - organization <- organizationDAO.findOne(uploadInfo.organization)(GlobalAccessContext) ?~> Messages( + organization <- organizationDAO.findOne(uploadInfo.organizationId)(GlobalAccessContext) ?~> Messages( "organization.notFound", - uploadInfo.organization) ~> NOT_FOUND - _ <- organizationService.assertUsedStorageNotExceeded(organization, uploadInfo.totalFileSizeInBytes) ?~> "dataset.upload.storageExceeded" ~> FORBIDDEN + uploadInfo.organizationId) ~> NOT_FOUND + _ <- organizationService.assertUsedStorageNotExceeded( + organization, + uploadInfo.resumableUploadInfo.totalFileSizeInBytes) ?~> "dataset.upload.storageExceeded" ~> FORBIDDEN _ <- Fox.fromBool(organization._id == user._organization) ?~> "notAllowed" ~> FORBIDDEN - _ <- datasetService.assertValidDatasetName(uploadInfo.name) + _ <- datasetService.assertValidDatasetName(uploadInfo.datasetName) _ <- Fox.fromBool(dataStore.onlyAllowedOrganization.forall(_ == organization._id)) ?~> "dataset.upload.Datastore.restricted" _ <- Fox.serialCombined(uploadInfo.layersToLink.getOrElse(List.empty))(l => layerToLinkService.validateLayerToLink(l, user)) ?~> "dataset.upload.invalidLinkedLayers" _ <- Fox.runIf(request.body.requireUniqueName.getOrElse(false))( - datasetService.assertNewDatasetNameUnique(request.body.name, organization._id)) + datasetService.assertNewDatasetNameUnique(request.body.datasetName, organization._id)) preliminaryDataSource = UnusableDataSource(DataSourceId("", ""), None, DataSourceStatus.notYetUploaded) dataset <- datasetService.createAndSetUpDataset( - uploadInfo.name, + uploadInfo.datasetName, dataStore, preliminaryDataSource, uploadInfo.folderId, @@ -83,16 +96,73 @@ class WKRemoteDataStoreController @Inject()( isVirtual = uploadInfo.isVirtual.getOrElse(true), creationType = DatasetCreationType.Upload ) ?~> "dataset.upload.creation.failed" - _ <- datasetService.addInitialTeams(dataset, uploadInfo.initialTeams, user)(AuthorizedAccessContext(user)) - additionalInfo = ReserveAdditionalInformation(dataset._id, dataset.directoryName) + _ <- datasetService.addInitialTeams(dataset, uploadInfo.initialTeamIds, user)(AuthorizedAccessContext(user)) + additionalInfo = DatasetUploadAdditionalInfo(dataset._id, dataset.directoryName) } yield Ok(Json.toJson(additionalInfo)) } } - def getUnfinishedUploadsForUser(name: String, - key: String, - token: String, - organizationId: String): Action[AnyContent] = + def reserveMagUpload(name: String, key: String, token: String): Action[MagUploadInfo] = + Action.async(validateJson[MagUploadInfo]) { implicit request => + dataStoreService.validateAccess(name, key) { dataStore => + // DS write access was asserted already at this point. + for { + user <- bearerTokenService.userForToken(token) + dataset <- datasetDAO.findOne(request.body.datasetId)(AuthorizedAccessContext(user)) + _ <- Fox.fromBool(dataset.isVirtual) ?~> "dataset.reserveMagUpload.notVirtual" + (dataSource, dataLayer) <- datasetService.getDataSourceAndLayerFor(dataset, request.body.layerName) + _ <- Fox.fromBool(!dataLayer.mags.exists(_.mag.maxDim == request.body.mag.mag.maxDim)) ?~> s"New mag ${request.body.mag.mag} conflicts with existing mag of the layer." + _ <- Fox.fromBool(dataset._dataStore == dataStore.name) ?~> "Cannot upload mag to existing dataset via different datastore." + _ <- uploadToPathsService.handleExistingPendingMag(dataset, + request.body.layerName, + request.body.mag.mag, + request.body.overwritePending) + _ <- datasetMagDAO.insertWithUploadPending(request.body.datasetId, + request.body.layerName, + request.body.mag.mag, + request.body.mag.axisOrder, + request.body.mag.channelIndex) + } yield Ok(Json.toJson(MagUploadAdditionalInfo(dataSource.id))) + } + } + + def reserveAttachmentUpload(name: String, key: String, token: String): Action[AttachmentUploadInfo] = + Action.async(validateJson[AttachmentUploadInfo]) { implicit request => + dataStoreService.validateAccess(name, key) { dataStore => + // DS write access was asserted already at this point. + for { + user <- bearerTokenService.userForToken(token) + dataset <- datasetDAO.findOne(request.body.datasetId)(AuthorizedAccessContext(user)) + _ <- Fox.fromBool(dataset.isVirtual) ?~> "dataset.reserveAttachmentUpload.notVirtual" + (dataSource, dataLayer) <- datasetService.getDataSourceAndLayerFor(dataset, request.body.layerName) + isSingletonAttachment = LayerAttachmentType.isSingletonAttachment(request.body.attachmentType) + existsError = if (isSingletonAttachment) "attachment.singleton.alreadyFilled" else "attachment.name.taken" + existingAttachmentOpt = dataLayer.attachments.flatMap( + _.getByTypeAndNameAlwaysReturnSingletons(request.body.attachmentType, request.body.attachment.name)) + _ <- Fox.fromBool(existingAttachmentOpt.isEmpty) ?~> existsError + _ <- Fox.fromBool(dataset._dataStore == dataStore.name) ?~> "Cannot upload attachment to existing dataset via different datastore." + dummyAttachmentPath <- UPath.fromString("").toFox + _ <- uploadToPathsService.handleExistingPendingAttachment(dataset, + request.body.layerName, + request.body.attachmentType, + request.body.attachment.name, + request.body.overwritePending) + _ <- datasetAttachmentDAO.insertWithUploadPending( + request.body.datasetId, + request.body.layerName, + request.body.attachment.name, + request.body.attachmentType, + request.body.attachment.dataFormat, + dummyAttachmentPath + ) + } yield Ok(Json.toJson(AttachmentUploadAdditionalInfo(dataSource.id))) + } + } + + def getUnfinishedDatasetUploadsForUser(name: String, + key: String, + token: String, + organizationId: String): Action[AnyContent] = Action.async { implicit request => dataStoreService.validateAccess(name, key) { _ => for { @@ -150,14 +220,56 @@ class WKRemoteDataStoreController @Inject()( _ <- Fox.runIf(!request.body.needsConversion)(datasetService.scanRealpathsIfVirtual(updated)) _ <- Fox.runIf(request.body.needsConversion) { for { - voxelSizeFactor <- request.body.voxelSizeFactor.toFox ?~> "dataset.upload.needsConversion.missingVoxelSize" - _ <- jobService.submitConvertToWkwJob(dataset, user, voxelSizeFactor, request.body.voxelSizeUnit) + voxelSize <- request.body.voxelSize.toFox ?~> "dataset.upload.needsConversion.missingVoxelSize" + _ <- jobService.submitConvertToWkwJob(dataset, user, voxelSize) } yield () } } yield Ok } } + def reportMagUpload(name: String, key: String): Action[ReportMagUploadParameters] = + Action.async(validateJson[ReportMagUploadParameters]) { implicit request => + dataStoreService.validateAccess(name, key) { _ => + for { + dataset <- datasetDAO.findOne(request.body.datasetId)(GlobalAccessContext) ?~> Messages( + "dataset.notFound", + request.body.datasetId) ~> NOT_FOUND + _ <- datasetMagDAO.findOneWithPendingUpload(request.body.datasetId, + request.body.layerName, + request.body.mag.mag) ?~> "dataset.finishMagUpload.notPending" + _ <- request.body.mag.path.toFox ?~> "dataset.finishMagUpload.pathNotSet" + _ <- datasetMagDAO.finishUpload(request.body.datasetId, request.body.layerName, request.body.mag) + dataStoreClient <- datasetService.clientFor(dataset)(GlobalAccessContext) + _ <- dataStoreClient.invalidateDatasetInDSCache(dataset._id) + _ <- usedStorageService.refreshStorageReportForDataset(dataset) + } yield Ok + } + } + + def reportAttachmentUpload(name: String, key: String): Action[ReportAttachmentUploadParameters] = + Action.async(validateJson[ReportAttachmentUploadParameters]) { implicit request => + dataStoreService.validateAccess(name, key) { _ => + for { + dataset <- datasetDAO.findOne(request.body.datasetId)(GlobalAccessContext) ?~> Messages( + "dataset.notFound", + request.body.datasetId) ~> NOT_FOUND + _ <- datasetAttachmentDAO.findOneWithPendingUpload( + request.body.datasetId, + request.body.layerName, + request.body.attachmentType, + request.body.attachment.name) ?~> "dataset.finishAttachmentUpload.notPending" + _ <- datasetAttachmentDAO.finishUpload(request.body.datasetId, + request.body.layerName, + request.body.attachmentType, + request.body.attachment) + dataStoreClient <- datasetService.clientFor(dataset)(GlobalAccessContext) + _ <- dataStoreClient.invalidateDatasetInDSCache(dataset._id) + _ <- usedStorageService.refreshStorageReportForDataset(dataset) + } yield Ok + } + } + def statusUpdate(name: String, key: String): Action[DataStoreStatus] = Action.async(validateJson[DataStoreStatus]) { implicit request => dataStoreService.validateAccess(name, key) { _ => diff --git a/app/models/dataset/Dataset.scala b/app/models/dataset/Dataset.scala index ff6a37fc678..1a1a3c84b28 100755 --- a/app/models/dataset/Dataset.scala +++ b/app/models/dataset/Dataset.scala @@ -784,7 +784,7 @@ case class DataSourceMagRow(_dataset: ObjectId, _organization: String, directoryName: String) -class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) +class DatasetMagDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) extends SQLDAO[MagWithPaths, DatasetMagsRow, DatasetMags](sqlClient) { protected val collection = DatasetMags protected def resultConverter = GetResultDatasetMagsRow @@ -806,8 +806,8 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte useRealPaths: Boolean): Fox[List[MagLocator]] = for { rows <- run( - q"""SELECT _dataset, dataLayerName, mag, path, realPath, hasLocalData, axisOrder, channelIndex, credentialId, uploadToPathIsPending - FROM webknossos.dataset_mags WHERE _dataset = $datasetId AND dataLayerName = $dataLayerName AND NOT uploadToPathIsPending""" + q"""SELECT _dataset, dataLayerName, mag, path, realPath, hasLocalData, axisOrder, channelIndex, credentialId, uploadToPathIsPending, uploadIsPending + FROM webknossos.dataset_mags WHERE _dataset = $datasetId AND dataLayerName = $dataLayerName AND NOT uploadToPathIsPending AND NOT uploadIsPending""" .as[DatasetMagsRow]) magLocators <- Fox.combined(rows.map(parseMagLocator(_, useRealPaths))) } yield magLocators @@ -846,11 +846,11 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte def updateMags(datasetId: ObjectId, dataLayers: List[StaticLayer]): Fox[Unit] = { val clearQuery = - q"DELETE FROM webknossos.dataset_mags WHERE _dataset = $datasetId AND NOT uploadToPathIsPending".asUpdate + q"DELETE FROM webknossos.dataset_mags WHERE _dataset = $datasetId AND NOT uploadToPathIsPending AND NOT uploadIsPending".asUpdate val insertQueries = dataLayers.flatMap { layer: StaticLayer => layer.mags.map { mag => - q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, path, axisOrder, channelIndex, credentialId, uploadToPathIsPending) - VALUES($datasetId, ${layer.name}, ${mag.mag}, ${mag.path}, ${mag.axisOrder.map(Json.toJson(_))}, ${mag.channelIndex}, ${mag.credentialId}, ${false}) + q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, path, axisOrder, channelIndex, credentialId, uploadToPathIsPending, uploadIsPending) + VALUES($datasetId, ${layer.name}, ${mag.mag}, ${mag.path}, ${mag.axisOrder.map(Json.toJson(_))}, ${mag.channelIndex}, ${mag.credentialId}, ${false}, ${false}) """.asUpdate } } @@ -958,44 +958,83 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte row.credentialid ) - def insertPending(datasetId: ObjectId, - layerName: String, - mag: Vec3Int, - axisOrder: Option[AxisOrder], - channelIndex: Option[Int], - path: UPath): Fox[Unit] = + def insertWithUploadToPathPending(datasetId: ObjectId, + layerName: String, + mag: Vec3Int, + axisOrder: Option[AxisOrder], + channelIndex: Option[Int], + path: UPath): Fox[Unit] = for { _ <- run( - q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, path, axisOrder, channelIndex, uploadToPathIsPending) - VALUES($datasetId, $layerName, $mag, $path, ${axisOrder.map(Json.toJson(_))}, $channelIndex, ${true}) - """.asUpdate) + q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, path, axisOrder, channelIndex, uploadToPathIsPending, uploadIsPending) + VALUES($datasetId, $layerName, $mag, $path, ${axisOrder.map(Json.toJson(_))}, $channelIndex, ${true}, ${false}) + """.asUpdate) + } yield () + + def insertWithUploadPending(datasetId: ObjectId, + layerName: String, + mag: Vec3Int, + axisOrder: Option[AxisOrder], + channelIndex: Option[Int]): Fox[Unit] = + for { + _ <- run( + q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, path, axisOrder, channelIndex, uploadToPathIsPending, uploadIsPending) + VALUES($datasetId, $layerName, $mag, $None, ${axisOrder + .map(Json.toJson(_))}, $channelIndex, ${false}, ${true})""".asUpdate) } yield () def finishUploadToPath(datasetId: ObjectId, layerName: String, mag: Vec3Int): Fox[Unit] = for { _ <- run( q"""UPDATE webknossos.dataset_mags - SET uploadToPathIsPending = ${false} - WHERE _dataset = $datasetId - AND dataLayerName = $layerName - AND mag = $mag::webknossos.VECTOR3 - AND uploadToPathIsPending""".asUpdate + SET uploadToPathIsPending = ${false}, + uploadIsPending = ${false} + WHERE _dataset = $datasetId + AND dataLayerName = $layerName + AND mag = $mag::webknossos.VECTOR3""".asUpdate ) } yield () - def findPendingMagLocatorPath(datasetId: ObjectId, layerName: String, mag: Vec3Int): Fox[UPath] = + def finishUpload(datasetId: ObjectId, layerName: String, mag: MagLocator): Fox[Unit] = for { - rows <- run(q"""SELECT path - FROM webknossos.dataset_mags - WHERE _dataset = $datasetId - AND dataLayerName = $layerName - AND mag = $mag::webknossos.VECTOR3 - AND uploadToPathIsPending - AND path IS NOT NULL - """.as[String]) - first <- rows.headOption.toFox - firstAsUpath <- UPath.fromString(first).toFox - } yield firstAsUpath + _ <- run( + q"""UPDATE webknossos.dataset_mags + SET uploadToPathIsPending = ${false}, + uploadIsPending = ${false}, + path = ${mag.path} + WHERE _dataset = $datasetId + AND dataLayerName = $layerName + AND mag = ${mag.mag}::webknossos.VECTOR3""".asUpdate + ) + } yield () + + def findOneWithPendingUpload(datasetId: ObjectId, layerName: String, mag: Vec3Int): Fox[MagLocator] = + for { + rows <- run( + q"""SELECT _dataset, dataLayerName, mag, path, realPath, hasLocalData, axisOrder, channelIndex, credentialId, uploadToPathIsPending, uploadIsPending + FROM webknossos.dataset_mags + WHERE _dataset = $datasetId + AND dataLayerName = $layerName + AND mag = $mag::webknossos.VECTOR3 + AND uploadIsPending + LIMIT 1""".as[DatasetMagsRow]) + row <- rows.headOption.toFox + magLocator <- parseMagLocator(row, useRealPaths = true) + } yield magLocator + + def findOneWithPendingUploadToPath(datasetId: ObjectId, layerName: String, mag: Vec3Int): Fox[MagLocator] = + for { + rows <- run( + q"""SELECT _dataset, dataLayerName, mag, path, realPath, hasLocalData, axisOrder, channelIndex, credentialId, uploadToPathIsPending, uploadIsPending + FROM webknossos.dataset_mags + WHERE _dataset = $datasetId + AND dataLayerName = $layerName + AND mag = $mag::webknossos.VECTOR3 + AND uploadToPathIsPending + LIMIT 1""".as[DatasetMagsRow]) + row <- rows.headOption.toFox + magLocator <- parseMagLocator(row, useRealPaths = true) + } yield magLocator def deletePendingMagLocator(datasetId: ObjectId, layerName: String, mag: Vec3Int): Fox[Unit] = for { @@ -1003,16 +1042,16 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte WHERE _dataset = $datasetId AND dataLayerName = $layerName AND mag = $mag::webknossos.VECTOR3 - AND uploadToPathIsPending""".asUpdate) + AND (uploadToPathIsPending OR uploadIsPending)""".asUpdate) } yield () } class DatasetLayerDAO @Inject()(sqlClient: SqlClient, - datasetMagsDAO: DatasetMagsDAO, + datasetMagsDAO: DatasetMagDAO, datasetCoordinateTransformationsDAO: DatasetCoordinateTransformationsDAO, datasetLayerAdditionalAxesDAO: DatasetLayerAdditionalAxesDAO, - datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO)(implicit ec: ExecutionContext) + datasetLayerAttachmentsDAO: DatasetLayerAttachmentDAO)(implicit ec: ExecutionContext) extends SimpleSQLDAO(sqlClient) { private def parseAndFillLayerRow(row: DatasetLayersRow, @@ -1195,13 +1234,13 @@ case class StorageRelevantDataLayerAttachment( datasetDirectoryName: String, ) -class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) +class DatasetLayerAttachmentDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) extends SimpleSQLDAO(sqlClient) { private def parseAttachmentRow(row: DatasetLayerAttachmentsRow, useRealPaths: Boolean): Fox[LayerAttachment] = for { dataFormat <- LayerAttachmentDataformat.fromString(row.dataformat).toFox ?~> "Could not parse data format" - realPathWithFallback = if (useRealPaths) row.realpath.getOrElse(row.path) else (row.path) + realPathWithFallback = if (useRealPaths) row.realpath.getOrElse(row.path) else row.path path <- UPath.fromString(realPathWithFallback).toFox } yield LayerAttachment(row.name, path, dataFormat) @@ -1231,25 +1270,26 @@ class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: Ex useRealPaths: Boolean): Fox[AttachmentWrapper] = for { rows <- run( - q"""SELECT _dataset, layerName, name, path, realpath, hasLocalData, type, dataFormat, uploadToPathIsPending + q"""SELECT _dataset, layerName, name, path, realpath, hasLocalData, type, dataFormat, uploadToPathIsPending, uploadIsPending FROM webknossos.dataset_layer_attachments WHERE _dataset = $datasetId AND layerName = $layerName - AND NOT uploadToPathIsPending""".as[DatasetLayerAttachmentsRow]) + AND NOT uploadToPathIsPending + AND NOT uploadIsPending""".as[DatasetLayerAttachmentsRow]) attachments <- parseAttachments(rows.toList, useRealPaths) ?~> "Could not parse attachments" } yield attachments def updateAttachments(datasetId: ObjectId, dataLayers: List[StaticLayer]): Fox[Unit] = { def insertQuery(attachment: LayerAttachment, layerName: String, attachmentType: LayerAttachmentType.Value) = { val query = - q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat, uploadToPathIsPending) + q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat, uploadToPathIsPending, uploadIsPending) VALUES($datasetId, $layerName, ${attachment.name}, ${attachment.path}, $attachmentType::webknossos.LAYER_ATTACHMENT_TYPE, - ${attachment.dataFormat}::webknossos.LAYER_ATTACHMENT_DATAFORMAT, ${false})""" + ${attachment.dataFormat}::webknossos.LAYER_ATTACHMENT_DATAFORMAT, ${false}, ${false})""" query.asUpdate } val clearQuery = - q"DELETE FROM webknossos.dataset_layer_attachments WHERE _dataset = $datasetId AND NOT uploadToPathIsPending".asUpdate + q"DELETE FROM webknossos.dataset_layer_attachments WHERE _dataset = $datasetId AND NOT uploadToPathIsPending AND NOT uploadIsPending".asUpdate val insertQueries = dataLayers.flatMap { layer: StaticLayer => layer.attachments match { case Some(attachments) => @@ -1289,17 +1329,30 @@ class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: Ex ) } yield () - def insertPending(datasetId: ObjectId, - layerName: String, - attachmentName: String, - attachmentType: LayerAttachmentType.Value, - attachmentDataformat: LayerAttachmentDataformat.Value, - attachmentPath: UPath): Fox[Unit] = + def insertWithUploadToPathPending(datasetId: ObjectId, + layerName: String, + attachmentName: String, + attachmentType: LayerAttachmentType.Value, + attachmentDataformat: LayerAttachmentDataformat.Value, + attachmentPath: UPath): Fox[Unit] = for { _ <- run( - q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat, uploadToPathIsPending) - VALUES($datasetId, $layerName, $attachmentName, $attachmentPath, $attachmentType, $attachmentDataformat, ${true}) - """.asUpdate) + q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat, uploadToPathIsPending, uploadIsPending) + VALUES($datasetId, $layerName, $attachmentName, $attachmentPath, $attachmentType, $attachmentDataformat, ${true}, ${false}) + """.asUpdate) + } yield () + + def insertWithUploadPending(datasetId: ObjectId, + layerName: String, + attachmentName: String, + attachmentType: LayerAttachmentType.Value, + attachmentDataformat: LayerAttachmentDataformat.Value, + attachmentPath: UPath): Fox[Unit] = + for { + _ <- run( + q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat, uploadToPathIsPending, uploadIsPending) + VALUES($datasetId, $layerName, $attachmentName, $attachmentPath, $attachmentType, $attachmentDataformat, ${false}, ${true}) + """.asUpdate) } yield () def countAttachmentsIncludingPending(datasetId: ObjectId, @@ -1321,18 +1374,84 @@ class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: Ex def finishUploadToPath(datasetId: ObjectId, layerName: String, - attachmentName: String, - attachmentType: LayerAttachmentType.Value): Fox[Unit] = + attachmentType: LayerAttachmentType.Value, + attachmentName: String): Fox[Unit] = for { _ <- run(q"""UPDATE webknossos.dataset_layer_attachments - SET uploadToPathIsPending = ${false} + SET uploadToPathIsPending = ${false}, + uploadIsPending = ${false} WHERE _dataset = $datasetId AND layerName = $layerName - AND name = $attachmentName AND type = $attachmentType + AND name = $attachmentName """.asUpdate) } yield () + def findOneWithPendingUpload(datasetId: ObjectId, + layerName: String, + attachmentType: LayerAttachmentType.Value, + attachmentName: String): Fox[LayerAttachment] = + for { + rows <- run( + q"""SELECT _dataset, layerName, name, path, realpath, hasLocalData, type, dataFormat, uploadToPathIsPending, uploadIsPending + FROM webknossos.dataset_layer_attachments + WHERE _dataset = $datasetId + AND layerName = $layerName + AND type = $attachmentType + AND name = $attachmentName + AND uploadIsPending + LIMIT 1""".as[DatasetLayerAttachmentsRow]) + row <- rows.headOption.toFox + attachment <- parseAttachmentRow(row, useRealPaths = false) + } yield attachment + + def findOneWithPendingUploadToPath(datasetId: ObjectId, + layerName: String, + attachmentType: LayerAttachmentType.Value, + attachmentName: String): Fox[LayerAttachment] = + for { + rows <- run( + q"""SELECT _dataset, layerName, name, path, realpath, hasLocalData, type, dataFormat, uploadToPathIsPending, uploadIsPending + FROM webknossos.dataset_layer_attachments + WHERE _dataset = $datasetId + AND layerName = $layerName + AND type = $attachmentType + AND name = $attachmentName + AND uploadToPathIsPending + LIMIT 1""".as[DatasetLayerAttachmentsRow]) + row <- rows.headOption.toFox + attachment <- parseAttachmentRow(row, useRealPaths = true) + } yield attachment + + def deletePendingAttachment(datasetId: ObjectId, + layerName: String, + attachmentType: LayerAttachmentType.Value, + attachmentName: String): Fox[Unit] = + for { + _ <- run(q"""DELETE FROM webknossos.dataset_layer_attachments + WHERE _dataset = $datasetId + AND layerName = $layerName + AND type = $attachmentType + AND name = $attachmentName + AND (uploadIsPending OR uploadToPathIsPending)""".asUpdate) + } yield () + + def finishUpload(datasetId: ObjectId, + layerName: String, + attachmentType: LayerAttachmentType.Value, + attachment: LayerAttachment): Fox[Unit] = + for { + _ <- run(q"""UPDATE webknossos.dataset_layer_attachments + SET uploadToPathIsPending = ${false}, + uploadIsPending = ${false}, + path = ${attachment.path} + WHERE _dataset = $datasetId + AND layerName = $layerName + AND type = $attachmentType + AND name = ${attachment.name} + """.asUpdate) + } yield () + implicit def GetResultStorageRelevantDataLayerAttachment: GetResult[StorageRelevantDataLayerAttachment] = GetResult( r => diff --git a/app/models/dataset/DatasetService.scala b/app/models/dataset/DatasetService.scala index 0ee780a4902..f77cd37ac90 100644 --- a/app/models/dataset/DatasetService.scala +++ b/app/models/dataset/DatasetService.scala @@ -49,8 +49,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO, dataStoreDAO: DataStoreDAO, datasetLastUsedTimesDAO: DatasetLastUsedTimesDAO, datasetDataLayerDAO: DatasetLayerDAO, - datasetMagsDAO: DatasetMagsDAO, - datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO, + datasetMagsDAO: DatasetMagDAO, + datasetLayerAttachmentsDAO: DatasetLayerAttachmentDAO, teamDAO: TeamDAO, folderDAO: FolderDAO, multiUserDAO: MultiUserDAO, diff --git a/app/models/dataset/UploadToPathsService.scala b/app/models/dataset/UploadToPathsService.scala index bd9d91893ef..66eb809824a 100644 --- a/app/models/dataset/UploadToPathsService.scala +++ b/app/models/dataset/UploadToPathsService.scala @@ -3,7 +3,7 @@ package models.dataset import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.objectid.ObjectId -import com.scalableminds.util.tools.{Box, Empty, Failure, Fox, FoxImplicits, Full, TextUtils} +import com.scalableminds.util.tools.{Box, Failure, Fox, FoxImplicits, Full, TextUtils} import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.helpers.UPath import com.scalableminds.webknossos.datastore.models.datasource.LayerAttachmentDataformat.LayerAttachmentDataformat @@ -46,8 +46,8 @@ class UploadToPathsService @Inject()(datasetService: DatasetService, datasetDAO: DatasetDAO, dataStoreDAO: DataStoreDAO, layerToLinkService: LayerToLinkService, - datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO, - datasetMagsDAO: DatasetMagsDAO, + datasetLayerAttachmentsDAO: DatasetLayerAttachmentDAO, + datasetMagDAO: DatasetMagDAO, pathDeletionService: PathDeletionService, folderDAO: FolderDAO, conf: WkConf) @@ -235,7 +235,9 @@ class UploadToPathsService @Inject()(datasetService: DatasetService, val defaultDirName = LayerAttachmentType.defaultDirectoryNameFor(attachmentType) val suffix = LayerAttachmentDataformat.suffixFor(attachmentDataformat) val safeAttachmentName = - TextUtils.normalizeStrong(attachmentName).getOrElse(s"$attachmentType-${ObjectId.generate}") + TextUtils + .normalizeStrong(attachmentName) + .getOrElse(s"${attachmentType}__${RandomIDGenerator.generateBlocking(12)}") layerPath / defaultDirName / (safeAttachmentName + suffix) } @@ -253,6 +255,11 @@ class UploadToPathsService @Inject()(datasetService: DatasetService, mp: MessagesProvider): Fox[UPath] = for { _ <- datasetService.usableDataSourceFor(dataset) + _ <- handleExistingPendingAttachment(dataset, + parameters.layerName, + parameters.attachmentType, + parameters.attachmentName, + parameters.overwritePending.getOrElse(false)) isSingletonAttachment = LayerAttachmentType.isSingletonAttachment(parameters.attachmentType) existingAttachmentsCount <- datasetLayerAttachmentsDAO.countAttachmentsIncludingPending( dataset._id, @@ -267,12 +274,12 @@ class UploadToPathsService @Inject()(datasetService: DatasetService, parameters.attachmentDataformat, parameters.attachmentType, datasetPath / parameters.layerName) - _ <- datasetLayerAttachmentsDAO.insertPending(dataset._id, - parameters.layerName, - parameters.attachmentName, - parameters.attachmentType, - parameters.attachmentDataformat, - attachmentPath) + _ <- datasetLayerAttachmentsDAO.insertWithUploadToPathPending(dataset._id, + parameters.layerName, + parameters.attachmentName, + parameters.attachmentType, + parameters.attachmentDataformat, + attachmentPath) } yield attachmentPath def reserveMagUploadToPath(dataset: Dataset, parameters: ReserveMagUploadToPathRequest)( @@ -280,36 +287,63 @@ class UploadToPathsService @Inject()(datasetService: DatasetService, mp: MessagesProvider): Fox[UPath] = for { _ <- datasetService.usableDataSourceFor(dataset) - _ <- handleExistingPendingMagIfExists(dataset, parameters.layerName, parameters.mag, parameters.overwritePending) + _ <- handleExistingPendingMag(dataset, parameters.layerName, parameters.mag, parameters.overwritePending) datasetParent <- selectPathPrefixDatasetParent(parameters.pathPrefix, dataset._organization) datasetPath = datasetParent / dataset.directoryName magPath = generateMagPath(parameters.mag, datasetPath / parameters.layerName) - _ <- datasetMagsDAO.insertPending(dataset._id, - parameters.layerName, - parameters.mag, - parameters.axisOrder, - parameters.channelIndex, - magPath) + _ <- datasetMagDAO.insertWithUploadToPathPending(dataset._id, + parameters.layerName, + parameters.mag, + parameters.axisOrder, + parameters.channelIndex, + magPath) } yield magPath - private def handleExistingPendingMagIfExists(dataset: Dataset, - layerName: String, - mag: Vec3Int, - overwritePending: Boolean)(implicit ec: ExecutionContext): Fox[Unit] = + def handleExistingPendingMag(dataset: Dataset, layerName: String, mag: Vec3Int, overwritePending: Boolean)( + implicit ec: ExecutionContext): Fox[Unit] = for { - existingMagLocatorPathBox <- datasetMagsDAO.findPendingMagLocatorPath(dataset._id, layerName, mag).shiftBox - _ <- existingMagLocatorPathBox match { - case Full(existingMagLocatorPath) => - if (overwritePending) { - for { - client <- datasetService.clientFor(dataset)(GlobalAccessContext) - _ <- pathDeletionService.deletePaths(client, Seq(existingMagLocatorPath)) - _ <- datasetMagsDAO.deletePendingMagLocator(dataset._id, layerName, mag) - } yield () - } else Fox.failure("dataset.reserveMagUploadToPath.exists") - case Empty => Fox.successful(()) - case f: Failure => f.toFox - } + withPendingUploadToPathsBox <- datasetMagDAO.findOneWithPendingUploadToPath(dataset._id, layerName, mag).shiftBox + withPendingUploadBox <- datasetMagDAO.findOneWithPendingUpload(dataset._id, layerName, mag).shiftBox + _ <- if (overwritePending) { + for { + _ <- Fox.runOptional(withPendingUploadToPathsBox.toOption) { oldPending => + deletePathsForOldPending(dataset, oldPending.path) + } + _ <- datasetMagDAO.deletePendingMagLocator(dataset._id, layerName, mag) + } yield () + } else + Fox.runIf(withPendingUploadToPathsBox.isDefined || withPendingUploadBox.isDefined) { + Fox.failure("Conflict with existing pending mag. Pass overwritePending to overwrite.") + } + } yield () + + def handleExistingPendingAttachment(dataset: Dataset, + layerName: String, + attachmentType: LayerAttachmentType, + attachmentName: String, + overwritePending: Boolean)(implicit ec: ExecutionContext): Fox[Unit] = + for { + withPendingUploadToPathsBox <- datasetLayerAttachmentsDAO + .findOneWithPendingUploadToPath(dataset._id, layerName, attachmentType, attachmentName) + .shiftBox + withPendingUploadBox <- datasetLayerAttachmentsDAO + .findOneWithPendingUpload(dataset._id, layerName, attachmentType, attachmentName) + .shiftBox + _ <- if (overwritePending) { + datasetLayerAttachmentsDAO.deletePendingAttachment(dataset._id, layerName, attachmentType, attachmentName) + } else + Fox.runIf(withPendingUploadToPathsBox.isDefined || withPendingUploadBox.isDefined) { + Fox.failure("Conflict with existing pending attachment. Pass overwritePending to overwrite.") + } } yield () + private def deletePathsForOldPending(dataset: Dataset, pathOpt: Option[UPath])( + implicit ec: ExecutionContext): Fox[_] = + Fox.runOptional(pathOpt) { path => + for { + client <- datasetService.clientFor(dataset)(GlobalAccessContext) + _ <- pathDeletionService.deletePaths(client, Seq(path)) + } yield () + } + } diff --git a/app/models/job/JobService.scala b/app/models/job/JobService.scala index b0b8b46fdd0..0774577394b 100644 --- a/app/models/job/JobService.scala +++ b/app/models/job/JobService.scala @@ -1,7 +1,7 @@ package models.job import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} -import com.scalableminds.util.geometry.{BoundingBox, Vec3Double} +import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.webknossos.datastore.models.VoxelSize import models.dataset.Dataset import com.scalableminds.util.mvc.Formatter @@ -15,7 +15,6 @@ import models.job.JobCommand.JobCommand import models.organization.{CreditTransactionService, OrganizationDAO} import models.user.{MultiUserDAO, User, UserDAO, UserService} import com.scalableminds.util.tools.Full -import com.scalableminds.webknossos.datastore.models.LengthUnit.LengthUnit import org.apache.pekko.actor.ActorSystem import play.api.libs.json.{JsObject, JsValue, Json} import security.WkSilhouetteEnvironment @@ -230,13 +229,9 @@ class JobService @Inject()(wkConf: WkConf, _ = analyticsService.track(RunJobEvent(owner, command)) } yield job - def submitConvertToWkwJob(dataset: Dataset, - user: User, - voxelSizeFactor: Vec3Double, - voxelSizeUnit: Option[LengthUnit]): Fox[Unit] = + def submitConvertToWkwJob(dataset: Dataset, user: User, voxelSize: VoxelSize): Fox[Unit] = for { organization <- organizationDAO.findOne(dataset._organization)(GlobalAccessContext) ?~> "organization.notFound" - voxelSize = VoxelSize.fromFactorAndUnitWithDefault(voxelSizeFactor, voxelSizeUnit) commandArgs = Json.obj( "organization_id" -> organization._id, "organization_display_name" -> organization.name, diff --git a/app/models/storage/UsedStorageService.scala b/app/models/storage/UsedStorageService.scala index d4a4f9db436..372a0b5012a 100644 --- a/app/models/storage/UsedStorageService.scala +++ b/app/models/storage/UsedStorageService.scala @@ -14,8 +14,8 @@ import models.dataset.{ DataStore, DataStoreDAO, Dataset, - DatasetLayerAttachmentsDAO, - DatasetMagsDAO, + DatasetLayerAttachmentDAO, + DatasetMagDAO, StorageRelevantDataLayerAttachment, WKRemoteDataStoreClient } @@ -34,8 +34,8 @@ class UsedStorageService @Inject()(val actorSystem: ActorSystem, val lifecycle: ApplicationLifecycle, organizationDAO: OrganizationDAO, dataStoreDAO: DataStoreDAO, - datasetMagDAO: DatasetMagsDAO, - datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO, + datasetMagDAO: DatasetMagDAO, + datasetLayerAttachmentsDAO: DatasetLayerAttachmentDAO, rpc: RPC, config: WkConf)(implicit val ec: ExecutionContext) extends LazyLogging diff --git a/app/utils/sql/SimpleSQLDAO.scala b/app/utils/sql/SimpleSQLDAO.scala index 9bf6815f704..19bfc7eb958 100644 --- a/app/utils/sql/SimpleSQLDAO.scala +++ b/app/utils/sql/SimpleSQLDAO.scala @@ -23,7 +23,10 @@ class SimpleSQLDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext implicit protected def sqlInterpolationWrapper(s: StringContext): SqlInterpolator = sqlInterpolation(s) + // Concurrent access for Serializable transactions leads to this error, can be solved by retry. protected lazy val transactionSerializationError = "could not serialize access" + // This error tends to occur only after schema changes (type recreation), e.g. during tests. Can be solved by retry. + private lazy val cacheLookupFailedForTypeError = "cache lookup failed for type" protected def run[R](query: DBIOAction[R, NoStream, Nothing], retryCount: Int = 0, @@ -76,7 +79,7 @@ class SimpleSQLDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext _ <- run( composedQuery.transactionally.withTransactionIsolation(Serializable), retryCount = 50, - retryIfErrorContains = List(transactionSerializationError) + retryIfErrorContains = List(transactionSerializationError, cacheLookupFailedForTypeError) ) } yield () } diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index ed85202a572..269e83291ad 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -139,9 +139,13 @@ PUT /datastores/:name/datasources/realpaths GET /datastores/:name/datasources/:datasetId controllers.WKRemoteDataStoreController.getDataSource(name: String, key: String, datasetId: ObjectId) PUT /datastores/:name/datasources/:datasetId controllers.WKRemoteDataStoreController.updateDataSource(name: String, key: String, datasetId: ObjectId) PATCH /datastores/:name/status controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String) -POST /datastores/:name/reserveUpload controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String) -GET /datastores/:name/getUnfinishedUploadsForUser controllers.WKRemoteDataStoreController.getUnfinishedUploadsForUser(name: String, key: String, token: String, organizationName: String) +POST /datastores/:name/reserveDatasetUpload controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String) +POST /datastores/:name/reserveMagUpload controllers.WKRemoteDataStoreController.reserveMagUpload(name: String, key: String, token: String) +POST /datastores/:name/reserveAttachmentUpload controllers.WKRemoteDataStoreController.reserveAttachmentUpload(name: String, key: String, token: String) +GET /datastores/:name/getUnfinishedDatasetUploadsForUser controllers.WKRemoteDataStoreController.getUnfinishedDatasetUploadsForUser(name: String, key: String, token: String, organizationName: String) POST /datastores/:name/reportDatasetUpload controllers.WKRemoteDataStoreController.reportDatasetUpload(name: String, key: String, token: String, datasetId: ObjectId) +POST /datastores/:name/reportMagUpload controllers.WKRemoteDataStoreController.reportMagUpload(name: String, key: String) +POST /datastores/:name/reportAttachmentUpload controllers.WKRemoteDataStoreController.reportAttachmentUpload(name: String, key: String) POST /datastores/:name/deleteDataset controllers.WKRemoteDataStoreController.deleteDataset(name: String, key: String) GET /datastores/:name/findDatasetId controllers.WKRemoteDataStoreController.findDatasetId(name: String, key: String, datasetDirectoryName: String, organizationId: String) GET /datastores/:name/jobExportProperties controllers.WKRemoteDataStoreController.jobExportProperties(name: String, key: String, jobId: ObjectId) diff --git a/conf/webknossos.versioned.routes b/conf/webknossos.versioned.routes index f7b8feb1d9e..db0938c73e6 100644 --- a/conf/webknossos.versioned.routes +++ b/conf/webknossos.versioned.routes @@ -4,6 +4,7 @@ # Note: keep this in sync with the reported version numbers in the com.scalableminds.util.mvc.ApiVersioning trait # version log + # changed in v14: Dataset upload routes and parameters have been refactored, introduced upload domain # changed in v13: Attachments not mentioned in the dataSource passed to updatePartial will now be deleted. # changed in v12: Dataset upload now expects layersToLink in new format with datasetId instead of orgaId+directoryName # changed in v11: Datasets reserveManualUpload flow via WK side. Note: older versions of the route are *not* supported for security reasons. @@ -17,6 +18,8 @@ # new in v3: annotation info and finish request now take timestamp # new in v2: annotation json contains visibility enum instead of booleans +-> /v14/ webknossos.latest.Routes + -> /v13/ webknossos.latest.Routes PATCH /v12/datasets/:datasetId/updatePartial controllers.LegacyApiController.updatePartialV12(datasetId: ObjectId) diff --git a/frontend/javascripts/admin/dataset/dataset_upload_view.tsx b/frontend/javascripts/admin/dataset/dataset_upload_view.tsx index 46ecf598b3b..68502361855 100644 --- a/frontend/javascripts/admin/dataset/dataset_upload_view.tsx +++ b/frontend/javascripts/admin/dataset/dataset_upload_view.tsx @@ -341,23 +341,26 @@ class DatasetUploadView extends React.Component { : `${dayjs(Date.now()).format("YYYY-MM-DD_HH-mm")}__${newDatasetName}__${getRandomString()}`; const filePaths = formValues.zipFile.map((file) => file.path || ""); const totalFileSizeInBytes = getFileSize(formValues.zipFile); - const reserveUploadInformation = { + const resumableUploadInfo = { uploadId, - name: newDatasetName, - directoryName: "", - newDatasetId: "", - organization: activeUser.organization, totalFileCount: formValues.zipFile.length, filePaths: filePaths, totalFileSizeInBytes, + }; + const datasetUploadInfo = { + resumableUploadInfo, + datasetName: newDatasetName, + organizationId: activeUser.organization, layersToLink: [], - initialTeams: formValues.initialTeams.map((team: APITeam) => team.id), + initialTeamIds: formValues.initialTeams.map((team: APITeam) => team.id), folderId: formValues.targetFolderId, needsConversion: this.state.needsConversion, + voxelSizeFactor: this.state.needsConversion ? formValues.voxelSizeFactor : undefined, + voxelSizeUnit: this.state.needsConversion ? formValues.voxelSizeUnit : undefined, }; const datastoreUrl = formValues.datastoreUrl; await refreshToken(); - await reserveDatasetUpload(datastoreUrl, reserveUploadInformation); + await reserveDatasetUpload(datastoreUrl, datasetUploadInfo); const resumableUpload = await createResumableUpload(datastoreUrl, uploadId); this.setState({ uploadId, @@ -382,17 +385,11 @@ class DatasetUploadView extends React.Component { throw new Error("Form couldn't be initialized."); } - const uploadInfo = { - uploadId, - needsConversion: this.state.needsConversion, - voxelSizeFactor: this.state.needsConversion ? formValues.voxelSizeFactor : undefined, - voxelSizeUnit: this.state.needsConversion ? formValues.voxelSizeUnit : undefined, - }; this.setState({ isFinishing: true, }); - finishDatasetUpload(datastoreUrl, uploadInfo).then( - async ({ newDatasetId }) => { + finishDatasetUpload(datastoreUrl, uploadId).then( + async ({ datasetId }) => { const { needsConversion } = this.state; this.setState({ isUploading: false, @@ -408,7 +405,7 @@ class DatasetUploadView extends React.Component { name: "", zipFile: [], }); - this.props.onUploaded(newDatasetId, newDatasetName, needsConversion); + this.props.onUploaded(datasetId, newDatasetName, needsConversion); }, (error) => { sendFailedRequestAnalyticsEvent("finish_dataset_upload", error, { @@ -475,9 +472,7 @@ class DatasetUploadView extends React.Component { resumableUpload.cancel(); if (uploadId) { - await cancelDatasetUpload(datastoreUrl, { - uploadId, - }); + await cancelDatasetUpload(datastoreUrl, uploadId); } this.setState({ isUploading: false, diff --git a/frontend/javascripts/admin/rest_api.ts b/frontend/javascripts/admin/rest_api.ts index bff3ddd7c69..d89beb06592 100644 --- a/frontend/javascripts/admin/rest_api.ts +++ b/frontend/javascripts/admin/rest_api.ts @@ -1205,7 +1205,7 @@ export function createResumableUpload( const resumable = new ResumableUpload({ testChunks: true, - target: `${datastoreUrl}/data/datasets`, + target: `${datastoreUrl}/data/datasets/upload/dataset`, query: function () { return { token: activeToken, @@ -1252,25 +1252,32 @@ export function createResumableUpload( return resumable; }); } -type ReserveUploadInformation = { + +type ResumableUploadInfo = { uploadId: string; - name: string; - directoryName: string; - newDatasetId: string; - organization: string; totalFileCount: number; filePaths: Array; - initialTeams: Array; + totalFileSizeInBytes: number; +}; +type DatasetUploadInfo = { + resumableUploadInfo: ResumableUploadInfo; + datasetName: string; + organizationId: string; + layersToLink: Array; // Always set as empty by frontend, only used by libs + initialTeamIds: Array; folderId: string | null; + needsConversion: boolean; + voxelSizeFactor: Vector3 | null | undefined; + voxelSizeUnit: string | null | undefined; }; export function reserveDatasetUpload( datastoreHost: string, - reserveUploadInformation: ReserveUploadInformation, + datasetUploadInfo: DatasetUploadInfo, ): Promise { return doWithToken((token) => - Request.sendJSONReceiveJSON(`/data/datasets/reserveUpload?token=${token}`, { - data: reserveUploadInformation, + Request.sendJSONReceiveJSON(`/data/datasets/upload/dataset/reserveUpload?token=${token}`, { + data: datasetUploadInfo, host: datastoreHost, }), ); @@ -1291,7 +1298,7 @@ export function getUnfinishedUploads( ): Promise { return doWithToken(async (token) => { const unfinishedUploads = (await Request.receiveJSON( - `/data/datasets/getUnfinishedUploads?token=${token}&organizationName=${organizationName}`, + `/data/datasets/upload/dataset/unfinishedUploads?token=${token}&organizationName=${organizationName}`, { host: datastoreHost, }, @@ -1304,29 +1311,34 @@ type NewDatasetReply = { newDatasetId: string; }; +type FinishUploadReply = { + datasetId: string; +}; + export function finishDatasetUpload( datastoreHost: string, - uploadInformation: ArbitraryObject, -): Promise { + uploadId: string, +): Promise { return doWithToken((token) => - Request.sendJSONReceiveJSON(`/data/datasets/finishUpload?token=${token}`, { - data: uploadInformation, - host: datastoreHost, - }), + Request.receiveJSON( + `/data/datasets/upload/dataset/finishUpload?uploadId=${uploadId}&token=${token}`, + { + host: datastoreHost, + method: "POST", + }, + ), ); } -export function cancelDatasetUpload( - datastoreHost: string, - cancelUploadInformation: { - uploadId: string; - }, -): Promise { +export function cancelDatasetUpload(datastoreHost: string, uploadId: string): Promise { return doWithToken((token) => - Request.sendJSONReceiveJSON(`/data/datasets/cancelUpload?token=${token}`, { - data: cancelUploadInformation, - host: datastoreHost, - }), + Request.receiveJSON( + `/data/datasets/upload/dataset/cancelUpload?uploadId=${uploadId}&token=${token}`, + { + host: datastoreHost, + method: "POST", + }, + ), ); } diff --git a/frontend/javascripts/test/backend_snapshot_tests/datasets.e2e.ts b/frontend/javascripts/test/backend_snapshot_tests/datasets.e2e.ts index 3df84a6fbab..6855523f080 100644 --- a/frontend/javascripts/test/backend_snapshot_tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend_snapshot_tests/datasets.e2e.ts @@ -188,20 +188,22 @@ describe("Dataset API (E2E)", () => { it("Dataset upload", async () => { const uploadId = "test-dataset-upload-" + Date.now(); - const reserveUpload = await fetch("/data/datasets/reserveUpload", { + const reserveUpload = await fetch("/data/datasets/upload/dataset/reserveUpload", { method: "POST", headers: new Headers({ "Content-Type": "application/json", }), body: JSON.stringify({ - filePaths: ["test-dataset-upload.zip"], + resumableUploadInfo: { + filePaths: ["test-dataset.zip"], + totalFileCount: 1, + uploadId: uploadId, + }, folderId: "570b9f4e4bb848d0885ea917", - initialTeams: [], + initialTeamIds: [], layersToLink: [], - name: "test-dataset-upload", - organization: "Organization_X", - totalFileCount: 1, - uploadId: uploadId, + datasetName: "test-dataset-upload", + organizationId: "Organization_X", }), }); @@ -248,7 +250,7 @@ describe("Dataset API (E2E)", () => { let content_type = `multipart/form-data; boundary=${boundary}`; - const uploadResult = await fetch("/data/datasets", { + const uploadResult = await fetch("/data/datasets/upload/dataset", { method: "POST", headers: new Headers({ "Content-Type": content_type, @@ -260,23 +262,22 @@ describe("Dataset API (E2E)", () => { expect.fail("Dataset upload failed"); } - const finishResult = await fetch("/data/datasets/finishUpload", { - method: "POST", - headers: new Headers({ - "Content-Type": "application/json", - }), - body: JSON.stringify({ - uploadId: uploadId, - needsConversion: false, - }), - }); + const finishResult = await fetch( + `/data/datasets/upload/dataset/finishUpload?uploadId=${uploadId}`, + { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + }, + ); if (finishResult.status !== 200) { expect.fail("Dataset upload failed at finish"); } - const { newDatasetId } = await finishResult.json(); - const result = await fetch(`/api/datasets/${newDatasetId}/health`, { + const { datasetId } = await finishResult.json(); + const result = await fetch(`/api/datasets/${datasetId}/health`, { headers: new Headers(), }); diff --git a/schema/evolutions/161-upload-mags-attachments.sql b/schema/evolutions/161-upload-mags-attachments.sql new file mode 100644 index 00000000000..85a091a84e2 --- /dev/null +++ b/schema/evolutions/161-upload-mags-attachments.sql @@ -0,0 +1,10 @@ +START TRANSACTION; + +do $$ begin if (select schemaVersion from webknossos.releaseInformation) <> 160 then raise exception 'Previous schema version mismatch'; end if; end; $$ language plpgsql; + +ALTER TABLE webknossos.dataset_layer_attachments ADD COLUMN uploadIsPending BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE webknossos.dataset_mags ADD COLUMN uploadIsPending BOOLEAN NOT NULL DEFAULT FALSE; + +UPDATE webknossos.releaseInformation SET schemaVersion = 161; + +COMMIT TRANSACTION; diff --git a/schema/evolutions/reversions/161-upload-mags-attachments.sql b/schema/evolutions/reversions/161-upload-mags-attachments.sql new file mode 100644 index 00000000000..8d26649fb3d --- /dev/null +++ b/schema/evolutions/reversions/161-upload-mags-attachments.sql @@ -0,0 +1,10 @@ +START TRANSACTION; + +do $$ begin if (select schemaVersion from webknossos.releaseInformation) <> 161 then raise exception 'Previous schema version mismatch'; end if; end; $$ language plpgsql; + +ALTER TABLE webknossos.dataset_layer_attachments DROP COLUMN uploadIsPending; +ALTER TABLE webknossos.dataset_mags DROP COLUMN uploadIsPending; + +UPDATE webknossos.releaseInformation SET schemaVersion = 160; + +COMMIT TRANSACTION; diff --git a/schema/schema.sql b/schema/schema.sql index eb5527821f0..8d3566055fe 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -21,7 +21,7 @@ CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(160); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(161); COMMIT TRANSACTION; @@ -177,6 +177,7 @@ CREATE TABLE webknossos.dataset_layer_attachments( type webknossos.LAYER_ATTACHMENT_TYPE NOT NULL, dataFormat webknossos.LAYER_ATTACHMENT_DATAFORMAT NOT NULL, uploadToPathIsPending BOOLEAN NOT NULL DEFAULT FALSE, + uploadIsPending BOOLEAN NOT NULL DEFAULT FALSE, PRIMARY KEY(_dataset, layerName, name, type) ); @@ -197,6 +198,7 @@ CREATE TABLE webknossos.dataset_mags( channelIndex INT, credentialId TEXT, uploadToPathIsPending BOOLEAN NOT NULL DEFAULT FALSE, + uploadIsPending BOOLEAN NOT NULL DEFAULT FALSE, PRIMARY KEY (_dataset, dataLayerName, mag) ); diff --git a/test/db/dataset_mags.csv b/test/db/dataset_mags.csv index c1d580b1e6e..4ac65a6d11b 100644 --- a/test/db/dataset_mags.csv +++ b/test/db/dataset_mags.csv @@ -1,26 +1,26 @@ -_dataset,dataLayerName,mag,path,realPath,hasLocalData,channelIndex,credentialId,uploadToPathIsPending -'59e9cfbdba632ac2ab8b23b3','color_1','(1,1,1)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_1','(2,2,2)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_1','(4,4,4)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_1','(8,8,8)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_1','(16,16,16)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_2','(1,1,1)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_2','(2,2,2)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_2','(4,4,4)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_2','(8,8,8)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_2','(16,16,16)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_3','(1,1,1)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_3','(2,2,2)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_3','(4,4,4)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_3','(8,8,8)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b3','color_3','(16,16,16)','',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','color','(1,1,1)','./color/1',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','color','(2,2,1)','./color/2-2-1',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','color','(4,4,1)','./color/4-4-1',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','color','(8,8,2)','./color/8-8-2',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','color','(16,16,4)','./color/16-16-4',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','segmentation','(1,1,1)','./segmentation/1',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','segmentation','(2,2,1)','./segmentation/2-2-1',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','segmentation','(4,4,1)','./segmentation/4-4-1',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','segmentation','(8,8,2)','./segmentation/8-8-2',,false,,,,false -'59e9cfbdba632ac2ab8b23b5','segmentation','(16,16,4)','./segmentation/16-16-4',,false,,,,false +_dataset,dataLayerName,mag,path,realPath,hasLocalData,channelIndex,credentialId,uploadToPathIsPending,uploadIsPending +'59e9cfbdba632ac2ab8b23b3','color_1','(1,1,1)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_1','(2,2,2)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_1','(4,4,4)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_1','(8,8,8)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_1','(16,16,16)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_2','(1,1,1)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_2','(2,2,2)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_2','(4,4,4)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_2','(8,8,8)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_2','(16,16,16)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_3','(1,1,1)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_3','(2,2,2)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_3','(4,4,4)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_3','(8,8,8)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b3','color_3','(16,16,16)','',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','color','(1,1,1)','./color/1',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','color','(2,2,1)','./color/2-2-1',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','color','(4,4,1)','./color/4-4-1',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','color','(8,8,2)','./color/8-8-2',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','color','(16,16,4)','./color/16-16-4',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','segmentation','(1,1,1)','./segmentation/1',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','segmentation','(2,2,1)','./segmentation/2-2-1',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','segmentation','(4,4,1)','./segmentation/4-4-1',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','segmentation','(8,8,2)','./segmentation/8-8-2',,false,,,,false,false +'59e9cfbdba632ac2ab8b23b5','segmentation','(16,16,4)','./segmentation/16-16-4',,false,,,,false,false diff --git a/unreleased_changes/9402.md b/unreleased_changes/9402.md new file mode 100644 index 00000000000..253e5347def --- /dev/null +++ b/unreleased_changes/9402.md @@ -0,0 +1,5 @@ +### Added +- Added routes for uploading attachments and mags to existing datasets via the python libs client. + +### Postgres Evolutions +- [161-upload-mags-attachments.sql](schema/evolutions/161-upload-mags-attachments.sql) diff --git a/util/src/main/scala/com/scalableminds/util/mvc/ApiVersioning.scala b/util/src/main/scala/com/scalableminds/util/mvc/ApiVersioning.scala index 82d821beaf7..114a240de12 100644 --- a/util/src/main/scala/com/scalableminds/util/mvc/ApiVersioning.scala +++ b/util/src/main/scala/com/scalableminds/util/mvc/ApiVersioning.scala @@ -5,7 +5,7 @@ import play.api.mvc.RequestHeader trait ApiVersioning { - protected val CURRENT_API_VERSION: Int = 13 + protected val CURRENT_API_VERSION: Int = 14 protected val OLDEST_SUPPORTED_API_VERSION: Int = 5 protected lazy val apiVersioningInfo: JsObject = diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSLegacyApiController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSLegacyApiController.scala index d4bf8cfb4fd..836d6d6c16a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSLegacyApiController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSLegacyApiController.scala @@ -12,7 +12,12 @@ import com.scalableminds.webknossos.datastore.models.{ } import com.scalableminds.webknossos.datastore.models.datasource.{UnusableDataSource, UsableDataSource} import com.scalableminds.webknossos.datastore.services.mesh.FullMeshRequest -import com.scalableminds.webknossos.datastore.services.uploading.{LinkedLayerIdentifier, ReserveUploadInformation} +import com.scalableminds.webknossos.datastore.services.uploading.{ + DatasetUploadInfo, + LinkedLayerIdentifier, + ResumableUploadInfo, + UploadDomain +} import com.scalableminds.webknossos.datastore.services.{ DSRemoteWebknossosClient, DataSourceService, @@ -20,15 +25,16 @@ import com.scalableminds.webknossos.datastore.services.{ DatasetCache, UserAccessRequest } -import play.api.libs.json.{Json, OFormat} -import play.api.mvc.{Action, AnyContent, PlayBodyParsers, RawBuffer, Result} +import play.api.libs.Files +import play.api.libs.json.{JsObject, Json, OFormat} +import play.api.mvc.{Action, AnyContent, MultipartFormData, PlayBodyParsers, RawBuffer, Result} import scala.concurrent.{ExecutionContext, Future} case class LegacyReserveManualUploadInformation( datasetName: String, organization: String, - initialTeamIds: List[ObjectId], + initialTeamIds: Seq[ObjectId], folderId: Option[ObjectId], requireUniqueName: Boolean = false, ) @@ -37,7 +43,7 @@ object LegacyReserveManualUploadInformation { Json.format[LegacyReserveManualUploadInformation] } -case class LegacyReserveUploadInformation( +case class LegacyReserveUploadInformationV11( uploadId: String, // upload id that was also used in chunk upload (this time without file paths) name: String, // dataset name organization: String, @@ -47,10 +53,12 @@ case class LegacyReserveUploadInformation( layersToLink: Option[List[LegacyLinkedLayerIdentifier]], initialTeams: List[ObjectId], // team ids folderId: Option[ObjectId], - requireUniqueName: Option[Boolean] + requireUniqueName: Option[Boolean], + isVirtual: Option[Boolean], // Only set (to false) for legacy manual uploads + needsConversion: Option[Boolean] // None means false ) -object LegacyReserveUploadInformation { - implicit val jsonFormat: OFormat[LegacyReserveUploadInformation] = Json.format[LegacyReserveUploadInformation] +object LegacyReserveUploadInformationV11 { + implicit val jsonFormat: OFormat[LegacyReserveUploadInformationV11] = Json.format[LegacyReserveUploadInformationV11] } case class LegacyLinkedLayerIdentifier(organizationId: Option[String], @@ -73,6 +81,30 @@ object LegacyLinkedLayerIdentifier { implicit val jsonFormat: OFormat[LegacyLinkedLayerIdentifier] = Json.format[LegacyLinkedLayerIdentifier] } +case class LegacyUploadInformation(uploadId: String, needsConversion: Option[Boolean]) + +object LegacyUploadInformation { + implicit val jsonFormat: OFormat[LegacyUploadInformation] = Json.format[LegacyUploadInformation] +} + +case class ReserveUploadInformationV13( + uploadId: String, // upload id that was also used in chunk upload (this time without file paths) + name: String, // dataset name + organization: String, + totalFileCount: Long, + filePaths: Option[List[String]], + totalFileSizeInBytes: Option[Long], + layersToLink: Option[List[LinkedLayerIdentifier]], + initialTeams: List[ObjectId], // team ids + folderId: Option[ObjectId], + requireUniqueName: Option[Boolean], + isVirtual: Option[Boolean], // Only set (to false) for legacy manual uploads + needsConversion: Option[Boolean] // None means false +) +object ReserveUploadInformationV13 { + implicit val jsonFormat: OFormat[ReserveUploadInformationV13] = Json.format[ReserveUploadInformationV13] +} + class DSLegacyApiController @Inject()( accessTokenService: DataStoreAccessTokenService, remoteWebknossosClient: DSRemoteWebknossosClient, @@ -81,7 +113,8 @@ class DSLegacyApiController @Inject()( meshController: DSMeshController, dataSourceController: DataSourceController, dataSourceService: DataSourceService, - datasetCache: DatasetCache + datasetCache: DatasetCache, + uploadController: UploadController )(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with Zarr3OutputHelper @@ -89,28 +122,84 @@ class DSLegacyApiController @Inject()( override def allowRemoteOrigin: Boolean = true - def reserveUploadV11(): Action[LegacyReserveUploadInformation] = - Action.async(validateJson[LegacyReserveUploadInformation]) { implicit request => + def testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String): Action[AnyContent] = + uploadController.testChunk(resumableChunkNumber, resumableIdentifier, UploadDomain.dataset.toString) + + def finishUploadV13(): Action[LegacyUploadInformation] = Action.async(validateJson[LegacyUploadInformation]) { + implicit request => + for { + result <- uploadController.finishUpload(UploadDomain.dataset.toString, request.body.uploadId)( + request.withBody(play.api.mvc.AnyContentAsEmpty)) + } yield + if (result.header.status == OK) { + result.body match { + case play.api.http.HttpEntity.Strict(data, _) => + val json = Json.parse(data.toArray).as[JsObject] + Ok((json - "datasetId") ++ Json.obj("newDatasetId" -> (json \ "datasetId").get)) + case _ => result + } + } else result + } + + def reserveDatasetUploadV13(): Action[ReserveUploadInformationV13] = + Action.async(validateJson[ReserveUploadInformationV13]) { implicit request => + uploadController.reserveDatasetUpload()( + request.withBody(DatasetUploadInfo( + resumableUploadInfo = ResumableUploadInfo( + uploadId = request.body.uploadId, + totalFileCount = request.body.totalFileCount, + filePaths = request.body.filePaths, + totalFileSizeInBytes = request.body.totalFileSizeInBytes + ), + datasetName = request.body.name, + organizationId = request.body.organization, + layersToLink = request.body.layersToLink, + initialTeamIds = request.body.initialTeams, + folderId = request.body.folderId, + requireUniqueName = request.body.requireUniqueName, + isVirtual = request.body.isVirtual, + needsConversion = None, + voxelSizeFactor = None, + voxelSizeUnit = None + ))) + } + + def uploadChunkV13(): Action[MultipartFormData[Files.TemporaryFile]] = + Action.async(parse.multipartFormData) { implicit request => + uploadController.uploadChunk(UploadDomain.dataset.toString)(request) + } + + def getUnfinishedUploadsV13(organizationName: String): Action[AnyContent] = + Action.async { implicit request => + uploadController.getUnfinishedUploads(organizationName, UploadDomain.dataset.toString)(request) + } + + def reserveUploadV11(): Action[LegacyReserveUploadInformationV11] = + Action.async(validateJson[LegacyReserveUploadInformationV11]) { implicit request => accessTokenService.validateAccessFromTokenContext( UserAccessRequest.administrateDatasets(request.body.organization)) { for { adaptedLayersToLink <- Fox.serialCombined(request.body.layersToLink.getOrElse(List.empty))(adaptLayerToLink) - adaptedRequestBody = ReserveUploadInformation( - uploadId = request.body.uploadId, - name = request.body.name, - organization = request.body.organization, - totalFileCount = request.body.totalFileCount, - filePaths = request.body.filePaths, - totalFileSizeInBytes = request.body.totalFileSizeInBytes, + adaptedRequestBody = DatasetUploadInfo( + resumableUploadInfo = ResumableUploadInfo( + uploadId = request.body.uploadId, + totalFileCount = request.body.totalFileCount, + filePaths = request.body.filePaths, + totalFileSizeInBytes = request.body.totalFileSizeInBytes, + ), + datasetName = request.body.name, + organizationId = request.body.organization, layersToLink = Some(adaptedLayersToLink), - initialTeams = request.body.initialTeams, + initialTeamIds = request.body.initialTeams, folderId = request.body.folderId, requireUniqueName = request.body.requireUniqueName, - isVirtual = None, - needsConversion = None + isVirtual = request.body.isVirtual, + needsConversion = None, + voxelSizeFactor = None, + voxelSizeUnit = None ) - result <- Fox.fromFuture(dataSourceController.reserveUpload()(request.withBody(adaptedRequestBody))) + result <- Fox.fromFuture(uploadController.reserveDatasetUpload()(request.withBody(adaptedRequestBody))) } yield result } } @@ -136,20 +225,24 @@ class DSLegacyApiController @Inject()( accessTokenService.validateAccessFromTokenContext( UserAccessRequest.administrateDatasets(request.body.organization)) { for { - reservedDatasetInfo <- remoteWebknossosClient.reserveDataSourceUpload( - ReserveUploadInformation( - "aManualUpload", - request.body.datasetName, - request.body.organization, - 0, - Some(List.empty), - None, - None, - request.body.initialTeamIds, - request.body.folderId, - Some(request.body.requireUniqueName), - Some(false), - needsConversion = None + reservedDatasetInfo <- remoteWebknossosClient.reserveDatasetUpload( + DatasetUploadInfo( + resumableUploadInfo = ResumableUploadInfo( + uploadId = "aManualUpload", + totalFileCount = 0, + filePaths = Some(List.empty), + totalFileSizeInBytes = None + ), + datasetName = request.body.datasetName, + organizationId = request.body.organization, + layersToLink = None, + initialTeamIds = request.body.initialTeamIds, + folderId = request.body.folderId, + requireUniqueName = Some(request.body.requireUniqueName), + isVirtual = Some(false), + needsConversion = None, + voxelSizeFactor = None, + voxelSizeUnit = None ) ) ?~> "dataset.upload.validation.failed" } yield diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index 279ff1bc027..77595979737 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -31,7 +31,6 @@ import com.scalableminds.webknossos.datastore.services.mesh.{ MeshMappingHelper } import com.scalableminds.webknossos.datastore.services.segmentindex.SegmentIndexFileService -import com.scalableminds.webknossos.datastore.services.uploading._ import com.scalableminds.webknossos.datastore.services.connectome.{ ByAgglomerateIdsRequest, BySynapseIdsRequest, @@ -39,13 +38,9 @@ import com.scalableminds.webknossos.datastore.services.connectome.{ } import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService import com.scalableminds.webknossos.datastore.storage.DataVaultService -import com.scalableminds.webknossos.datastore.slacknotification.DSSlackNotificationService -import play.api.data.Form -import play.api.data.Forms.{longNumber, nonEmptyText, number, tuple} -import play.api.i18n.Messages -import play.api.libs.Files + import play.api.libs.json.{Json, OFormat} -import play.api.mvc.{Action, AnyContent, MultipartFormData, PlayBodyParsers} +import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import java.io.File import java.net.URI @@ -71,11 +66,9 @@ class DataSourceController @Inject()( segmentIndexFileService: SegmentIndexFileService, agglomerateService: AgglomerateService, storageUsageService: DSUsedStorageService, - slackNotificationService: DSSlackNotificationService, datasetErrorLoggingService: DSDatasetErrorLoggingService, exploreRemoteLayerService: ExploreRemoteLayerService, fullMeshService: DSFullMeshService, - uploadService: UploadService, managedS3Service: ManagedS3Service, meshFileService: MeshFileService, dataVaultService: DataVaultService, @@ -114,136 +107,6 @@ class DataSourceController @Inject()( } } - def reserveUpload(): Action[ReserveUploadInformation] = - Action.async(validateJson[ReserveUploadInformation]) { implicit request => - accessTokenService.validateAccessFromTokenContext( - UserAccessRequest.administrateDatasets(request.body.organization)) { - for { - isKnownUpload <- uploadService.isKnownUpload(request.body.uploadId) - _ <- if (!isKnownUpload) { - for { - reserveUploadAdditionalInfo <- dsRemoteWebknossosClient.reserveDataSourceUpload(request.body) ?~> "dataset.upload.validation.failed" - _ <- uploadService.reserveUpload(request.body, reserveUploadAdditionalInfo) - } yield () - } else Fox.successful(()) - } yield Ok - } - } - - def getUnfinishedUploads(organizationName: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDatasets(organizationName)) { - for { - unfinishedUploads <- dsRemoteWebknossosClient.getUnfinishedUploadsForUser(organizationName) - unfinishedUploadsWithUploadIds <- Fox.fromFuture( - uploadService.addUploadIdsToUnfinishedUploads(unfinishedUploads)) - unfinishedUploadsWithUploadIdsWithoutDataSourceId = unfinishedUploadsWithUploadIds.map(_.withoutDataSourceId) - } yield Ok(Json.toJson(unfinishedUploadsWithUploadIdsWithoutDataSourceId)) - } - } - - /* Upload a byte chunk for a new dataset - Expects: - - As file attachment: A raw byte chunk of the dataset - - As form parameter: - - name (string): dataset name - - owningOrganization (string): owning organization name - - resumableChunkNumber (int): chunk index - - resumableChunkSize (int): chunk size in bytes - - resumableTotalChunks (string): total chunk count of the upload - - totalFileCount (string): total file count of the upload - - resumableIdentifier (string): identifier of the resumable upload and file ("{uploadId}/{filepath}") - - As GET parameter: - - token (string): datastore token identifying the uploading user - */ - def uploadChunk(): Action[MultipartFormData[Files.TemporaryFile]] = - Action.async(parse.multipartFormData) { implicit request => - log(Some(slackNotificationService.noticeFailedUploadRequest)) { - val uploadForm = Form( - tuple( - "resumableChunkNumber" -> number, - "resumableChunkSize" -> number, - "resumableCurrentChunkSize" -> number, - "resumableTotalChunks" -> longNumber, - "resumableIdentifier" -> nonEmptyText - )).fill((-1, -1, -1, -1, "")) - - uploadForm - .bindFromRequest(request.body.dataParts) - .fold( - hasErrors = formWithErrors => Fox.successful(JsonBadRequest(formWithErrors.errors.head.message)), - success = { - case (chunkNumber, chunkSize, currentChunkSize, totalChunkCount, uploadFileId) => - for { - datasetId <- uploadService - .getDatasetIdByUploadId(uploadService.extractDatasetUploadId(uploadFileId)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService - .validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) { - for { - isKnownUpload <- uploadService.isKnownUploadByFileId(uploadFileId) - _ <- Fox.fromBool(isKnownUpload) ?~> "dataset.upload.validation.failed" - chunkFile <- request.body.file("file").toFox ?~> "zip.file.notFound" - _ <- uploadService.handleUploadChunk(uploadFileId, - chunkSize, - currentChunkSize, - totalChunkCount, - chunkNumber, - new File(chunkFile.ref.path.toString)) - } yield Ok - } - } yield result - } - ) - } - } - - def testChunk(resumableChunkNumber: Int, resumableIdentifier: String): Action[AnyContent] = - Action.async { implicit request => - for { - datasetId <- uploadService.getDatasetIdByUploadId(uploadService.extractDatasetUploadId(resumableIdentifier)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) { - for { - isKnownUpload <- uploadService.isKnownUploadByFileId(resumableIdentifier) - _ <- Fox.fromBool(isKnownUpload) ?~> "dataset.upload.validation.failed" - isPresent <- uploadService.isChunkPresent(resumableIdentifier, resumableChunkNumber) - } yield if (isPresent) Ok else NoContent - } - } yield result - } - - def finishUpload(): Action[UploadInformation] = Action.async(validateJson[UploadInformation]) { implicit request => - log(Some(slackNotificationService.noticeFailedUploadRequest)) { - logTime(slackNotificationService.noticeSlowRequest) { - for { - datasetId <- uploadService - .getDatasetIdByUploadId(request.body.uploadId) ?~> s"Cannot find running upload with upload id ${request.body.uploadId}" - response <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) { - for { - _ <- uploadService.finishUpload(request.body, datasetId) ?~> Messages("dataset.upload.finishFailed", - datasetId) - } yield Ok(Json.obj("newDatasetId" -> datasetId)) - } - } yield response - } - } - } - - def cancelUpload(): Action[CancelUploadInformation] = - Action.async(validateJson[CancelUploadInformation]) { implicit request => - val datasetIdFox = uploadService.isKnownUpload(request.body.uploadId).flatMap { - case false => Fox.failure("dataset.upload.validation.failed") - case true => uploadService.getDatasetIdByUploadId(request.body.uploadId) - } - datasetIdFox.flatMap { datasetId => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.deleteDataset(datasetId)) { - for { - _ <- dsRemoteWebknossosClient.deleteDataset(datasetId) ?~> "dataset.delete.webknossos.failed" - _ <- uploadService.cancelUpload(request.body) ?~> "Could not cancel the upload." - } yield Ok - } - } - } - def listMappings( datasetId: ObjectId, dataLayerName: String diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/UploadController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/UploadController.scala new file mode 100644 index 00000000000..add3636fc49 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/UploadController.scala @@ -0,0 +1,214 @@ +package com.scalableminds.webknossos.datastore.controllers + +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.services.{ + DSRemoteWebknossosClient, + DataStoreAccessTokenService, + UserAccessRequest +} +import com.scalableminds.webknossos.datastore.services.uploading.{ + AttachmentUploadInfo, + DatasetUploadInfo, + MagUploadInfo, + UploadDomain, + UploadService +} +import com.scalableminds.webknossos.datastore.slacknotification.DSSlackNotificationService +import play.api.data.Form +import play.api.data.Forms.tuple +import play.api.i18n.Messages +import play.api.libs.Files +import play.api.libs.json.Json +import play.api.mvc.{Action, AnyContent, MultipartFormData, PlayBodyParsers} +import play.api.data.Forms.{longNumber, nonEmptyText, number} + +import java.io.File +import javax.inject.Inject +import scala.concurrent.ExecutionContext + +class UploadController @Inject()( + accessTokenService: DataStoreAccessTokenService, + uploadService: UploadService, + dsRemoteWebknossosClient: DSRemoteWebknossosClient, + slackNotificationService: DSSlackNotificationService)(implicit bodyParsers: PlayBodyParsers, ec: ExecutionContext) + extends Controller { + + def reserveDatasetUpload(): Action[DatasetUploadInfo] = + Action.async(validateJson[DatasetUploadInfo]) { implicit request => + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.administrateDatasets(request.body.organizationId)) { + for { + isKnownUpload <- uploadService.isKnownUpload(request.body.resumableUploadInfo.uploadId, UploadDomain.dataset) + _ <- Fox.runIf(!isKnownUpload) { + for { + reserveUploadAdditionalInfo <- dsRemoteWebknossosClient.reserveDatasetUpload(request.body) ?~> "dataset.upload.validation.failed" + _ <- uploadService.reserveDatasetUpload(request.body, + reserveUploadAdditionalInfo.newDatasetId, + reserveUploadAdditionalInfo.directoryName) + } yield () + } + } yield Ok + } + } + + def reserveMagUpload(): Action[MagUploadInfo] = + Action.async(validateJson[MagUploadInfo]) { implicit request => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(request.body.datasetId)) { + for { + isKnownUpload <- uploadService.isKnownUpload(request.body.resumableUploadInfo.uploadId, UploadDomain.mag) + _ <- Fox.runIf(!isKnownUpload) { + for { + reserveUploadAdditionalInfo <- dsRemoteWebknossosClient.reserveMagUpload(request.body) ?~> "dataset.upload.validation.failed" + _ <- uploadService.reserveMagUpload(request.body, reserveUploadAdditionalInfo.dataSourceId) + } yield () + } + } yield Ok + } + } + + def reserveAttachmentUpload(): Action[AttachmentUploadInfo] = + Action.async(validateJson[AttachmentUploadInfo]) { implicit request => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(request.body.datasetId)) { + for { + isKnownUpload <- uploadService.isKnownUpload(request.body.resumableUploadInfo.uploadId, + UploadDomain.attachment) + _ <- Fox.runIf(!isKnownUpload) { + for { + reserveUploadAdditionalInfo <- dsRemoteWebknossosClient.reserveAttachmentUpload(request.body) ?~> "dataset.upload.validation.failed" + _ <- uploadService.reserveAttachmentUpload(request.body, reserveUploadAdditionalInfo.dataSourceId) + } yield () + } + } yield Ok + } + } + + def getUnfinishedUploads(organizationName: String, uploadDomain: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDatasets(organizationName)) { + for { + uploadDomainValidated <- UploadDomain.fromString(uploadDomain).toFox + _ <- Fox.fromBool(uploadDomainValidated == UploadDomain.dataset) ?~> "Listing unfinished downloads is only supported for datasets." + unfinishedUploads <- dsRemoteWebknossosClient.getUnfinishedUploadsForUser(organizationName) + unfinishedUploadsWithUploadIds <- Fox.fromFuture( + uploadService.enrichUnfinishedUploadInfoWithUploadIds(unfinishedUploads)) + unfinishedUploadsWithUploadIdsWithoutDataSourceId = unfinishedUploadsWithUploadIds.map(_.withoutDataSourceId) + } yield Ok(Json.toJson(unfinishedUploadsWithUploadIdsWithoutDataSourceId)) + } + } + + /* Upload a byte chunk for a new dataset + Expects: + - As file attachment: A raw byte chunk of the dataset + - As form parameter: + - name (string): dataset name + - owningOrganization (string): owning organization name + - resumableChunkNumber (int): chunk index + - resumableChunkSize (int): chunk size in bytes + - resumableTotalChunks (string): total chunk count of the upload + - totalFileCount (string): total file count of the upload + - resumableIdentifier (string): identifier of the resumable upload and file ("{uploadId}/{filepath}") + - As GET parameter: + - token (string): datastore token identifying the uploading user + */ + def uploadChunk(uploadDomain: String): Action[MultipartFormData[Files.TemporaryFile]] = + Action.async(parse.multipartFormData) { implicit request => + log(Some(slackNotificationService.noticeFailedUploadRequest)) { + val uploadForm = Form( + tuple( + "resumableChunkNumber" -> number, + "resumableChunkSize" -> number, + "resumableCurrentChunkSize" -> number, + "resumableTotalChunks" -> longNumber, + "resumableIdentifier" -> nonEmptyText + )).fill((-1, -1, -1, -1, "")) + + uploadForm + .bindFromRequest(request.body.dataParts) + .fold( + hasErrors = formWithErrors => Fox.successful(JsonBadRequest(formWithErrors.errors.head.message)), + success = { + case (chunkNumber, chunkSize, currentChunkSize, totalChunkCount, uploadFileId) => + for { + uploadDomainValidated <- UploadDomain.fromString(uploadDomain).toFox + datasetId <- uploadService.getDatasetIdByUploadId( + uploadService.extractDatasetUploadId(uploadFileId), + uploadDomainValidated) ?~> "dataset.upload.validation.failed" + result <- accessTokenService + .validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) { + for { + isKnownUpload <- uploadService.isKnownUploadByFileId(uploadFileId, uploadDomainValidated) + _ <- Fox.fromBool(isKnownUpload) ?~> "dataset.upload.validation.failed" + chunkFile <- request.body.file("file").toFox ?~> "zip.file.notFound" + _ <- uploadService.handleUploadChunk(uploadFileId, + chunkSize, + currentChunkSize, + totalChunkCount, + chunkNumber, + new File(chunkFile.ref.path.toString), + uploadDomainValidated) + } yield Ok + } + } yield result + } + ) + } + } + + def testChunk(resumableChunkNumber: Int, resumableIdentifier: String, uploadDomain: String): Action[AnyContent] = + Action.async { implicit request => + for { + uploadDomainValidated <- UploadDomain.fromString(uploadDomain).toFox + datasetId <- uploadService.getDatasetIdByUploadId(uploadService.extractDatasetUploadId(resumableIdentifier), + uploadDomainValidated) ?~> "dataset.upload.validation.failed" + result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) { + for { + isKnownUpload <- uploadService.isKnownUploadByFileId(resumableIdentifier, uploadDomainValidated) + _ <- Fox.fromBool(isKnownUpload) ?~> "dataset.upload.validation.failed" + isPresent <- uploadService.isChunkPresent(resumableIdentifier, resumableChunkNumber, uploadDomainValidated) + } yield if (isPresent) Ok else NoContent + } + } yield result + } + + def finishUpload(uploadDomain: String, uploadId: String): Action[AnyContent] = Action.async { implicit request => + log(Some(slackNotificationService.noticeFailedUploadRequest)) { + logTime(slackNotificationService.noticeSlowRequest) { + for { + uploadDomainValidated <- UploadDomain.fromString(uploadDomain).toFox + datasetId <- uploadService + .getDatasetIdByUploadId(uploadId, uploadDomainValidated) ?~> s"Cannot find running upload with upload id $uploadId" + response <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) { + for { + _ <- (uploadDomainValidated match { + case UploadDomain.dataset => uploadService.finishDatasetUpload(uploadId, datasetId) + case UploadDomain.mag => uploadService.finishMagUpload(uploadId, datasetId) + case UploadDomain.attachment => uploadService.finishAttachmentUpload(uploadId, datasetId) + }) ?~> Messages("dataset.upload.finishFailed", datasetId) + } yield Ok(Json.obj("datasetId" -> datasetId)) + } + } yield response + } + } + } + + def cancelUpload(uploadDomain: String, uploadId: String): Action[AnyContent] = Action.async { implicit request => + for { + uploadDomainValidated <- UploadDomain.fromString(uploadDomain).toFox + _ <- Fox + .fromBool(uploadDomainValidated == UploadDomain.dataset) ?~> "Cancel upload is only supported for datasets." + datasetIdFox = uploadService.isKnownUpload(uploadId, uploadDomainValidated).flatMap { + case false => Fox.failure("dataset.upload.validation.failed") + case true => uploadService.getDatasetIdByUploadId(uploadId, uploadDomainValidated) + } + result <- datasetIdFox.flatMap { datasetId => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.deleteDataset(datasetId)) { + for { + _ <- dsRemoteWebknossosClient.deleteDataset(datasetId) ?~> "dataset.delete.webknossos.failed" + _ <- uploadService.cancelUpload(uploadDomainValidated, uploadId) ?~> "Could not cancel the upload." + } yield Ok + } + } + } yield result + } + +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala index 0db10442e2b..d12eba3c025 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala @@ -12,7 +12,10 @@ case class MagLocator(mag: Vec3Int, credentials: Option[LegacyDataVaultCredential] = None, axisOrder: Option[AxisOrder] = None, channelIndex: Option[Int] = None, - credentialId: Option[String] = None) + credentialId: Option[String] = None) { + + def withoutCredentials: MagLocator = this.copy(credentials = None, credentialId = None) +} object MagLocator extends MagFormatHelper { implicit val jsonFormat: OFormat[MagLocator] = Json.format[MagLocator] diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/DatasetDeleter.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/DatasetDeleter.scala index 118c178adb4..0c7d8098ee8 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/DatasetDeleter.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/DatasetDeleter.scala @@ -14,12 +14,10 @@ trait DatasetDeleter extends LazyLogging with DirectoryConstants with FoxImplici def deleteOnDisk(datasetId: ObjectId, organizationId: String, datasetName: String, - isInConversion: Boolean = false, + path: Option[Path] = None, reason: Option[String] = None)(implicit ec: ExecutionContext): Fox[Unit] = { - val dataSourcePath = - if (isInConversion) dataBaseDir.resolve(organizationId).resolve(forConversionDir).resolve(datasetName) - else dataBaseDir.resolve(organizationId).resolve(datasetName) + val dataSourcePath = path.getOrElse(dataBaseDir.resolve(organizationId).resolve(datasetName)) if (Files.exists(dataSourcePath)) { val trashPath: Path = dataBaseDir.resolve(organizationId).resolve(trashDir) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnfinishedUpload.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnfinishedUpload.scala index 15ddcbda3f9..dae7af30425 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnfinishedUpload.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnfinishedUpload.scala @@ -4,13 +4,13 @@ import com.scalableminds.util.time.Instant import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import play.api.libs.json.{Format, Json} -case class UnfinishedUpload(uploadId: String, +case class UnfinishedUpload(uploadId: String, // Dummy value on wk-side, then filled in by datastore via redis dataSourceId: DataSourceId, datasetName: String, folderId: String, created: Instant, - filePaths: Option[List[String]], - allowedTeams: List[String]) { + filePaths: Option[Seq[String]], + allowedTeams: Seq[String]) { def withoutDataSourceId: UnfinishedUploadWithoutDataSourceId = UnfinishedUploadWithoutDataSourceId(uploadId, datasetName, folderId, created, filePaths, allowedTeams) } @@ -23,8 +23,8 @@ case class UnfinishedUploadWithoutDataSourceId(uploadId: String, datasetName: String, folderId: String, created: Instant, - filePaths: Option[List[String]], - allowedTeams: List[String]) + filePaths: Option[Seq[String]], + allowedTeams: Seq[String]) object UnfinishedUploadWithoutDataSourceId { implicit val dataSourceIdFormat: Format[UnfinishedUploadWithoutDataSourceId] = diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayerAttachments.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayerAttachments.scala index 8f44b048fb5..5354a56a7d1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayerAttachments.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayerAttachments.scala @@ -61,6 +61,16 @@ case class DataLayerAttachments( case LayerAttachmentType.cumsum => cumsum.find(_.name == name) } + def getByTypeAndNameAlwaysReturnSingletons(attachmentType: LayerAttachmentType, + name: String): Option[LayerAttachment] = + attachmentType match { + case LayerAttachmentType.mesh => meshes.find(_.name == name) + case LayerAttachmentType.agglomerate => agglomerates.find(_.name == name) + case LayerAttachmentType.segmentIndex => segmentIndex + case LayerAttachmentType.connectome => connectomes.find(_.name == name) + case LayerAttachmentType.cumsum => cumsum + } + def mapped(attachmentMapping: LayerAttachment => LayerAttachment): DataLayerAttachments = DataLayerAttachments( meshes = meshes.map(attachmentMapping(_)), @@ -158,6 +168,8 @@ case class LayerAttachment(name: String, def relativizedIn(dataSourcePath: UPath): LayerAttachment = this.copy(path = this.path.relativizedIn(dataSourcePath)) + + def withoutCredential: LayerAttachment = this.copy(credentialId = None) } object LayerAttachment { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala index 7c9301774dd..66037dd7395 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala @@ -15,9 +15,15 @@ import com.scalableminds.webknossos.datastore.models.annotation.AnnotationSource import com.scalableminds.webknossos.datastore.models.datasource.{DataSource, DataSourceId} import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.uploading.{ + AttachmentUploadAdditionalInfo, + AttachmentUploadInfo, + DatasetUploadAdditionalInfo, + DatasetUploadInfo, + MagUploadAdditionalInfo, + MagUploadInfo, + ReportAttachmentUploadParameters, ReportDatasetUploadParameters, - ReserveAdditionalInformation, - ReserveUploadInformation + ReportMagUploadParameters } import com.scalableminds.webknossos.datastore.storage.DataVaultCredential import com.typesafe.scalalogging.LazyLogging @@ -90,20 +96,31 @@ class DSRemoteWebknossosClient @Inject()( def getUnfinishedUploadsForUser(organizationName: String)(implicit tc: TokenContext): Fox[List[UnfinishedUpload]] = for { - unfinishedUploads <- rpc(s"$webknossosUri/api/datastores/$dataStoreName/getUnfinishedUploadsForUser") + unfinishedUploads <- rpc(s"$webknossosUri/api/datastores/$dataStoreName/getUnfinishedDatasetUploadsForUser") .addQueryParam("key", dataStoreKey) .addQueryParam("organizationName", organizationName) .withTokenFromContext .getWithJsonResponse[List[UnfinishedUpload]] } yield unfinishedUploads - def reportUpload(datasetId: ObjectId, parameters: ReportDatasetUploadParameters)(implicit tc: TokenContext): Fox[_] = + def reportDatasetUpload(datasetId: ObjectId, parameters: ReportDatasetUploadParameters)( + implicit tc: TokenContext): Fox[_] = rpc(s"$webknossosUri/api/datastores/$dataStoreName/reportDatasetUpload") .addQueryParam("key", dataStoreKey) .addQueryParam("datasetId", datasetId) .withTokenFromContext .postJson[ReportDatasetUploadParameters](parameters) + def reportMagUpload(parameters: ReportMagUploadParameters): Fox[_] = + rpc(s"$webknossosUri/api/datastores/$dataStoreName/reportMagUpload") + .addQueryParam("key", dataStoreKey) + .postJson[ReportMagUploadParameters](parameters) + + def reportAttachmentUpload(parameters: ReportAttachmentUploadParameters): Fox[_] = + rpc(s"$webknossosUri/api/datastores/$dataStoreName/reportAttachmentUpload") + .addQueryParam("key", dataStoreKey) + .postJson[ReportAttachmentUploadParameters](parameters) + def reportDataSources(dataSources: List[DataSource], organizationId: Option[String]): Fox[_] = rpc(s"$webknossosUri/api/datastores/$dataStoreName/datasources") .addQueryParam("key", dataStoreKey) @@ -117,14 +134,24 @@ class DSRemoteWebknossosClient @Inject()( .silent .putJson(dataSourcePaths) - def reserveDataSourceUpload(info: ReserveUploadInformation)( - implicit tc: TokenContext): Fox[ReserveAdditionalInformation] = - for { - reserveUploadInfo <- rpc(s"$webknossosUri/api/datastores/$dataStoreName/reserveUpload") - .addQueryParam("key", dataStoreKey) - .withTokenFromContext - .postJsonWithJsonResponse[ReserveUploadInformation, ReserveAdditionalInformation](info) - } yield reserveUploadInfo + def reserveDatasetUpload(info: DatasetUploadInfo)(implicit tc: TokenContext): Fox[DatasetUploadAdditionalInfo] = + rpc(s"$webknossosUri/api/datastores/$dataStoreName/reserveDatasetUpload") + .addQueryParam("key", dataStoreKey) + .withTokenFromContext + .postJsonWithJsonResponse[DatasetUploadInfo, DatasetUploadAdditionalInfo](info) + + def reserveMagUpload(info: MagUploadInfo)(implicit tc: TokenContext): Fox[MagUploadAdditionalInfo] = + rpc(s"$webknossosUri/api/datastores/$dataStoreName/reserveMagUpload") + .addQueryParam("key", dataStoreKey) + .withTokenFromContext + .postJsonWithJsonResponse[MagUploadInfo, MagUploadAdditionalInfo](info) + + def reserveAttachmentUpload(info: AttachmentUploadInfo)( + implicit tc: TokenContext): Fox[AttachmentUploadAdditionalInfo] = + rpc(s"$webknossosUri/api/datastores/$dataStoreName/reserveAttachmentUpload") + .addQueryParam("key", dataStoreKey) + .withTokenFromContext + .postJsonWithJsonResponse[AttachmentUploadInfo, AttachmentUploadAdditionalInfo](info) def updateDataSource(dataSource: DataSource, datasetId: ObjectId)(implicit tc: TokenContext): Fox[_] = rpc(s"$webknossosUri/api/datastores/$dataStoreName/datasources/${datasetId.toString}") diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index db740ffc23f..335c3f4d604 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -56,10 +56,10 @@ class DataSourceService @Inject()( _ = if (inboxCheckVerboseCounter >= 10) inboxCheckVerboseCounter = 0 } yield () - def assertDataDirWritable(organizationId: String): Fox[Unit] = { - val orgaPath = dataBaseDir.resolve(organizationId) + def ensureDataDirWritable(dataSourceId: DataSourceId): Fox[Unit] = { + val orgaPath = dataBaseDir.resolve(dataSourceId.organizationId) if (orgaPath.toFile.exists()) { - Fox.fromBool(Files.isWritable(dataBaseDir.resolve(organizationId))) ?~> "Datastore cannot write to organization data directory." + Fox.fromBool(Files.isWritable(orgaPath)) ?~> "Datastore cannot write to organization data directory." } else { tryo { Files.createDirectory(orgaPath) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadDomain.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadDomain.scala new file mode 100644 index 00000000000..3a203f19541 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadDomain.scala @@ -0,0 +1,8 @@ +package com.scalableminds.webknossos.datastore.services.uploading + +import com.scalableminds.util.enumeration.ExtendedEnumeration; + +object UploadDomain extends ExtendedEnumeration { + type UploadDomain = Value + val dataset, mag, attachment = Value +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadMetadataStore.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadMetadataStore.scala new file mode 100644 index 00000000000..37d38f4c833 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadMetadataStore.scala @@ -0,0 +1,254 @@ +package com.scalableminds.webknossos.datastore.services.uploading + +import com.scalableminds.util.objectid.ObjectId +import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.dataformats.MagLocator +import com.scalableminds.webknossos.datastore.models.VoxelSize +import com.scalableminds.webknossos.datastore.models.datasource.LayerAttachmentType.LayerAttachmentType +import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, LayerAttachment} +import com.scalableminds.webknossos.datastore.services.uploading.UploadDomain.UploadDomain +import com.scalableminds.webknossos.datastore.storage.DataStoreRedisStore +import play.api.libs.json.Json + +import javax.inject.Inject +import scala.concurrent.ExecutionContext + +trait UploadMetadataStore extends FoxImplicits { + + protected def domain: UploadDomain + protected def store: DataStoreRedisStore + + protected def keyPrefix = s"upload___${domain}___" + + /* + * Redis stores different information for each running upload, with different prefixes in the keys. + * Note that Redis synchronizes all db accesses, so we do not need to do it. + */ + private def redisKeyForFileCount(uploadId: String): String = + s"$keyPrefix${uploadId}___fileCount" + + private def redisKeyForTotalFileSizeInBytes(uploadId: String): String = + s"$keyPrefix${uploadId}___totalFileSizeInBytes" + + private def redisKeyForFileNameSet(uploadId: String): String = + s"$keyPrefix${uploadId}___fileNameSet" + + private def redisKeyForFileChunkCount(uploadId: String, fileName: String): String = + s"$keyPrefix${uploadId}___file___${fileName}___chunkCount" + + private def redisKeyForFileChunkSet(uploadId: String, fileName: String): String = + s"$keyPrefix${uploadId}___file___${fileName}___chunkSet" + + private def redisKeyForDataSourceId(uploadId: String): String = + s"$keyPrefix${uploadId}___dataSourceId" + + private def redisKeyForDatasetId(uploadId: String): String = + s"$keyPrefix${uploadId}___datasetId" + + private def redisKeyForFilePaths(uploadId: String): String = + s"$keyPrefix${uploadId}___filePaths" + + def isKnownUpload(uploadId: String): Fox[Boolean] = + store.contains(redisKeyForFileCount(uploadId)) + + def findDataSourceId(uploadId: String)(implicit ec: ExecutionContext): Fox[DataSourceId] = + store.findParsed[DataSourceId](redisKeyForDataSourceId(uploadId)) + + def findDatasetId(uploadId: String)(implicit ec: ExecutionContext): Fox[ObjectId] = + store.findParsed[ObjectId](redisKeyForDatasetId(uploadId)) + + def findFilePaths(uploadId: String)(implicit ec: ExecutionContext): Fox[Seq[String]] = + store.findParsed[Seq[String]](redisKeyForFilePaths(uploadId)) + + def findTotalFileSizeInBytes(uploadId: String): Fox[Long] = + store.findLong(redisKeyForTotalFileSizeInBytes(uploadId)) + + def findFileCount(uploadId: String): Fox[Long] = + store.findLong(redisKeyForFileCount(uploadId)) + + def findFileNames(uploadId: String): Fox[Set[String]] = + store.findSet(redisKeyForFileNameSet(uploadId)) + + def findFileChunkCount(uploadId: String, filePath: String): Fox[Long] = + store.findLong(redisKeyForFileChunkCount(uploadId, filePath)) + + def findFileChunkSet(uploadId: String, filePath: String): Fox[Set[String]] = + store.findSet(redisKeyForFileChunkSet(uploadId, filePath)) + + def isFileKnown(uploadId: String, filePath: String): Fox[Boolean] = + store.contains(redisKeyForFileChunkCount(uploadId, filePath)) + + def isFileChunkSetKnown(uploadId: String, filePath: String): Fox[Boolean] = + store.contains(redisKeyForFileChunkSet(uploadId, filePath)) + + def isChunkPresent(uploadId: String, filePath: String, chunkNumber: Long): Fox[Boolean] = + store.isContainedInSet(redisKeyForFileChunkSet(uploadId, filePath), String.valueOf(chunkNumber)) + + def insertTotalFileCount(uploadId: String, totalFileCount: Long): Fox[Unit] = + store.insert(redisKeyForFileCount(uploadId), String.valueOf(totalFileCount)) + + def insertTotalFileSizeInBytes(uploadId: String, totalFileSizeInBytes: Option[Long])( + implicit ec: ExecutionContext): Fox[Option[Unit]] = + Fox.runOptional(totalFileSizeInBytes) { + store.insertLong(redisKeyForTotalFileSizeInBytes(uploadId), _) + } + + def insertFilePathIntoSet(uploadId: String, filePath: String): Fox[Boolean] = + store.insertIntoSet(redisKeyForFileNameSet(uploadId), filePath) + + def insertFileChunkCount(uploadId: String, filePath: String, totalChunkCount: Long): Fox[Unit] = + store.insert(redisKeyForFileChunkCount(uploadId, filePath), String.valueOf(totalChunkCount)) + + def insertFileChunkIntoSet(uploadId: String, filePath: String, chunkNumber: Long): Fox[Boolean] = + store.insertIntoSet(redisKeyForFileChunkSet(uploadId, filePath), String.valueOf(chunkNumber)) + + def removeFileChunkFromSet(uploadId: String, filePath: String, chunkNumber: Long): Fox[Boolean] = + store.removeFromSet(redisKeyForFileChunkSet(uploadId, filePath), String.valueOf(chunkNumber)) + + def insertDatasetId(uploadId: String, datasetId: ObjectId): Fox[Unit] = + store.insertSerialized(redisKeyForDatasetId(uploadId), datasetId) + + def insertDataSourceId(uploadId: String, dataSourceId: DataSourceId): Fox[Unit] = + store.insertSerialized(redisKeyForDataSourceId(uploadId), dataSourceId) + + def insertFilePaths(uploadId: String, filePaths: Option[Seq[String]]): Fox[Unit] = + store.insertSerialized(redisKeyForFilePaths(uploadId), filePaths.getOrElse(Seq.empty)) + + def cleanUp(uploadId: String)(implicit ec: ExecutionContext): Fox[Unit] = + for { + _ <- store.remove(redisKeyForFileCount(uploadId)) + fileNames <- store.findSet(redisKeyForFileNameSet(uploadId)) + _ <- Fox.serialCombined(fileNames.toList) { fileName => + for { + _ <- store.remove(redisKeyForFileChunkCount(uploadId, fileName)) + _ <- store.remove(redisKeyForFileChunkSet(uploadId, fileName)) + } yield () + } + _ <- store.remove(redisKeyForFileNameSet(uploadId)) + _ <- store.remove(redisKeyForTotalFileSizeInBytes(uploadId)) + _ <- store.remove(redisKeyForDataSourceId(uploadId)) + _ <- store.remove(redisKeyForDatasetId(uploadId)) + _ <- store.remove(redisKeyForFilePaths(uploadId)) + } yield () + +} + +class DatasetUploadMetadataStore @Inject()(protected val store: DataStoreRedisStore) extends UploadMetadataStore { + protected val domain: UploadDomain = UploadDomain.dataset + + private def redisKeyForUploadIdByDataSourceId(datasourceId: DataSourceId): String = + s"${keyPrefix}___${Json.stringify(Json.toJson(datasourceId))}___datasourceId" + + private def redisKeyForLinkedLayerIdentifier(uploadId: String): String = + s"$keyPrefix${uploadId}___linkedLayerIdentifier" + + private def redisKeyForNeedsConversion(uploadId: String): String = + s"$keyPrefix${uploadId}___needsConversion" + + private def redisKeyForVoxelSize(uploadId: String): String = + s"$keyPrefix${uploadId}___voxelSize" + + def findUploadIdByDataSourceId(dataSourceId: DataSourceId): Fox[String] = + store.find(redisKeyForUploadIdByDataSourceId(dataSourceId)) + + def findLinkedLayerIdentifiers(uploadId: String)(implicit ec: ExecutionContext): Fox[Seq[LinkedLayerIdentifier]] = + store.findParsed[Seq[LinkedLayerIdentifier]](redisKeyForLinkedLayerIdentifier(uploadId)) + + def findNeedsConversion(uploadId: String)(implicit ec: ExecutionContext): Fox[Boolean] = + store.findParsed[Boolean](redisKeyForNeedsConversion(uploadId)) + + def findVoxelSize(uploadId: String)(implicit ec: ExecutionContext): Fox[VoxelSize] = + store.findParsed[VoxelSize](redisKeyForVoxelSize(uploadId)) + + // Only here the uploadId is not key but value. This is used to re-connect to unfinished uploads. + def insertUploadIdByDataSourceId(dataSourceId: DataSourceId, uploadId: String): Fox[Unit] = + store.insertSerialized(redisKeyForUploadIdByDataSourceId(dataSourceId), uploadId) + + def insertLinkedLayerIdentifiers(uploadId: String, + linkedLayerIdentifiers: Option[Seq[LinkedLayerIdentifier]]): Fox[_] = + store.insertSerialized(redisKeyForLinkedLayerIdentifier(uploadId), linkedLayerIdentifiers.getOrElse(Seq.empty)) + + def insertNeedsConversion(uploadId: String, needsConversion: Boolean): Fox[_] = + store.insertSerialized(redisKeyForNeedsConversion(uploadId), needsConversion) + + def insertVoxelSize(uploadId: String, voxelSize: VoxelSize): Fox[_] = + store.insertSerialized(redisKeyForVoxelSize(uploadId), voxelSize) + + override def cleanUp(uploadId: String)(implicit ec: ExecutionContext): Fox[Unit] = + for { + dataSourceId <- findDataSourceId(uploadId) + _ <- store.remove(redisKeyForLinkedLayerIdentifier(uploadId)) + _ <- store.remove(redisKeyForNeedsConversion(uploadId)) + _ <- store.remove(redisKeyForUploadIdByDataSourceId(dataSourceId)) + _ <- super.cleanUp(uploadId) + } yield () + +} + +class MagUploadMetadataStore @Inject()(protected val store: DataStoreRedisStore) extends UploadMetadataStore { + protected val domain: UploadDomain = UploadDomain.mag + + private def redisKeyForMag(uploadId: String): String = + s"$keyPrefix${uploadId}___mag" + + private def redisKeyForLayerName(uploadId: String): String = + s"$keyPrefix${uploadId}___layerName" + + def insertMag(uploadId: String, mag: MagLocator): Fox[Unit] = + store.insertSerialized[MagLocator](redisKeyForMag(uploadId), mag) + + def insertLayerName(uploadId: String, layerName: String): Fox[Unit] = + store.insert(redisKeyForLayerName(uploadId), layerName) + + def findMag(uploadId: String)(implicit ec: ExecutionContext): Fox[MagLocator] = + store.findParsed[MagLocator](redisKeyForMag(uploadId)) + + def findLayerName(uploadId: String): Fox[String] = + store.find(redisKeyForLayerName(uploadId)) + + override def cleanUp(uploadId: String)(implicit ec: ExecutionContext): Fox[Unit] = + for { + _ <- store.remove(redisKeyForMag(uploadId)) + _ <- store.remove(redisKeyForLayerName(uploadId)) + _ <- super.cleanUp(uploadId) + } yield () +} + +class AttachmentUploadMetadataStore @Inject()(protected val store: DataStoreRedisStore) extends UploadMetadataStore { + protected val domain: UploadDomain = UploadDomain.attachment + + private def redisKeyForAttachment(uploadId: String): String = + s"$keyPrefix${uploadId}___attachment" + + private def redisKeyForAttachmentType(uploadId: String): String = + s"$keyPrefix${uploadId}___attachmentType" + + private def redisKeyForLayerName(uploadId: String): String = + s"$keyPrefix${uploadId}___layerName" + + def insertAttachment(uploadId: String, attachment: LayerAttachment): Fox[Unit] = + store.insertSerialized[LayerAttachment](redisKeyForAttachment(uploadId), attachment) + + def insertAttachmentType(uploadId: String, attachmentType: LayerAttachmentType): Fox[Unit] = + store.insertSerialized[LayerAttachmentType](redisKeyForAttachmentType(uploadId), attachmentType) + + def insertLayerName(uploadId: String, layerName: String): Fox[Unit] = + store.insert(redisKeyForLayerName(uploadId), layerName) + + def findAttachment(uploadId: String)(implicit ec: ExecutionContext): Fox[LayerAttachment] = + store.findParsed[LayerAttachment](redisKeyForAttachment(uploadId)) + + def findAttachmentType(uploadId: String)(implicit ec: ExecutionContext): Fox[LayerAttachmentType] = + store.findParsed[LayerAttachmentType](redisKeyForAttachmentType(uploadId)) + + def findLayerName(uploadId: String): Fox[String] = + store.find(redisKeyForLayerName(uploadId)) + + override def cleanUp(uploadId: String)(implicit ec: ExecutionContext): Fox[Unit] = + for { + _ <- store.remove(redisKeyForAttachmentType(uploadId)) + _ <- store.remove(redisKeyForAttachment(uploadId)) + _ <- store.remove(redisKeyForLayerName(uploadId)) + _ <- super.cleanUp(uploadId) + } yield () +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala index 58303aa28b7..8b6f5cefb6f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala @@ -9,6 +9,7 @@ import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Box.tryo import com.scalableminds.util.tools._ import com.scalableminds.webknossos.datastore.DataStoreConfig +import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.wkw.WKWDataFormatHelper import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header.FILENAME_ATTRIBUTES_JSON import com.scalableminds.webknossos.datastore.datareaders.n5.{N5Header, N5Metadata} @@ -19,14 +20,16 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader import com.scalableminds.webknossos.datastore.explore.ExploreLocalLayerService import com.scalableminds.webknossos.datastore.helpers.{DatasetDeleter, DirectoryConstants, UPath} import com.scalableminds.webknossos.datastore.models.LengthUnit.LengthUnit -import com.scalableminds.webknossos.datastore.models.UnfinishedUpload +import com.scalableminds.webknossos.datastore.models.{UnfinishedUpload, VoxelSize} +import com.scalableminds.webknossos.datastore.models.datasource.LayerAttachmentType.LayerAttachmentType import com.scalableminds.webknossos.datastore.models.datasource.UsableDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON import com.scalableminds.webknossos.datastore.models.datasource._ +import com.scalableminds.webknossos.datastore.services.uploading.UploadDomain.UploadDomain import com.scalableminds.webknossos.datastore.services.{DSRemoteWebknossosClient, DataSourceService, ManagedS3Service} -import com.scalableminds.webknossos.datastore.storage.{DataStoreRedisStore, DataVaultService} +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.typesafe.scalalogging.LazyLogging import org.apache.commons.io.FileUtils -import play.api.libs.json.{Json, OFormat, Reads} +import play.api.libs.json.{Json, OFormat} import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest import java.io.{File, RandomAccessFile} @@ -35,28 +38,72 @@ import java.nio.file.{Files, Path} import scala.concurrent.{ExecutionContext, Future} import scala.jdk.FutureConverters._ -case class ReserveUploadInformation( +case class ResumableUploadInfo( uploadId: String, // upload id that was also used in chunk upload (this time without file paths) - name: String, // dataset name - organization: String, totalFileCount: Long, - filePaths: Option[List[String]], + filePaths: Option[Seq[String]], totalFileSizeInBytes: Option[Long], - layersToLink: Option[List[LinkedLayerIdentifier]], - initialTeams: List[ObjectId], // team ids +) +object ResumableUploadInfo { + implicit val jsonFormat: OFormat[ResumableUploadInfo] = Json.format[ResumableUploadInfo] +} + +case class DatasetUploadInfo( + resumableUploadInfo: ResumableUploadInfo, + datasetName: String, + organizationId: String, + layersToLink: Option[Seq[LinkedLayerIdentifier]], + initialTeamIds: Seq[ObjectId], // team ids folderId: Option[ObjectId], requireUniqueName: Option[Boolean], isVirtual: Option[Boolean], // Only set (to false) for legacy manual uploads - needsConversion: Option[Boolean] // None means false + needsConversion: Option[Boolean], // None means false + voxelSizeFactor: Option[Vec3Double], + voxelSizeUnit: Option[LengthUnit] +) +object DatasetUploadInfo { + implicit val jsonFormat: OFormat[DatasetUploadInfo] = Json.format[DatasetUploadInfo] +} + +case class MagUploadInfo( + resumableUploadInfo: ResumableUploadInfo, + datasetId: ObjectId, + layerName: String, + mag: MagLocator, + overwritePending: Boolean +) +object MagUploadInfo { + implicit val jsonFormat: OFormat[MagUploadInfo] = Json.format[MagUploadInfo] +} + +case class AttachmentUploadInfo( + resumableUploadInfo: ResumableUploadInfo, + datasetId: ObjectId, + layerName: String, + attachmentType: LayerAttachmentType, + attachment: LayerAttachment, + overwritePending: Boolean ) -object ReserveUploadInformation { - implicit val jsonFormat: OFormat[ReserveUploadInformation] = Json.format[ReserveUploadInformation] +object AttachmentUploadInfo { + implicit val jsonFormat: OFormat[AttachmentUploadInfo] = Json.format[AttachmentUploadInfo] +} + +case class DatasetUploadAdditionalInfo(newDatasetId: ObjectId, directoryName: String) +object DatasetUploadAdditionalInfo { + implicit val jsonFormat: OFormat[DatasetUploadAdditionalInfo] = + Json.format[DatasetUploadAdditionalInfo] +} + +case class MagUploadAdditionalInfo(dataSourceId: DataSourceId) +object MagUploadAdditionalInfo { + implicit val jsonFormat: OFormat[MagUploadAdditionalInfo] = + Json.format[MagUploadAdditionalInfo] } -case class ReserveAdditionalInformation(newDatasetId: ObjectId, directoryName: String) -object ReserveAdditionalInformation { - implicit val jsonFormat: OFormat[ReserveAdditionalInformation] = - Json.format[ReserveAdditionalInformation] +case class AttachmentUploadAdditionalInfo(dataSourceId: DataSourceId) +object AttachmentUploadAdditionalInfo { + implicit val jsonFormat: OFormat[AttachmentUploadAdditionalInfo] = + Json.format[AttachmentUploadAdditionalInfo] } case class ReportDatasetUploadParameters( @@ -64,13 +111,31 @@ case class ReportDatasetUploadParameters( datasetSizeBytes: Long, dataSourceOpt: Option[UsableDataSource], // must be set if needsConversion is false layersToLink: Seq[LinkedLayerIdentifier], - voxelSizeFactor: Option[Vec3Double], - voxelSizeUnit: Option[LengthUnit] + voxelSize: Option[VoxelSize] ) object ReportDatasetUploadParameters { implicit val jsonFormat: OFormat[ReportDatasetUploadParameters] = Json.format[ReportDatasetUploadParameters] } +case class ReportMagUploadParameters( + datasetId: ObjectId, + layerName: String, + mag: MagLocator, + magSizeBytes: Long +) +object ReportMagUploadParameters { + implicit val jsonFormat: OFormat[ReportMagUploadParameters] = Json.format[ReportMagUploadParameters] +} +case class ReportAttachmentUploadParameters( + datasetId: ObjectId, + layerName: String, + attachmentType: LayerAttachmentType, + attachment: LayerAttachment, + attachmentSizeBytes: Long +) +object ReportAttachmentUploadParameters { + implicit val jsonFormat: OFormat[ReportAttachmentUploadParameters] = Json.format[ReportAttachmentUploadParameters] +} case class LinkedLayerIdentifier(datasetId: ObjectId, layerName: String, newLayerName: Option[String] = None) @@ -78,27 +143,10 @@ object LinkedLayerIdentifier { implicit val jsonFormat: OFormat[LinkedLayerIdentifier] = Json.format[LinkedLayerIdentifier] } -case class LinkedLayerIdentifiers(layersToLink: Option[List[LinkedLayerIdentifier]]) -object LinkedLayerIdentifiers { - implicit val jsonFormat: OFormat[LinkedLayerIdentifiers] = Json.format[LinkedLayerIdentifiers] -} - -case class UploadInformation(uploadId: String, - needsConversion: Option[Boolean], - voxelSizeFactor: Option[Vec3Double], - voxelSizeUnit: Option[LengthUnit]) - -object UploadInformation { - implicit val jsonFormat: OFormat[UploadInformation] = Json.format[UploadInformation] -} - -case class CancelUploadInformation(uploadId: String) -object CancelUploadInformation { - implicit val jsonFormat: OFormat[CancelUploadInformation] = Json.format[CancelUploadInformation] -} - class UploadService @Inject()(dataSourceService: DataSourceService, - runningUploadMetadataStore: DataStoreRedisStore, + datasetUploadMetadataStore: DatasetUploadMetadataStore, + magUploadMetadataStore: MagUploadMetadataStore, + attachmentUploadMetadataStore: AttachmentUploadMetadataStore, dataVaultService: DataVaultService, exploreLocalLayerService: ExploreLocalLayerService, dataStoreConfig: DataStoreConfig, @@ -110,89 +158,92 @@ class UploadService @Inject()(dataSourceService: DataSourceService, with WKWDataFormatHelper with LazyLogging { - /* - * Redis stores different information for each running upload, with different prefixes in the keys. - * Note that Redis synchronizes all db accesses, so we do not need to do it. - */ - private def redisKeyForFileCount(uploadId: String): String = - s"upload___${uploadId}___fileCount" - private def redisKeyForTotalFileSizeInBytes(uploadId: String): String = - s"upload___${uploadId}___totalFileSizeInBytes" - private def redisKeyForFileNameSet(uploadId: String): String = - s"upload___${uploadId}___fileNameSet" - private def redisKeyForDataSourceId(uploadId: String): String = - s"upload___${uploadId}___dataSourceId" - private def redisKeyForLinkedLayerIdentifier(uploadId: String): String = - s"upload___${uploadId}___linkedLayerIdentifier" - private def redisKeyForFileChunkCount(uploadId: String, fileName: String): String = - s"upload___${uploadId}___file___${fileName}___chunkCount" - private def redisKeyForFileChunkSet(uploadId: String, fileName: String): String = - s"upload___${uploadId}___file___${fileName}___chunkSet" - private def redisKeyForUploadId(datasourceId: DataSourceId): String = - s"upload___${Json.stringify(Json.toJson(datasourceId))}___datasourceId" - private def redisKeyForDatasetId(uploadId: String): String = - s"upload___${uploadId}___datasetId" - private def redisKeyForFilePaths(uploadId: String): String = - s"upload___${uploadId}___filePaths" - cleanUpOrphanUploads() + private def selectUploadMetadataStore(uploadDomain: UploadDomain) = uploadDomain match { + case UploadDomain.dataset => datasetUploadMetadataStore + case UploadDomain.mag => magUploadMetadataStore + case UploadDomain.attachment => attachmentUploadMetadataStore + } + override def dataBaseDir: Path = dataSourceService.dataBaseDir - def isKnownUploadByFileId(uploadFileId: String): Fox[Boolean] = isKnownUpload(extractDatasetUploadId(uploadFileId)) + def isKnownUploadByFileId(uploadFileId: String, uploadDomain: UploadDomain): Fox[Boolean] = + selectUploadMetadataStore(uploadDomain).isKnownUpload(extractDatasetUploadId(uploadFileId)) + + def isKnownUpload(uploadId: String, uploadDomain: UploadDomain): Fox[Boolean] = + selectUploadMetadataStore(uploadDomain).isKnownUpload(uploadId) - def isKnownUpload(uploadId: String): Fox[Boolean] = - runningUploadMetadataStore.contains(redisKeyForFileCount(uploadId)) + def getDatasetIdByUploadId(uploadId: String, uploadDomain: UploadDomain): Fox[ObjectId] = + selectUploadMetadataStore(uploadDomain).findDatasetId(uploadId) def extractDatasetUploadId(uploadFileId: String): String = uploadFileId.split("/").headOption.getOrElse("") - private def uploadDirectoryFor(organizationId: String, uploadId: String): Path = - dataBaseDir.resolve(organizationId).resolve(uploadingDir).resolve(uploadId) + private def uploadDirectoryFor(organizationId: String, uploadId: String, uploadDomain: UploadDomain): Path = + dataBaseDir.resolve(organizationId).resolve(uploadingDir).resolve(uploadDomain.toString).resolve(uploadId) private def uploadBackupDirectoryFor(organizationId: String, uploadId: String): Path = dataBaseDir.resolve(organizationId).resolve(trashDir).resolve(s"uploadBackup__$uploadId") - private def getDataSourceIdByUploadId(uploadId: String): Fox[DataSourceId] = - getObjectFromRedis[DataSourceId](redisKeyForDataSourceId(uploadId)) - - def getDatasetIdByUploadId(uploadId: String): Fox[ObjectId] = - getObjectFromRedis[ObjectId](redisKeyForDatasetId(uploadId)) - - def reserveUpload(reserveUploadInfo: ReserveUploadInformation, - reserveUploadAdditionalInfo: ReserveAdditionalInformation): Fox[Unit] = - for { - _ <- dataSourceService.assertDataDirWritable(reserveUploadInfo.organization) - newDataSourceId = DataSourceId(reserveUploadAdditionalInfo.directoryName, reserveUploadInfo.organization) - _ = logger.info( - f"Reserving ${uploadFullName(reserveUploadInfo.uploadId, reserveUploadAdditionalInfo.newDatasetId, newDataSourceId)}...") - _ <- Fox.fromBool( - !reserveUploadInfo.needsConversion.getOrElse(false) || !reserveUploadInfo.layersToLink - .exists(_.nonEmpty)) ?~> "Cannot use linked layers if the dataset needs conversion" - _ <- runningUploadMetadataStore.insert(redisKeyForFileCount(reserveUploadInfo.uploadId), - String.valueOf(reserveUploadInfo.totalFileCount)) - _ <- Fox.runOptional(reserveUploadInfo.totalFileSizeInBytes)( - runningUploadMetadataStore.insertLong(redisKeyForTotalFileSizeInBytes(reserveUploadInfo.uploadId), _)) - _ <- runningUploadMetadataStore.insert( - redisKeyForDataSourceId(reserveUploadInfo.uploadId), - Json.stringify(Json.toJson(newDataSourceId)) - ) - _ <- runningUploadMetadataStore.insert( - redisKeyForDatasetId(reserveUploadInfo.uploadId), - Json.stringify(Json.toJson(reserveUploadAdditionalInfo.newDatasetId)) - ) - _ <- runningUploadMetadataStore.insert( - redisKeyForUploadId(DataSourceId(reserveUploadAdditionalInfo.directoryName, reserveUploadInfo.organization)), - reserveUploadInfo.uploadId - ) - filePaths = Json.stringify(Json.toJson(reserveUploadInfo.filePaths.getOrElse(List.empty))) - _ <- runningUploadMetadataStore.insert(redisKeyForFilePaths(reserveUploadInfo.uploadId), filePaths) - _ <- runningUploadMetadataStore.insert( - redisKeyForLinkedLayerIdentifier(reserveUploadInfo.uploadId), - Json.stringify(Json.toJson(LinkedLayerIdentifiers(reserveUploadInfo.layersToLink))) - ) + def reserveDatasetUpload(datasetUploadInfo: DatasetUploadInfo, + datasetId: ObjectId, + directoryName: String): Fox[Unit] = { + val dataSourceId = DataSourceId(directoryName, datasetUploadInfo.organizationId) + val needsConversion = datasetUploadInfo.needsConversion.getOrElse(false) + for { + _ <- Fox.fromBool(!needsConversion || !datasetUploadInfo.layersToLink.exists(_.nonEmpty)) ?~> "Cannot use linked layers if the dataset needs conversion" + _ <- reserveResumableUpload(datasetUploadInfo.resumableUploadInfo, datasetId, dataSourceId, UploadDomain.dataset) + uploadId = datasetUploadInfo.resumableUploadInfo.uploadId + voxelSizeOpt = datasetUploadInfo.voxelSizeFactor.map( + VoxelSize.fromFactorAndUnitWithDefault(_, datasetUploadInfo.voxelSizeUnit)) + _ <- datasetUploadMetadataStore.insertUploadIdByDataSourceId(dataSourceId, uploadId) + _ <- datasetUploadMetadataStore.insertLinkedLayerIdentifiers(uploadId, datasetUploadInfo.layersToLink) + _ <- datasetUploadMetadataStore.insertNeedsConversion(uploadId, needsConversion) + _ <- Fox.runOptional(voxelSizeOpt)(datasetUploadMetadataStore.insertVoxelSize(uploadId, _)) + } yield () + } + + def reserveMagUpload(magUploadInfo: MagUploadInfo, dataSourceId: DataSourceId): Fox[Unit] = { + val uploadId = magUploadInfo.resumableUploadInfo.uploadId + for { + _ <- reserveResumableUpload(magUploadInfo.resumableUploadInfo, + magUploadInfo.datasetId, + dataSourceId, + UploadDomain.mag) + _ <- magUploadMetadataStore.insertMag(uploadId, magUploadInfo.mag.withoutCredentials) + _ <- magUploadMetadataStore.insertLayerName(uploadId, magUploadInfo.layerName) + } yield () + } + + def reserveAttachmentUpload(attachmentUploadInfo: AttachmentUploadInfo, dataSourceId: DataSourceId): Fox[Unit] = + for { + _ <- reserveResumableUpload(attachmentUploadInfo.resumableUploadInfo, + attachmentUploadInfo.datasetId, + dataSourceId, + UploadDomain.attachment) + uploadId = attachmentUploadInfo.resumableUploadInfo.uploadId + _ <- attachmentUploadMetadataStore.insertAttachment(uploadId, attachmentUploadInfo.attachment.withoutCredential) + _ <- attachmentUploadMetadataStore.insertAttachmentType(uploadId, attachmentUploadInfo.attachmentType) + _ <- attachmentUploadMetadataStore.insertLayerName(uploadId, attachmentUploadInfo.layerName) } yield () - def addUploadIdsToUnfinishedUploads( + private def reserveResumableUpload(resumableUploadInfo: ResumableUploadInfo, + datasetId: ObjectId, + dataSourceId: DataSourceId, + uploadDomain: UploadDomain): Fox[Unit] = + for { + _ <- dataSourceService.ensureDataDirWritable(dataSourceId) + uploadId = resumableUploadInfo.uploadId + _ = logger.info(f"Reserving ${uploadFullName(uploadDomain, uploadId, datasetId, dataSourceId)}...") + uploadMetadataStore = selectUploadMetadataStore(uploadDomain) + _ <- uploadMetadataStore.insertDataSourceId(uploadId, dataSourceId) + _ <- uploadMetadataStore.insertDatasetId(uploadId, datasetId) + _ <- uploadMetadataStore.insertTotalFileCount(uploadId, resumableUploadInfo.totalFileCount) + _ <- uploadMetadataStore.insertTotalFileSizeInBytes(uploadId, resumableUploadInfo.totalFileSizeInBytes) + _ <- uploadMetadataStore.insertFilePaths(uploadId, resumableUploadInfo.filePaths) + } yield () + + def enrichUnfinishedUploadInfoWithUploadIds( unfinishedUploadsWithoutIds: List[UnfinishedUpload]): Future[List[UnfinishedUpload]] = for { maybeUnfinishedUploads: List[Box[Option[UnfinishedUpload]]] <- Fox.sequence( @@ -200,15 +251,14 @@ class UploadService @Inject()(dataSourceService: DataSourceService, unfinishedUploadsWithoutIds.map( unfinishedUpload => { for { - uploadIdOpt <- runningUploadMetadataStore.find(redisKeyForUploadId(unfinishedUpload.dataSourceId)) - updatedUploadOpt = uploadIdOpt.map(uploadId => unfinishedUpload.copy(uploadId = uploadId)) + uploadIdBox <- datasetUploadMetadataStore + .findUploadIdByDataSourceId(unfinishedUpload.dataSourceId) + .shiftBox + updatedUploadOpt = uploadIdBox.toOption.map(uploadId => unfinishedUpload.copy(uploadId = uploadId)) updatedUploadWithFilePathsOpt <- Fox.runOptional(updatedUploadOpt)(updatedUpload => for { - filePathsStringOpt <- runningUploadMetadataStore.find(redisKeyForFilePaths(updatedUpload.uploadId)) - filePathsOpt <- filePathsStringOpt.map(JsonHelper.parseAs[List[String]]).toFox - uploadUpdatedWithFilePaths <- filePathsOpt - .map(filePaths => updatedUpload.copy(filePaths = Some(filePaths))) - .toFox + filePaths <- datasetUploadMetadataStore.findFilePaths(updatedUpload.uploadId) + uploadUpdatedWithFilePaths = updatedUpload.copy(filePaths = Some(filePaths)) } yield uploadUpdatedWithFilePaths) } yield updatedUploadWithFilePathsOpt } @@ -219,27 +269,28 @@ class UploadService @Inject()(dataSourceService: DataSourceService, private def isOutsideUploadDir(uploadDir: Path, filePath: String): Boolean = uploadDir.relativize(uploadDir.resolve(filePath)).startsWith("../") - private def getFilePathAndDirOfUploadId(uploadFileId: String): Fox[(String, Path)] = { + private def getFilePathAndDirForUploadFileId(uploadFileId: String, + uploadDomain: UploadDomain): Fox[(String, Path)] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) val uploadId = extractDatasetUploadId(uploadFileId) for { - dataSourceId <- getDataSourceIdByUploadId(uploadId) - uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId) + dataSourceId <- uploadMetadataStore.findDataSourceId(uploadId) + uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId, uploadDomain) filePathRaw = uploadFileId.split("/").tail.mkString("/") filePath = if (filePathRaw.charAt(0) == '/') filePathRaw.drop(1) else filePathRaw _ <- Fox.fromBool(!isOutsideUploadDir(uploadDir, filePath)) ?~> s"Invalid file path: $filePath" } yield (filePath, uploadDir) } - def isChunkPresent(uploadFileId: String, currentChunkNumber: Long): Fox[Boolean] = { + def isChunkPresent(uploadFileId: String, currentChunkNumber: Long, uploadDomain: UploadDomain): Fox[Boolean] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) val uploadId = extractDatasetUploadId(uploadFileId) for { - (filePath, _) <- getFilePathAndDirOfUploadId(uploadFileId) - isFileKnown <- runningUploadMetadataStore.contains(redisKeyForFileChunkCount(uploadId, filePath)) - isFilesChunkSetKnown <- Fox.runIf(isFileKnown)( - runningUploadMetadataStore.contains(redisKeyForFileChunkSet(uploadId, filePath))) + (filePath, _) <- getFilePathAndDirForUploadFileId(uploadFileId, uploadDomain) + isFileKnown <- uploadMetadataStore.isFileKnown(uploadId, filePath) + isFilesChunkSetKnown <- Fox.runIf(isFileKnown)(uploadMetadataStore.isFileChunkSetKnown(uploadId, filePath)) isChunkPresent <- Fox.runIf(isFileKnown)( - runningUploadMetadataStore.isContainedInSet(redisKeyForFileChunkSet(uploadId, filePath), - String.valueOf(currentChunkNumber))) + uploadMetadataStore.isChunkPresent(uploadId, filePath, currentChunkNumber)) } yield isFileKnown && isFilesChunkSetKnown.getOrElse(false) && isChunkPresent.getOrElse(false) } @@ -248,22 +299,22 @@ class UploadService @Inject()(dataSourceService: DataSourceService, currentChunkSize: Long, totalChunkCount: Long, currentChunkNumber: Long, - chunkFile: File): Fox[Unit] = { + chunkFile: File, + uploadDomain: UploadDomain): Fox[Unit] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) val uploadId = extractDatasetUploadId(uploadFileId) for { - datasetId <- getDatasetIdByUploadId(uploadId) - dataSourceId <- getDataSourceIdByUploadId(uploadId) - (filePath, uploadDir) <- getFilePathAndDirOfUploadId(uploadFileId) - isFileKnown <- runningUploadMetadataStore.contains(redisKeyForFileChunkCount(uploadId, filePath)) + datasetId <- uploadMetadataStore.findDatasetId(uploadId) + dataSourceId <- uploadMetadataStore.findDataSourceId(uploadId) + (filePath, uploadDir) <- getFilePathAndDirForUploadFileId(uploadFileId, uploadDomain) + isFileKnown <- uploadMetadataStore.isFileKnown(uploadId, filePath) _ <- Fox.runIf(!isFileKnown) { - runningUploadMetadataStore - .insertIntoSet(redisKeyForFileNameSet(uploadId), filePath) - .flatMap(_ => - runningUploadMetadataStore.insert(redisKeyForFileChunkCount(uploadId, filePath), - String.valueOf(totalChunkCount))) + for { + _ <- uploadMetadataStore.insertFilePathIntoSet(uploadId, filePath) + _ <- uploadMetadataStore.insertFileChunkCount(uploadId, filePath, totalChunkCount) + } yield () } - isNewChunk <- runningUploadMetadataStore.insertIntoSet(redisKeyForFileChunkSet(uploadId, filePath), - String.valueOf(currentChunkNumber)) + isNewChunk <- uploadMetadataStore.insertFileChunkIntoSet(uploadId, filePath, currentChunkNumber) _ <- Fox.runIf(isNewChunk) { try { val bytes = Files.readAllBytes(chunkFile.toPath) @@ -279,10 +330,9 @@ class UploadService @Inject()(dataSourceService: DataSourceService, Fox.successful(()) } catch { case e: Exception => - runningUploadMetadataStore.removeFromSet(redisKeyForFileChunkSet(uploadId, filePath), - String.valueOf(currentChunkNumber)) + uploadMetadataStore.removeFileChunkFromSet(uploadId, filePath, currentChunkNumber) val errorMsg = - s"Error receiving chunk $currentChunkNumber for ${uploadFullName(uploadId, datasetId, dataSourceId)}: ${e.getMessage}" + s"Error receiving chunk $currentChunkNumber for ${uploadFullName(uploadDomain, uploadId, datasetId, dataSourceId)}: ${e.getMessage}" logger.warn(errorMsg) Fox.failure(errorMsg) } @@ -290,88 +340,159 @@ class UploadService @Inject()(dataSourceService: DataSourceService, } yield () } - def cancelUpload(cancelUploadInformation: CancelUploadInformation): Fox[Unit] = { - val uploadId = cancelUploadInformation.uploadId + def cancelUpload(uploadDomain: UploadDomain, uploadId: String): Fox[Unit] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) for { - dataSourceId <- getDataSourceIdByUploadId(uploadId) - datasetId <- getDatasetIdByUploadId(uploadId) - knownUpload <- isKnownUpload(uploadId) + dataSourceId <- uploadMetadataStore.findDataSourceId(uploadId) + datasetId <- uploadMetadataStore.findDatasetId(uploadId) + knownUpload <- uploadMetadataStore.isKnownUpload(uploadId) } yield if (knownUpload) { - logger.info(f"Cancelling ${uploadFullName(uploadId, datasetId, dataSourceId)}...") - cleanUpUploadedDataset(uploadDirectoryFor(dataSourceId.organizationId, uploadId), - uploadId, - reason = "Cancelled by user") + logger.info(f"Cancelling ${uploadFullName(uploadDomain, uploadId, datasetId, dataSourceId)}...") + cleanUpUploaded(uploadId, reason = "Cancelled by user", uploadDomain) } else Fox.failure(s"Unknown upload") } - private def uploadFullName(uploadId: String, datasetId: ObjectId, dataSourceId: DataSourceId) = - s"upload $uploadId of dataset $datasetId ($dataSourceId)" - - def finishUpload(uploadInformation: UploadInformation, datasetId: ObjectId)(implicit tc: TokenContext): Fox[Unit] = { - val uploadId = uploadInformation.uploadId + private def uploadFullName(uploadDomain: UploadDomain, + uploadId: String, + datasetId: ObjectId, + dataSourceId: DataSourceId) = + s"upload $uploadId for $uploadDomain (dataset $datasetId - $dataSourceId)" + def finishDatasetUpload(uploadId: String, datasetId: ObjectId)(implicit tc: TokenContext): Fox[Unit] = for { - dataSourceId <- getDataSourceIdByUploadId(uploadId) - _ = logger.info(s"Finishing ${uploadFullName(uploadId, datasetId, dataSourceId)}...") - linkedLayerIdentifiers <- getObjectFromRedis[LinkedLayerIdentifiers](redisKeyForLinkedLayerIdentifier(uploadId)) - needsConversion = uploadInformation.needsConversion.getOrElse(false) - uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId) + dataSourceId <- datasetUploadMetadataStore.findDataSourceId(uploadId) + needsConversion <- datasetUploadMetadataStore.findNeedsConversion(uploadId) + voxelSizeBox <- datasetUploadMetadataStore.findVoxelSize(uploadId).shiftBox + _ = logger.info(s"Finishing ${uploadFullName(UploadDomain.dataset, uploadId, datasetId, dataSourceId)}...") + linkedLayerIdentifiers <- datasetUploadMetadataStore.findLinkedLayerIdentifiers(uploadId) + uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId, UploadDomain.dataset) _ <- backupRawUploadedData(uploadDir, uploadBackupDirectoryFor(dataSourceId.organizationId, uploadId), datasetId).toFox - _ <- checkWithinRequestedFileSize(uploadDir, uploadId, datasetId) ?~> "dataset.upload.fileSizeCheck.failed" - _ <- checkAllChunksUploaded(uploadId) ?~> "dataset.upload.allChunksUploadedCheck.failed" - unpackToDir = unpackToDirFor(dataSourceId) - _ <- PathUtils.ensureDirectoryBox(unpackToDir.getParent).toFox ?~> "dataset.import.fileAccessDenied" - unpackResult <- unpackDataset(uploadDir, unpackToDir, datasetId).shiftBox - _ <- cleanUpUploadedDataset(uploadDir, uploadId, reason = "Upload complete, data unpacked.") + _ <- checkWithinRequestedFileSize(uploadDir, uploadId, datasetId, UploadDomain.dataset) ?~> "dataset.upload.fileSizeCheck.failed" + _ <- checkAllChunksUploaded(uploadId, UploadDomain.dataset) ?~> "dataset.upload.allChunksUploadedCheck.failed" + unpackToDir = unpackToDirFor(dataSourceId, UploadDomain.dataset, uploadId) + unpackResult <- unpackOrMoveUploaded(uploadDir, unpackToDir, datasetId, UploadDomain.dataset).shiftBox + _ <- cleanUpUploaded(uploadId, reason = "Upload complete, data unpacked.", UploadDomain.dataset) _ <- cleanUpOnFailure(unpackResult, datasetId, dataSourceId, - needsConversion, - label = s"unpacking to dataset to $unpackToDir") + unpackToDir, + label = s"unpacking dataset to $unpackToDir") postProcessingResult <- exploreUploadedDataSourceIfNeeded(needsConversion, unpackToDir, dataSourceId).shiftBox _ <- cleanUpOnFailure(postProcessingResult, datasetId, dataSourceId, - needsConversion, + unpackToDir, label = s"processing dataset at $unpackToDir") - datasetSizeBytes <- tryo(FileUtils.sizeOfDirectoryAsBigInteger(new File(unpackToDir.toString)).longValue).toFox ?~> "dataset.upload.measureTotalSize.failed" - dataSourceWithAbsolutePathsOpt <- moveUnpackedToTarget(unpackToDir, needsConversion, datasetId, dataSourceId) ?~> "dataset.upload.moveUnpackedToTarget.failed" - - _ <- remoteWebknossosClient.reportUpload( + datasetSizeBytes <- measureDirectorySizeBytes(unpackToDir) ?~> "dataset.upload.measureTotalSize.failed" + dataSourceWithAbsolutePathsOpt <- moveUnpackedDatasetToTarget( + unpackToDir, + needsConversion, + datasetId, + dataSourceId) ?~> "dataset.upload.moveUnpackedToTarget.failed" + _ <- remoteWebknossosClient.reportDatasetUpload( datasetId, ReportDatasetUploadParameters( - uploadInformation.needsConversion.getOrElse(false), + needsConversion, datasetSizeBytes, dataSourceWithAbsolutePathsOpt, - linkedLayerIdentifiers.layersToLink.getOrElse(List.empty), - uploadInformation.voxelSizeFactor, - uploadInformation.voxelSizeUnit + linkedLayerIdentifiers, + voxelSizeBox.toOption ) ) ?~> "dataset.upload.reportUpload.failed" } yield () - } - private def checkWithinRequestedFileSize(uploadDir: Path, uploadId: String, datasetId: ObjectId): Fox[Unit] = + private def measureDirectorySizeBytes(path: Path): Fox[Long] = + tryo(FileUtils.sizeOfDirectoryAsBigInteger(path.toFile).longValue).toFox + + def finishMagUpload(uploadId: String, datasetId: ObjectId): Fox[Unit] = + for { + dataSourceId <- magUploadMetadataStore.findDataSourceId(uploadId) + mag <- magUploadMetadataStore.findMag(uploadId) + layerName <- magUploadMetadataStore.findLayerName(uploadId) + uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId, UploadDomain.mag) + unpackToDir = unpackToDirFor(dataSourceId, UploadDomain.mag, uploadId) + .resolve(mag.mag.toMagLiteral(allowScalar = true)) + _ <- checkWithinRequestedFileSize(uploadDir, uploadId, datasetId, UploadDomain.mag) ?~> "dataset.upload.fileSizeCheck.failed" + _ <- checkAllChunksUploaded(uploadId, UploadDomain.mag) ?~> "dataset.upload.allChunksUploadedCheck.failed" + unpackResult <- unpackOrMoveUploaded(uploadDir, unpackToDir, datasetId, UploadDomain.mag).shiftBox + _ <- cleanUpOnFailure(unpackResult, + datasetId, + dataSourceId, + unpackToDir, + label = s"unpacking mag to $unpackToDir") + _ <- cleanUpUploaded(uploadId, reason = "Upload complete, data unpacked.", UploadDomain.mag) + magSizeBytes <- measureDirectorySizeBytes(unpackToDir) ?~> "dataset.upload.measureTotalSize.failed" + finalPath <- moveUnpackedMagOrAttachmentToTarget(unpackToDir, + layerName, + datasetId, + dataSourceId, + s"${mag.mag.toMagLiteral(true)}__${ObjectId.generate}", + UploadDomain.mag) + magAdapted = mag.copy(path = Some(finalPath)) + _ <- remoteWebknossosClient.reportMagUpload( + ReportMagUploadParameters(datasetId, layerName, magAdapted, magSizeBytes)) + } yield () + + def finishAttachmentUpload(uploadId: String, datasetId: ObjectId): Fox[Unit] = + for { + dataSourceId <- attachmentUploadMetadataStore.findDataSourceId(uploadId) + attachment <- attachmentUploadMetadataStore.findAttachment(uploadId) + attachmentType <- attachmentUploadMetadataStore.findAttachmentType(uploadId) + layerName <- attachmentUploadMetadataStore.findLayerName(uploadId) + uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId, UploadDomain.attachment) + unpackToDir = unpackToDirFor(dataSourceId, UploadDomain.attachment, uploadId) + _ <- checkWithinRequestedFileSize(uploadDir, uploadId, datasetId, UploadDomain.attachment) ?~> "dataset.upload.fileSizeCheck.failed" + _ <- checkAllChunksUploaded(uploadId, UploadDomain.attachment) ?~> "dataset.upload.allChunksUploadedCheck.failed" + unpackResult <- unpackOrMoveUploaded(uploadDir, unpackToDir, datasetId, UploadDomain.attachment).shiftBox + _ <- cleanUpOnFailure(unpackResult, + datasetId, + dataSourceId, + unpackToDir, + label = s"unpacking attachment to $unpackToDir") + _ <- cleanUpUploaded(uploadId, reason = "Upload complete, data unpacked.", UploadDomain.attachment) + attachmentSizeBytes <- measureDirectorySizeBytes(unpackToDir) ?~> "dataset.upload.measureTotalSize.failed" + finalPath <- moveUnpackedMagOrAttachmentToTarget( + unpackToDir, + layerName, + datasetId, + dataSourceId, + s"$attachmentType/${TextUtils.normalizeStrong(attachment.name)}__${ObjectId.generate}", + UploadDomain.attachment + ) + attachmentAdapted = attachment.copy(path = finalPath) + _ <- remoteWebknossosClient.reportAttachmentUpload( + ReportAttachmentUploadParameters(datasetId, layerName, attachmentType, attachmentAdapted, attachmentSizeBytes)) + } yield () + + private def checkWithinRequestedFileSize(uploadDir: Path, + uploadId: String, + datasetId: ObjectId, + uploadDomain: UploadDomain): Fox[Unit] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) for { - totalFileSizeInBytesOpt <- runningUploadMetadataStore.find(redisKeyForTotalFileSizeInBytes(uploadId)) ?~> "Could not look up reserved total file size" - _ <- totalFileSizeInBytesOpt.map { reservedFileSize => + totalFileSizeInBytesBox <- uploadMetadataStore + .findTotalFileSizeInBytes(uploadId) + .shiftBox ?~> "Could not look up reserved total file size" + _ <- totalFileSizeInBytesBox.map { reservedFileSize => for { actualFileSize <- tryo(FileUtils.sizeOfDirectoryAsBigInteger(uploadDir.toFile).longValue).toFox ?~> "Could not measure actual file size" - _ <- if (actualFileSize > reservedFileSize.toLong) { - cleanUpDatasetExceedingSize(uploadDir, uploadId) + _ <- if (actualFileSize > reservedFileSize) { + cleanUpExceedingSize(uploadId, uploadDomain) Fox.failure( - f"Uploaded dataset $datasetId exceeds the reserved size of $reservedFileSize bytes, got $actualFileSize bytes.") + f"Uploaded $uploadDomain $datasetId exceeds the reserved size of $reservedFileSize bytes, got $actualFileSize bytes.") } else Fox.successful(()) } yield () }.getOrElse(Fox.successful(())) } yield () + } - private def cleanUpDatasetExceedingSize(uploadDir: Path, uploadId: String): Fox[Unit] = + private def cleanUpExceedingSize(uploadId: String, uploadDomain: UploadDomain): Fox[Unit] = for { - datasetId <- getDatasetIdByUploadId(uploadId) - _ <- cleanUpUploadedDataset(uploadDir, uploadId, reason = "Exceeded reserved fileSize") - _ <- remoteWebknossosClient.deleteDataset(datasetId) + datasetId <- getDatasetIdByUploadId(uploadId, uploadDomain) + _ <- cleanUpUploaded(uploadId, reason = "Exceeded reserved fileSize", uploadDomain) + // Datasets need to be cleaned up in postgres as well. The other domains don’t (overwritePending mechanism is used there) + _ <- Fox.runIf(uploadDomain == UploadDomain.dataset)(remoteWebknossosClient.deleteDataset(datasetId)) } yield () private def deleteFilesNotReferencedInDataSource(unpackedDir: Path, dataSource: UsableDataSource): Fox[Unit] = @@ -389,10 +510,40 @@ class UploadService @Inject()(dataSourceService: DataSourceService, }) } yield () - private def moveUnpackedToTarget(unpackedDir: Path, - needsConversion: Boolean, - datasetId: ObjectId, - dataSourceId: DataSourceId): Fox[Option[UsableDataSource]] = + private def moveUnpackedMagOrAttachmentToTarget(unpackedDir: Path, + layerName: String, + datasetId: ObjectId, + dataSourceId: DataSourceId, + dirName: String, + domain: UploadDomain): Fox[UPath] = + if (dataStoreConfig.Datastore.S3Upload.enabled) { + for { + s3UploadBucket <- managedS3Service.s3UploadBucketOpt.toFox + _ = logger.info(s"finishUpload for $domain ($datasetId): Copying data to s3 bucket $s3UploadBucket...") + beforeS3Upload = Instant.now + s3ObjectKey = s"${dataStoreConfig.Datastore.S3Upload.objectKeyPrefix}/${dataSourceId.organizationId}/${dataSourceId.directoryName}/$layerName/$dirName" + _ <- uploadDirectoryToS3(unpackedDir, s3UploadBucket, s3ObjectKey) + _ = Instant.logSince(beforeS3Upload, s"Forwarding of uploaded mag for $datasetId ($dataSourceId) to S3", logger) + endPointHost = new URI(dataStoreConfig.Datastore.S3Upload.credentialName).getHost + finalUploadedS3Path <- UPath.fromString(s"s3://$endPointHost/$s3UploadBucket/$s3ObjectKey").toFox + } yield finalUploadedS3Path + } else { + val finalUploadedLocalPath = + dataBaseDir + .resolve(dataSourceId.organizationId) + .resolve(dataSourceId.directoryName) + .resolve(layerName) + .resolve(dirName) + logger.info(s"finishUpload for $domain ($datasetId): Moving data to final local path $finalUploadedLocalPath...") + for { + _ <- tryo(FileUtils.moveDirectory(unpackedDir.toFile, finalUploadedLocalPath.toFile)).toFox + } yield UPath.fromLocalPath(finalUploadedLocalPath) + } + + private def moveUnpackedDatasetToTarget(unpackedDir: Path, + needsConversion: Boolean, + datasetId: ObjectId, + dataSourceId: DataSourceId): Fox[Option[UsableDataSource]] = if (needsConversion) { logger.info(s"finishUpload for $datasetId: Moving data to input dir for worker conversion...") val forConversionPath = @@ -543,7 +694,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService, private def cleanUpOnFailure[T](result: Box[T], datasetId: ObjectId, dataSourceId: DataSourceId, - needsConversion: Boolean, + unpackToDir: Path, label: String): Fox[Unit] = result match { case Full(_) => @@ -552,7 +703,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService, deleteOnDisk(datasetId, dataSourceId.organizationId, dataSourceId.directoryName, - needsConversion, + Some(unpackToDir), Some("the upload failed")) Fox.failure(s"Unknown error $label") case Failure(msg, e, _) => @@ -560,7 +711,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService, deleteOnDisk(datasetId, dataSourceId.organizationId, dataSourceId.directoryName, - needsConversion, + Some(unpackToDir), Some("the upload failed")) remoteWebknossosClient.deleteDataset(datasetId) for { @@ -568,29 +719,29 @@ class UploadService @Inject()(dataSourceService: DataSourceService, } yield () } - private def checkAllChunksUploaded(uploadId: String): Fox[Unit] = - for { - fileCountStringOpt <- runningUploadMetadataStore.find(redisKeyForFileCount(uploadId)) ?~> "Could not look up reserved file count" - fileCountString <- fileCountStringOpt.toFox ?~> "dataset.upload.noFiles" - fileCount <- tryo(fileCountString.toLong).toFox ?~> "Could not look up reserved file count (toLong)" - fileNames <- runningUploadMetadataStore.findSet(redisKeyForFileNameSet(uploadId)) ?~> "Could not look up reserved file names" - _ <- Fox.fromBool(fileCount == fileNames.size) - list <- Fox.serialCombined(fileNames.toList) { fileName => - val chunkCount = - runningUploadMetadataStore - .find(redisKeyForFileChunkCount(uploadId, fileName)) - .map(s => s.getOrElse("").toLong) - val chunks = runningUploadMetadataStore.findSet(redisKeyForFileChunkSet(uploadId, fileName)) - chunks.flatMap(set => chunkCount.map(_ == set.size)) - } ?~> "Could not look up reserved file sizes" - _ <- Fox.fromBool(list.forall(identity)) + private def checkAllChunksUploaded(uploadId: String, uploadDomain: UploadDomain): Fox[Unit] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) + for { + fileCount <- uploadMetadataStore.findFileCount(uploadId) ?~> "Could not look up reserved file count." + fileNames <- uploadMetadataStore.findFileNames(uploadId) ?~> "Could not look up reserved file names." + _ <- Fox.fromBool(fileCount == fileNames.size) ?~> "Reserved file count does not match file names length." + _ <- Fox.serialCombined(fileNames) { fileName => + for { + chunkCount <- uploadMetadataStore.findFileChunkCount(uploadId, fileName) ?~> "Could not look up file chunk count." + chunkSet <- uploadMetadataStore.findFileChunkSet(uploadId, fileName) ?~> "Could not look up file chunk set." + _ <- Fox.fromBool(chunkCount == chunkSet.size) ?~> s"Chunks missing for uploaded file $fileName: expected $chunkCount, got ${chunkSet.size}." + } yield () + } } yield () + } - private def unpackToDirFor(dataSourceId: DataSourceId): Path = + private def unpackToDirFor(dataSourceId: DataSourceId, domain: UploadDomain, uploadId: String): Path = dataBaseDir .resolve(dataSourceId.organizationId) .resolve(uploadingDir) .resolve(unpackedDir) + .resolve(domain.toString) + .resolve(uploadId) .resolve(dataSourceId.directoryName) private def guessTypeOfUploadedDataSource(dataSourceDir: Path): UploadedDataSourceType.Value = @@ -724,8 +875,12 @@ class UploadService @Inject()(dataSourceService: DataSourceService, private def getPathDepth(path: Path) = path.toString.count(_ == '/') - private def unpackDataset(uploadDir: Path, unpackToDir: Path, datasetId: ObjectId): Fox[Unit] = + private def unpackOrMoveUploaded(uploadDir: Path, + unpackToDir: Path, + datasetId: ObjectId, + uploadDomain: UploadDomain): Fox[Unit] = for { + _ <- PathUtils.ensureDirectoryBox(unpackToDir.getParent).toFox ?~> "dataset.import.fileAccessDenied" shallowFileList <- PathUtils.listFiles(uploadDir, silent = false).toFox excludeFromPrefix = LayerCategory.values.map(_.toString).toList firstFile = shallowFileList.headOption @@ -733,7 +888,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService, _.toString.toLowerCase.endsWith(".zip"))) { for { zipFile <- firstFile.toFox - _ = logger.info(s"finishUpload for $datasetId: Unzipping dataset to $unpackToDir...") + _ = logger.info(s"finishUpload for $datasetId: Unzipping $uploadDomain to $unpackToDir...") _ <- ZipIO .unzipToDirectory( zipFile.toFile, @@ -752,7 +907,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService, _ <- Fox.fromBool(deepFileList.nonEmpty) ?~> "dataset.upload.noFiles" commonPrefixPreliminary = PathUtils.commonPrefix(deepFileList) _ = logger.info( - s"Detected dataset root during upload of $datasetId from ${deepFileList.length} files in $uploadDir with commonPrefixPreliminary=$commonPrefixPreliminary") + s"Detected $uploadDomain root during finishUpload of $datasetId from ${deepFileList.length} files in $uploadDir with commonPrefixPreliminary=$commonPrefixPreliminary") strippedPrefix = PathUtils.cutOffPathAtLastOccurrenceOf(commonPrefixPreliminary, excludeFromPrefix) commonPrefix = PathUtils.removeSingleFileNameFromPrefix(strippedPrefix, deepFileList.map(_.getFileName.toString)) @@ -767,32 +922,17 @@ class UploadService @Inject()(dataSourceService: DataSourceService, tryo(FileUtils.copyDirectory(uploadDir.toFile, backupDir.toFile)) } - private def cleanUpUploadedDataset(uploadDir: Path, uploadId: String, reason: String): Fox[Unit] = + private def cleanUpUploaded(uploadId: String, reason: String, uploadDomain: UploadDomain): Fox[Unit] = { + val uploadMetadataStore = selectUploadMetadataStore(uploadDomain) for { - _ <- Fox.successful(logger.info(s"Cleaning up uploaded dataset. Reason: $reason")) + dataSourceId <- uploadMetadataStore.findDataSourceId(uploadId) + uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId, uploadDomain) + _ <- Fox.successful(logger.info(s"Cleaning up uploaded $uploadDir. Reason: $reason")) _ <- PathUtils.deleteDirectoryRecursively(uploadDir).toFox - _ <- removeFromRedis(uploadId) - } yield () - - private def removeFromRedis(uploadId: String): Fox[Unit] = - for { - _ <- runningUploadMetadataStore.remove(redisKeyForFileCount(uploadId)) - fileNames <- runningUploadMetadataStore.findSet(redisKeyForFileNameSet(uploadId)) - _ <- Fox.serialCombined(fileNames.toList) { fileName => - for { - _ <- runningUploadMetadataStore.remove(redisKeyForFileChunkCount(uploadId, fileName)) - _ <- runningUploadMetadataStore.remove(redisKeyForFileChunkSet(uploadId, fileName)) - } yield () - } - _ <- runningUploadMetadataStore.remove(redisKeyForFileNameSet(uploadId)) - _ <- runningUploadMetadataStore.remove(redisKeyForTotalFileSizeInBytes(uploadId)) - dataSourceId <- getDataSourceIdByUploadId(uploadId) - _ <- runningUploadMetadataStore.remove(redisKeyForDataSourceId(uploadId)) - _ <- runningUploadMetadataStore.remove(redisKeyForDatasetId(uploadId)) - _ <- runningUploadMetadataStore.remove(redisKeyForLinkedLayerIdentifier(uploadId)) - _ <- runningUploadMetadataStore.remove(redisKeyForUploadId(dataSourceId)) - _ <- runningUploadMetadataStore.remove(redisKeyForFilePaths(uploadId)) + uploadMetadataStore = selectUploadMetadataStore(uploadDomain) + _ <- uploadMetadataStore.cleanUp(uploadId) } yield () + } private def cleanUpOrphanUploads(): Fox[Unit] = for { @@ -800,21 +940,28 @@ class UploadService @Inject()(dataSourceService: DataSourceService, _ <- Fox.serialCombined(organizationDirs)(cleanUpOrphanUploadsForOrga) } yield () - private def cleanUpOrphanUploadsForOrga(organizationDir: Path): Fox[Unit] = { - val orgaUploadingDir: Path = organizationDir.resolve(uploadingDir) + private def cleanUpOrphanUploadsForOrga(organizationDir: Path): Fox[Unit] = + for { + _ <- cleanUpOrphanUploadsForOrgaAndDomain(organizationDir, UploadDomain.dataset) + _ <- cleanUpOrphanUploadsForOrgaAndDomain(organizationDir, UploadDomain.mag) + _ <- cleanUpOrphanUploadsForOrgaAndDomain(organizationDir, UploadDomain.attachment) + } yield () + + private def cleanUpOrphanUploadsForOrgaAndDomain(organizationDir: Path, uploadDomain: UploadDomain): Fox[Unit] = { + val orgaUploadingDir: Path = organizationDir.resolve(uploadingDir).resolve(uploadDomain.toString) if (!Files.exists(orgaUploadingDir)) Fox.successful(()) else { for { uploadDirs <- PathUtils.listDirectories(orgaUploadingDir, silent = false).toFox _ <- Fox.serialCombined(uploadDirs) { uploadDir => - isKnownUpload(uploadDir.getFileName.toString).map { + isKnownUploadOfAnyDomain(uploadDir.getFileName.toString).map { case false => val deleteResult = PathUtils.deleteDirectoryRecursively(uploadDir) if (deleteResult.isDefined) { - logger.info(f"Deleted orphan dataset upload at $uploadDir") + logger.info(f"Deleted orphan $uploadDomain upload at $uploadDir") } else { - logger.warn(f"Failed to delete orphan dataset upload at $uploadDir") + logger.warn(f"Failed to delete orphan $uploadDomain upload at $uploadDir") } case true => () } @@ -823,12 +970,12 @@ class UploadService @Inject()(dataSourceService: DataSourceService, } } - private def getObjectFromRedis[T: Reads](key: String): Fox[T] = + private def isKnownUploadOfAnyDomain(uploadId: String): Fox[Boolean] = for { - objectStringOption <- runningUploadMetadataStore.find(key) - objectString <- objectStringOption.toFox - parsed <- JsonHelper.parseAs[T](objectString).toFox - } yield parsed + fromDataset <- datasetUploadMetadataStore.isKnownUpload(uploadId) + fromMag <- magUploadMetadataStore.isKnownUpload(uploadId) + fromAttachment <- attachmentUploadMetadataStore.isKnownUpload(uploadId) + } yield fromDataset || fromMag || fromAttachment } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RedisTemporaryStore.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RedisTemporaryStore.scala index 5d6a6e3f61e..c3b11d70af2 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RedisTemporaryStore.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RedisTemporaryStore.scala @@ -1,130 +1,108 @@ package com.scalableminds.webknossos.datastore.storage import com.redis._ -import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} import com.typesafe.scalalogging.LazyLogging +import play.api.libs.json.{Json, Reads, Writes} import scala.concurrent.ExecutionContext import scala.concurrent.duration.FiniteDuration -trait RedisTemporaryStore extends LazyLogging { +trait RedisTemporaryStore extends LazyLogging with FoxImplicits { implicit def ec: ExecutionContext protected def address: String protected def port: Int lazy val authority: String = f"$address:$port" - private lazy val r = new RedisClient(address, port) + private lazy val r = new RedisClientPool(address, port) - def find(id: String): Fox[Option[String]] = - withExceptionHandler { - r.get(id) - } + def find(id: String): Fox[String] = + withExceptionHandler(_.get(id)).map(_.toFox).flatten - def findLong(id: String): Fox[Option[Long]] = - withExceptionHandler { - r.get(id).map(s => s.toLong) - } + def findLong(id: String): Fox[Long] = + withExceptionHandler(_.get(id).map(s => s.toLong)).map(_.toFox).flatten def removeAllConditional(pattern: String): Fox[Unit] = - withExceptionHandler { - val keysOpt: Option[List[Option[String]]] = r.keys(pattern) + withExceptionHandler { client => + val keysOpt: Option[List[Option[String]]] = client.keys(pattern) keysOpt.foreach { keys: Seq[Option[String]] => keys.flatMap { key: Option[String] => - key.flatMap(r.del(_)) + key.flatMap(client.del(_)) } } } def findAllConditional(pattern: String): Fox[Seq[String]] = - withExceptionHandler { - val keysOpt: Option[List[Option[String]]] = r.keys(pattern) + withExceptionHandler { client => + val keysOpt: Option[List[Option[String]]] = client.keys(pattern) keysOpt.map { keys: Seq[Option[String]] => keys.flatMap { key: Option[String] => - key.flatMap(r.get(_)) + key.flatMap(client.get(_)) } }.getOrElse(Seq()) } def keys(pattern: String): Fox[List[String]] = - withExceptionHandler { - r.keys(pattern).map(_.flatten).getOrElse(List()) - } + withExceptionHandler(_.keys(pattern).map(_.flatten).getOrElse(List())) def insertKey(id: String, expirationOpt: Option[FiniteDuration] = None): Fox[Unit] = insert(id, "", expirationOpt) def insert(id: String, value: String, expirationOpt: Option[FiniteDuration] = None): Fox[Unit] = - withExceptionHandler { - expirationOpt - .map( - expiration => r.setex(id, expiration.toSeconds, value) - ) - .getOrElse( - r.set(id, value) - ) + withExceptionHandler { client => + expirationOpt.map(expiration => client.setex(id, expiration.toSeconds, value)).getOrElse(client.set(id, value)) } def insertLong(id: String, value: Long, expirationOpt: Option[FiniteDuration] = None): Fox[Unit] = - withExceptionHandler { - expirationOpt - .map( - expiration => r.setex(id, expiration.toSeconds, value) - ) - .getOrElse( - r.set(id, value) - ) + withExceptionHandler { client => + expirationOpt.map(expiration => client.setex(id, expiration.toSeconds, value)).getOrElse(client.set(id, value)) } def contains(id: String): Fox[Boolean] = - withExceptionHandler { - r.exists(id) - } + withExceptionHandler(_.exists(id)) def remove(id: String): Fox[Unit] = - withExceptionHandler { - r.del(id) - } + withExceptionHandler(_.del(id)) - def checkHealth(implicit ec: ExecutionContext): Fox[Unit] = - try { - val reply = r.ping + def checkHealth: Fox[Unit] = + withExceptionHandler { client => + val reply = client.ping if (!reply.contains("PONG")) throw new Exception(reply.getOrElse("No Reply")) - Fox.successful(()) - } catch { - case e: Exception => - logger.error(s"Redis health check failed at $address:$port (reply: ${e.getMessage})") - Fox.failure(s"Redis health check failed") - } - - def withExceptionHandler[B](f: => B): Fox[B] = - try { - r.synchronized { - Fox.successful(f) - } - } catch { - case e: Exception => - val msg = "Redis access exception: " + e.getMessage - logger.error(msg) - Fox.failure(msg) + () } def insertIntoSet(id: String, value: String): Fox[Boolean] = - withExceptionHandler { - r.sadd(id, value).getOrElse(0L) > 0 - } + withExceptionHandler(_.sadd(id, value).getOrElse(0L) > 0) def isContainedInSet(id: String, value: String): Fox[Boolean] = - withExceptionHandler { - r.sismember(id, value) - } + withExceptionHandler(_.sismember(id, value)) def removeFromSet(id: String, value: String): Fox[Boolean] = - withExceptionHandler { - r.srem(id, value).getOrElse(0L) > 0 - } + withExceptionHandler(_.srem(id, value).getOrElse(0L) > 0) def findSet(id: String): Fox[Set[String]] = - withExceptionHandler { - r.smembers(id).map(_.flatten).getOrElse(Set.empty) + withExceptionHandler(_.smembers(id).map(_.flatten).getOrElse(Set.empty)) + + def findParsed[T: Reads](key: String)(implicit ec: ExecutionContext): Fox[T] = + for { + objectString <- find(key) + parsed <- JsonHelper.parseAs[T](objectString).toFox + } yield parsed + + def insertSerialized[T: Writes](key: String, value: T): Fox[Unit] = { + val serialized = Json.stringify(Json.toJson(value)) + insert(key, serialized) + } + + private def withExceptionHandler[B](f: RedisClient => B): Fox[B] = + try { + r.withClient { client => + Fox.successful(f(client)) + } + } catch { + case e: Exception => + val msg = "Redis access exception: " + e.getMessage + logger.error(msg) + Fox.failure(msg) } } diff --git a/webknossos-datastore/conf/datastore.latest.routes b/webknossos-datastore/conf/datastore.latest.routes index 5338337b832..19c548b7e5d 100644 --- a/webknossos-datastore/conf/datastore.latest.routes +++ b/webknossos-datastore/conf/datastore.latest.routes @@ -1,136 +1,140 @@ # Defines latest version of datastore routes (Higher priority routes first) # Health endpoint -GET /health @com.scalableminds.webknossos.datastore.controllers.Application.health +GET /health @com.scalableminds.webknossos.datastore.controllers.Application.health # Read image data -POST /datasets/:datasetId/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossos(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/readData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboidPost(datasetId: ObjectId, dataLayerName: String) -GET /datasets/:datasetId/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboid(datasetId: ObjectId, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, depth: Int, mag: String, halfByte: Boolean ?= false, mappingName: Option[String]) -GET /datasets/:datasetId/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.thumbnailJpeg(datasetId: ObjectId, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, mag: String, mappingName: Option[String], intensityMin: Option[Double], intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]) -GET /datasets/:datasetId/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(datasetId: ObjectId, dataLayerName: String) -GET /datasets/:datasetId/layers/:dataLayerName/histogram @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.histogram(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossos(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/readData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboidPost(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboid(datasetId: ObjectId, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, depth: Int, mag: String, halfByte: Boolean ?= false, mappingName: Option[String]) +GET /datasets/:datasetId/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.thumbnailJpeg(datasetId: ObjectId, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, mag: String, mappingName: Option[String], intensityMin: Option[Double], intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]) +GET /datasets/:datasetId/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/histogram @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.histogram(datasetId: ObjectId, dataLayerName: String) # Read mag and attachment data via proxy -GET /datasets/:datasetId/proxy/layers/:dataLayerName/mags/:mag/*path @com.scalableminds.webknossos.datastore.controllers.DataProxyController.proxyMag(datasetId: ObjectId, dataLayerName: String, mag: String, path: String) -GET /datasets/:datasetId/proxy/layers/:dataLayerName/attachments/:attachmentType/:attachmentName/*path @com.scalableminds.webknossos.datastore.controllers.DataProxyController.proxyAttachment(datasetId: ObjectId, dataLayerName: String, attachmentType: String, attachmentName: String, path: String) -GET /datasets/:datasetId/proxy/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.DataProxyController.proxyDatasource(datasetId: ObjectId) +GET /datasets/:datasetId/proxy/layers/:dataLayerName/mags/:mag/*path @com.scalableminds.webknossos.datastore.controllers.DataProxyController.proxyMag(datasetId: ObjectId, dataLayerName: String, mag: String, path: String) +GET /datasets/:datasetId/proxy/layers/:dataLayerName/attachments/:attachmentType/:attachmentName/*path @com.scalableminds.webknossos.datastore.controllers.DataProxyController.proxyAttachment(datasetId: ObjectId, dataLayerName: String, attachmentType: String, attachmentName: String, path: String) +GET /datasets/:datasetId/proxy/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.DataProxyController.proxyDatasource(datasetId: ObjectId) # Knossos compatible routes -GET /datasets/:datasetId/layers/:dataLayerName/mag:mag/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(datasetId: ObjectId, dataLayerName: String, mag: Int, x: Int, y: Int, z: Int, cubeSize: Int) +GET /datasets/:datasetId/layers/:dataLayerName/mag:mag/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(datasetId: ObjectId, dataLayerName: String, mag: Int, x: Int, y: Int, z: Int, cubeSize: Int) # Zarr2 compatible routes -GET /zarr/:datasetId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 2) -GET /zarr/:datasetId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 2) -GET /zarr/:datasetId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(datasetId: ObjectId, dataLayerName="") -GET /zarr/:datasetId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(datasetId: ObjectId, zarrVersion: Int = 2) -GET /zarr/:datasetId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 2) -GET /zarr/:datasetId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 2) -GET /zarr/:datasetId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(datasetId: ObjectId, dataLayerName: String) -GET /zarr/:datasetId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(datasetId: ObjectId, dataLayerName: String) -GET /zarr/:datasetId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /zarr/:datasetId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /zarr/:datasetId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(datasetId: ObjectId, dataLayerName: String, mag: String) -GET /zarr/:datasetId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(datasetId: ObjectId, dataLayerName: String, mag: String, coordinates: String) - -GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(accessTokenOrId: String, dataLayerName="") -GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) +GET /zarr/:datasetId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 2) +GET /zarr/:datasetId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 2) +GET /zarr/:datasetId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(datasetId: ObjectId, dataLayerName="") +GET /zarr/:datasetId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(datasetId: ObjectId, zarrVersion: Int = 2) +GET /zarr/:datasetId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:datasetId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:datasetId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(datasetId: ObjectId, dataLayerName: String) +GET /zarr/:datasetId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(datasetId: ObjectId, dataLayerName: String) +GET /zarr/:datasetId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:datasetId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:datasetId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(datasetId: ObjectId, dataLayerName: String, mag: String) +GET /zarr/:datasetId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(datasetId: ObjectId, dataLayerName: String, mag: String, coordinates: String) + +GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(accessTokenOrId: String, dataLayerName="") +GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Zarr3 compatible routes -GET /zarr3_experimental/:datasetId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(datasetId: ObjectId, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(datasetId: ObjectId, dataLayerName: String) -GET /zarr3_experimental/:datasetId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:datasetId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(datasetId: ObjectId, dataLayerName: String, mag: String) -GET /zarr3_experimental/:datasetId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(datasetId: ObjectId, dataLayerName: String, mag: String, coordinates: String) - -GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) +GET /zarr3_experimental/:datasetId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceDirectoryContents(datasetId: ObjectId, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(datasetId: ObjectId, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerDirectoryContents(datasetId: ObjectId, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(datasetId: ObjectId, dataLayerName: String) +GET /zarr3_experimental/:datasetId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagDirectoryContents(datasetId: ObjectId, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:datasetId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(datasetId: ObjectId, dataLayerName: String, mag: String) +GET /zarr3_experimental/:datasetId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(datasetId: ObjectId, dataLayerName: String, mag: String, coordinates: String) + +GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceDirectoryContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagDirectoryContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Segmentation mappings -GET /datasets/:datasetId/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(datasetId: ObjectId, dataLayerName: String, mappingName: String) -GET /datasets/:datasetId/layers/:dataLayerName/mappings @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMappings(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(datasetId: ObjectId, dataLayerName: String, mappingName: String) +GET /datasets/:datasetId/layers/:dataLayerName/mappings @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMappings(datasetId: ObjectId, dataLayerName: String) # Agglomerate files -GET /datasets/:datasetId/layers/:dataLayerName/agglomerates @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listAgglomerates(datasetId: ObjectId, dataLayerName: String) -GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/tree/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateTree(datasetId: ObjectId, dataLayerName: String, mappingName: String, agglomerateId: Long) +GET /datasets/:datasetId/layers/:dataLayerName/agglomerates @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listAgglomerates(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/tree/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateTree(datasetId: ObjectId, dataLayerName: String, mappingName: String, agglomerateId: Long) # Alias route kept for one release cycle for backwards compatibility for open browser sessions. Remove after 2026-06 -GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/skeleton/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateTree(datasetId: ObjectId, dataLayerName: String, mappingName: String, agglomerateId: Long) -GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(datasetId: ObjectId, dataLayerName: String, mappingName: String, agglomerateId: Long) -GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(datasetId: ObjectId, dataLayerName: String, mappingName: String) -POST /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(datasetId: ObjectId, dataLayerName: String, mappingName: String) -GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/positionForSegment @com.scalableminds.webknossos.datastore.controllers.DataSourceController.positionForSegmentViaAgglomerateFile(datasetId: ObjectId, dataLayerName: String, mappingName: String, segmentId: Long) +GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/skeleton/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateTree(datasetId: ObjectId, dataLayerName: String, mappingName: String, agglomerateId: Long) +GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(datasetId: ObjectId, dataLayerName: String, mappingName: String, agglomerateId: Long) +GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(datasetId: ObjectId, dataLayerName: String, mappingName: String) +POST /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(datasetId: ObjectId, dataLayerName: String, mappingName: String) +GET /datasets/:datasetId/layers/:dataLayerName/agglomerates/:mappingName/positionForSegment @com.scalableminds.webknossos.datastore.controllers.DataSourceController.positionForSegmentViaAgglomerateFile(datasetId: ObjectId, dataLayerName: String, mappingName: String, segmentId: Long) # Mesh files -GET /datasets/:datasetId/layers/:dataLayerName/meshes @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshFiles(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/meshes/chunks @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshChunksForSegment(datasetId: ObjectId, dataLayerName: String, targetMappingName: Option[String], editableMappingTracingId: Option[String]) -POST /datasets/:datasetId/layers/:dataLayerName/meshes/chunks/data @com.scalableminds.webknossos.datastore.controllers.DSMeshController.readMeshChunk(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/meshes/fullMesh.stl @com.scalableminds.webknossos.datastore.controllers.DSMeshController.loadFullMeshStl(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/meshes @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshFiles(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/meshes/chunks @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshChunksForSegment(datasetId: ObjectId, dataLayerName: String, targetMappingName: Option[String], editableMappingTracingId: Option[String]) +POST /datasets/:datasetId/layers/:dataLayerName/meshes/chunks/data @com.scalableminds.webknossos.datastore.controllers.DSMeshController.readMeshChunk(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/meshes/fullMesh.stl @com.scalableminds.webknossos.datastore.controllers.DSMeshController.loadFullMeshStl(datasetId: ObjectId, dataLayerName: String) # Connectome files -GET /datasets/:datasetId/layers/:dataLayerName/connectomes @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listConnectomeFiles(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses/positions @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsePositions(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses/types @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapseTypes(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses/:direction @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapticPartnerForSynapses(datasetId: ObjectId, dataLayerName: String, direction: String) -POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsesForAgglomerates(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/connectomes @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listConnectomeFiles(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses/positions @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsePositions(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses/types @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapseTypes(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses/:direction @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapticPartnerForSynapses(datasetId: ObjectId, dataLayerName: String, direction: String) +POST /datasets/:datasetId/layers/:dataLayerName/connectomes/synapses @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsesForAgglomerates(datasetId: ObjectId, dataLayerName: String) # Ad-Hoc Meshing -POST /datasets/:datasetId/layers/:dataLayerName/adHocMesh @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestAdHocMesh(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/adHocMesh @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestAdHocMesh(datasetId: ObjectId, dataLayerName: String) # Segment-Index files -GET /datasets/:datasetId/layers/:dataLayerName/hasSegmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.checkSegmentIndexFile(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/segmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.querySegmentIndex(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/segmentIndex/:segmentId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentIndex(datasetId: ObjectId, dataLayerName: String, segmentId: String) -POST /datasets/:datasetId/layers/:dataLayerName/segmentStatistics/volume @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentVolume(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/segmentStatistics/boundingBox @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentBoundingBox(datasetId: ObjectId, dataLayerName: String) -POST /datasets/:datasetId/layers/:dataLayerName/segmentStatistics/surfaceArea @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentSurfaceArea(datasetId: ObjectId, dataLayerName: String) +GET /datasets/:datasetId/layers/:dataLayerName/hasSegmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.checkSegmentIndexFile(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/segmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.querySegmentIndex(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/segmentIndex/:segmentId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentIndex(datasetId: ObjectId, dataLayerName: String, segmentId: String) +POST /datasets/:datasetId/layers/:dataLayerName/segmentStatistics/volume @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentVolume(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/segmentStatistics/boundingBox @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentBoundingBox(datasetId: ObjectId, dataLayerName: String) +POST /datasets/:datasetId/layers/:dataLayerName/segmentStatistics/surfaceArea @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentSurfaceArea(datasetId: ObjectId, dataLayerName: String) + +# Uploads: Datasets, mags, attachments +GET /datasets/upload/:uploadDomain @com.scalableminds.webknossos.datastore.controllers.UploadController.testChunk(resumableChunkNumber: Int, resumableIdentifier: String, uploadDomain: String) +POST /datasets/upload/:uploadDomain @com.scalableminds.webknossos.datastore.controllers.UploadController.uploadChunk(uploadDomain: String) +POST /datasets/upload/dataset/reserveUpload @com.scalableminds.webknossos.datastore.controllers.UploadController.reserveDatasetUpload() +POST /datasets/upload/mag/reserveUpload @com.scalableminds.webknossos.datastore.controllers.UploadController.reserveMagUpload() +POST /datasets/upload/attachment/reserveUpload @com.scalableminds.webknossos.datastore.controllers.UploadController.reserveAttachmentUpload() +GET /datasets/upload/:uploadDomain/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.UploadController.getUnfinishedUploads(organizationName: String, uploadDomain: String) +POST /datasets/upload/:uploadDomain/finishUpload @com.scalableminds.webknossos.datastore.controllers.UploadController.finishUpload(uploadDomain: String, uploadId: String) +POST /datasets/upload/:uploadDomain/cancelUpload @com.scalableminds.webknossos.datastore.controllers.UploadController.cancelUpload(uploadDomain: String, uploadId: String) # DataSource management -GET /datasets @com.scalableminds.webknossos.datastore.controllers.DataSourceController.testChunk(resumableChunkNumber: Int, resumableIdentifier: String) -POST /datasets @com.scalableminds.webknossos.datastore.controllers.DataSourceController.uploadChunk() -GET /datasets/getUnfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getUnfinishedUploads(organizationName: String) -GET /datasets/baseDirAbsolute @com.scalableminds.webknossos.datastore.controllers.DataSourceController.baseDirAbsolute -POST /datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reserveUpload() -POST /datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.finishUpload() -POST /datasets/cancelUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.cancelUpload() -POST /datasets/measureUsedStorage/:organizationId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.measureUsedStorage(organizationId: String) -PUT /datasets/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.updateOnDisk(datasetId: ObjectId) -DELETE /datasets/:datasetId/deleteOnDisk @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deleteOnDisk(datasetId: ObjectId) -DELETE /datasets/deletePaths @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deletePaths() -POST /datasets/exploreRemote @com.scalableminds.webknossos.datastore.controllers.DataSourceController.exploreRemoteDataset() -POST /datasets/validatePaths @com.scalableminds.webknossos.datastore.controllers.DataSourceController.validatePaths() -DELETE /datasets/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.invalidateCache(datasetId: ObjectId) +GET /datasets/baseDirAbsolute @com.scalableminds.webknossos.datastore.controllers.DataSourceController.baseDirAbsolute +POST /datasets/measureUsedStorage/:organizationId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.measureUsedStorage(organizationId: String) +PUT /datasets/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.updateOnDisk(datasetId: ObjectId) +DELETE /datasets/:datasetId/deleteOnDisk @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deleteOnDisk(datasetId: ObjectId) +DELETE /datasets/deletePaths @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deletePaths() +POST /datasets/exploreRemote @com.scalableminds.webknossos.datastore.controllers.DataSourceController.exploreRemoteDataset() +POST /datasets/validatePaths @com.scalableminds.webknossos.datastore.controllers.DataSourceController.validatePaths() +DELETE /datasets/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.invalidateCache(datasetId: ObjectId) # Actions -POST /triggers/checkInboxBlocking @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheckBlocking(organizationId: Option[String]) -POST /triggers/createOrganizationDirectory @com.scalableminds.webknossos.datastore.controllers.DataSourceController.createOrganizationDirectory(organizationId: String) -POST /triggers/reload/:organizationId/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reload(organizationId: String, datasetId: ObjectId, layerName: Option[String]) -POST /triggers/scanRealPathsForVirtual @com.scalableminds.webknossos.datastore.controllers.DataSourceController.scanRealPathsForVirtual() +POST /triggers/checkInboxBlocking @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheckBlocking(organizationId: Option[String]) +POST /triggers/createOrganizationDirectory @com.scalableminds.webknossos.datastore.controllers.DataSourceController.createOrganizationDirectory(organizationId: String) +POST /triggers/reload/:organizationId/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reload(organizationId: String, datasetId: ObjectId, layerName: Option[String]) +POST /triggers/scanRealPathsForVirtual @com.scalableminds.webknossos.datastore.controllers.DataSourceController.scanRealPathsForVirtual() # Exports -GET /exports/:jobId/download @com.scalableminds.webknossos.datastore.controllers.ExportsController.download(jobId: ObjectId) +GET /exports/:jobId/download @com.scalableminds.webknossos.datastore.controllers.ExportsController.download(jobId: ObjectId) # AI Models -POST /aiModels/effectiveVoxelSize @com.scalableminds.webknossos.datastore.controllers.DSAiModelController.effectiveVoxelSize +POST /aiModels/effectiveVoxelSize @com.scalableminds.webknossos.datastore.controllers.DSAiModelController.effectiveVoxelSize diff --git a/webknossos-datastore/conf/datastore.versioned.routes b/webknossos-datastore/conf/datastore.versioned.routes index ca58be8906d..bca20dfd0af 100644 --- a/webknossos-datastore/conf/datastore.versioned.routes +++ b/webknossos-datastore/conf/datastore.versioned.routes @@ -1,14 +1,44 @@ # Note: keep this in sync with the reported version numbers in the com.scalableminds.util.mvc.ApiVersioning trait +# Version log in webknossos.versioned.routes + +-> /v14/ datastore.latest.Routes + +# Dataset upload +GET /v13/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v13/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +POST /v13/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveDatasetUploadV13() +GET /v13/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v13/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() -> /v13/ datastore.latest.Routes + +# Dataset upload +GET /v12/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v12/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +POST /v12/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveDatasetUploadV13() +GET /v12/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v12/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() + -> /v12/ datastore.latest.Routes +# Dataset upload +GET /v11/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v11/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v11/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v11/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v11/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v11/ datastore.latest.Routes POST /v10/datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveManualUploadV10() +# Dataset upload +GET /v10/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v10/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v10/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v10/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v10/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v10/ datastore.latest.Routes @@ -56,6 +86,12 @@ GET /v9/zarr3_experimental/:organizationId/:datasetDirectoryName/:data POST /v9/datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveManualUploadV10() +# Dataset upload +GET /v9/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v9/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v9/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v9/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v9/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v9/ datastore.latest.Routes @@ -104,6 +140,12 @@ GET /v8/zarr3_experimental/:organizationId/:datasetDirectoryName/:data POST /v8/datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveManualUploadV10() +# Dataset upload +GET /v8/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v8/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v8/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v8/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v8/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v8/ datastore.latest.Routes @@ -151,6 +193,12 @@ GET /v7/zarr3_experimental/:organizationId/:datasetDirectoryName/:data POST /v7/datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveManualUploadV10() +# Dataset upload +GET /v7/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v7/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v7/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v7/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v7/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v7/ datastore.latest.Routes @@ -198,6 +246,12 @@ GET /v6/zarr3_experimental/:organizationId/:datasetDirectoryName/:data POST /v6/datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveManualUploadV10() +# Dataset upload +GET /v6/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v6/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v6/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v6/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v6/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v6/ datastore.latest.Routes @@ -243,7 +297,17 @@ GET /v5/zarr3_experimental/:organizationId/:datasetDirectoryName/:data POST /v5/datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveManualUploadV10() +# Dataset upload +GET /v5/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /v5/datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() +GET /v5/datasets/unfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.getUnfinishedUploadsV13(organizationName: String) +POST /v5/datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.finishUploadV13() +# Dataset upload (v11 and older is separate!) POST /v5/datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.reserveUploadV11() -> /v5/ datastore.latest.Routes -> / datastore.latest.Routes + +# Dataset upload without version number: Old libs clients perform the chunk requests without version number. +GET /datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.testChunkV13(resumableChunkNumber: Int, resumableIdentifier: String) +POST /datasets @com.scalableminds.webknossos.datastore.controllers.DSLegacyApiController.uploadChunkV13() diff --git a/webknossos-tracingstore/conf/tracingstore.versioned.routes b/webknossos-tracingstore/conf/tracingstore.versioned.routes index ffba2a9b1fb..78fa34e430e 100644 --- a/webknossos-tracingstore/conf/tracingstore.versioned.routes +++ b/webknossos-tracingstore/conf/tracingstore.versioned.routes @@ -1,5 +1,7 @@ # Note: keep this in sync with the reported version numbers in the com.scalableminds.util.mvc.ApiVersioning trait +# Version log in webknossos.versioned.routes +-> /v14/ tracingstore.latest.Routes -> /v13/ tracingstore.latest.Routes -> /v12/ tracingstore.latest.Routes -> /v11/ tracingstore.latest.Routes