diff --git a/delta/plugins/storage/src/main/resources/contexts/files.json b/delta/plugins/storage/src/main/resources/contexts/files.json index 8284bc991e..5dc8d50879 100644 --- a/delta/plugins/storage/src/main/resources/contexts/files.json +++ b/delta/plugins/storage/src/main/resources/contexts/files.json @@ -25,7 +25,8 @@ "_storage": { "@id": "https://bluebrain.github.io/nexus/vocabulary/storage", "@type": "@id" - } + }, + "_sourceFile": "https://bluebrain.github.io/nexus/vocabulary/sourceFile" }, "@id": "https://bluebrain.github.io/nexus/contexts/files.json" } \ No newline at end of file diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index bda65e8600..0394c0f3eb 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -97,10 +97,10 @@ final class Files( for { pc <- fetchContext.onCreate(projectRef) iri <- generateId(pc) - _ <- test(CreateFile(iri, projectRef, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) + _ <- test(CreateFile(iri, projectRef, testStorageRef, testStorageType, testAttributes, caller.subject, tag, None)) (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, projectRef, pc) attributes <- extractFileAttributes(iri, entity, storage) - res <- eval(CreateFile(iri, projectRef, storageRef, storage.tpe, attributes, caller.subject, tag)) + res <- eval(CreateFile(iri, projectRef, storageRef, storage.tpe, attributes, caller.subject, tag, None)) } yield res }.span("createFile") @@ -126,10 +126,10 @@ final class Files( )(implicit caller: Caller): IO[FileResource] = { for { (iri, pc) <- id.expandIri(fetchContext.onCreate) - _ <- test(CreateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) + _ <- test(CreateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, caller.subject, tag, None)) (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) attributes <- extractFileAttributes(iri, entity, storage) - res <- eval(CreateFile(iri, id.project, storageRef, storage.tpe, attributes, caller.subject, tag)) + res <- eval(CreateFile(iri, id.project, storageRef, storage.tpe, attributes, caller.subject, tag, None)) } yield res }.span("createFile") @@ -405,12 +405,12 @@ final class Files( tag: Option[UserTag] )(implicit caller: Caller): IO[FileResource] = for { - _ <- test(CreateFile(iri, ref, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) + _ <- test(CreateFile(iri, ref, testStorageRef, testStorageType, testAttributes, caller.subject, tag, None)) (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, ref, pc) resolvedFilename <- IO.fromOption(filename.orElse(path.lastSegment))(InvalidFileLink(iri)) description <- FileDescription(resolvedFilename, mediaType) attributes <- linkFile(storage, path, description, iri) - res <- eval(CreateFile(iri, ref, storageRef, storage.tpe, attributes, caller.subject, tag)) + res <- eval(CreateFile(iri, ref, storageRef, storage.tpe, attributes, caller.subject, tag, None)) } yield res private def linkFile(storage: Storage, path: Uri.Path, desc: FileDescription, fileId: Iri): IO[FileAttributes] = @@ -597,7 +597,7 @@ object Files { ): Option[FileState] = { // format: off def created(e: FileCreated): Option[FileState] = Option.when(state.isEmpty) { - FileState(e.id, e.project, e.storage, e.storageType, e.attributes, Tags(e.tag, e.rev), e.rev, deprecated = false, e.instant, e.subject, e.instant, e.subject) + FileState(e.id, e.project, e.storage, e.storageType, e.attributes, e.sourceFile, Tags(e.tag, e.rev), e.rev, deprecated = false, e.instant, e.subject, e.instant, e.subject) } def updated(e: FileUpdated): Option[FileState] = state.map { s => @@ -641,7 +641,7 @@ object Files { def create(c: CreateFile) = state match { case None => clock.realTimeInstant.map( - FileCreated(c.id, c.project, c.storage, c.storageType, c.attributes, 1, _, c.subject, c.tag) + FileCreated(c.id, c.project, c.storage, c.storageType, c.attributes, 1, _, c.subject, c.tag, c.sourceFile) ) case Some(_) => IO.raiseError(ResourceAlreadyExists(c.id, c.project)) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index 37c94fd763..2004f86fa9 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -1,13 +1,14 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch +import akka.http.scaladsl.model.Uri import cats.data.NonEmptyList import cats.effect.IO import cats.implicits.toFunctorOps import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FetchFileResource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection @@ -20,12 +21,13 @@ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef import shapeless.syntax.typeable.typeableOps trait BatchCopy { def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit c: Caller - ): IO[NonEmptyList[FileAttributes]] + ): IO[NonEmptyList[FileAttributes]] // return source file Iri here along with new file attributes } object BatchCopy { @@ -49,14 +51,14 @@ object BatchCopy { private def copyToRemoteStorage(source: CopyFileSource, dest: RemoteDiskStorage)(implicit c: Caller) = for { - remoteCopyDetails <- source.files.traverse(fetchRemoteCopyDetails(dest, _)) + remoteCopyDetails <- source.files.traverse(r => fetchRemoteCopyDetails(dest, FileId(r, source.project))) _ <- validateFilesForStorage(dest, remoteCopyDetails.map(_.sourceAttributes.bytes)) attributes <- remoteDiskCopy.copyFiles(dest, remoteCopyDetails) } yield attributes private def copyToDiskStorage(source: CopyFileSource, dest: DiskStorage)(implicit c: Caller) = for { - diskCopyDetails <- source.files.traverse(fetchDiskCopyDetails(dest, _)) + diskCopyDetails <- source.files.traverse(r => fetchDiskCopyDetails(dest, FileId(r, source.project))) _ <- validateFilesForStorage(dest, diskCopyDetails.map(_.sourceAttributes.bytes)) attributes <- diskCopy.copyFiles(dest, diskCopyDetails) } yield attributes @@ -112,14 +114,23 @@ object BatchCopy { private def notEnoughSpace(totalSize: Long, spaceLeft: Long, destStorage: Iri) = IO.raiseError(TotalCopySizeTooLarge(totalSize, spaceLeft, destStorage)) - private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = { + private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller): IO[(File, Storage)] = for { file <- fetchFile.fetch(id) sourceStorage <- fetchStorage.fetch(file.value.storage, id.project) perm = sourceStorage.value.storageValue.readPermission _ <- aclCheck.authorizeForOr(id.project, perm)(AuthorizationFailed(id.project, perm)) } yield (file.value, sourceStorage.value) - } } + def extractSourceFileIri(file: FileResource, fileId: FileId): Iri = { + val lookupQuery = fileId.id match { + case _: IdSegmentRef.Latest => + // need to refer to specific revision, since the file may be updated after the copy + Uri.Query("rev" -> file.rev.toString) + case IdSegmentRef.Revision(_, rev) => Uri.Query("rev" -> rev.toString) + case IdSegmentRef.Tag(_, tag) => Uri.Query("tag" -> tag.value) + } + file.id.queryParams(lookupQuery) + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index 2d3d8f0fe5..ebf667418c 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -49,7 +49,8 @@ object BatchFiles { destFilesAttributes <- batchCopy.copyFiles(source, destStorage).adaptError { case e: CopyFileRejection => CopyRejection(source.project, dest.project, destStorage.id, e) } - fileResources <- createFileResources(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) + destAttrAndSourceIris = destFilesAttributes.zip(source.files) + fileResources <- createFileResources(pc, dest, destStorageRef, destStorage.tpe, destAttrAndSourceIris) } yield fileResources }.span("copyFiles") @@ -58,13 +59,22 @@ object BatchFiles { dest: CopyFileDestination, destStorageRef: ResourceRef.Revision, destStorageTpe: StorageType, - destFilesAttributes: NonEmptyList[FileAttributes] + destAttrAndSourceResourceRefs: NonEmptyList[(FileAttributes, ResourceRef)] )(implicit c: Caller): IO[NonEmptyList[FileResource]] = - destFilesAttributes.traverse { destFileAttributes => + destAttrAndSourceResourceRefs.traverse { case (destFileAttributes, source) => for { iri <- generateId(pc) command = - CreateFile(iri, dest.project, destStorageRef, destStorageTpe, destFileAttributes, c.subject, dest.tag) + CreateFile( + iri, + dest.project, + destStorageRef, + destStorageTpe, + destFileAttributes, + c.subject, + dest.tag, + Some(source) + ) resource <- evalCreateCommand(command) } yield resource } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala index 813e9b11c6..24f39f2931 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala @@ -15,7 +15,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, Tags} import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import io.circe.syntax._ -import io.circe.{Encoder, Json} +import io.circe.{Encoder, Json, JsonObject} /** * A representation of a file information @@ -39,7 +39,8 @@ final case class File( storage: ResourceRef.Revision, storageType: StorageType, attributes: FileAttributes, - tags: Tags + tags: Tags, + sourceFile: Option[ResourceRef] ) { def metadata: Metadata = Metadata(tags.tags) } @@ -56,7 +57,9 @@ object File { keywords.tpe -> storageType.iri.asJson, "_rev" -> file.storage.rev.asJson ) - file.attributes.asJsonObject.add("_storage", storageJson) + val attrJson = file.attributes.asJsonObject + val sourceFileJson = file.sourceFile.fold(JsonObject.empty)(f => JsonObject("_sourceFile" := f.asJson)) + sourceFileJson deepMerge attrJson add ("_storage", storageJson) } implicit def fileJsonLdEncoder(implicit showLocation: ShowFileLocation): JsonLdEncoder[File] = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala index 24c383c02c..bfdab793a0 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCommand.scala @@ -65,7 +65,8 @@ object FileCommand { storageType: StorageType, attributes: FileAttributes, subject: Subject, - tag: Option[UserTag] + tag: Option[UserTag], + sourceFile: Option[ResourceRef] ) extends FileCommand { override def rev: Int = 0 } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileEvent.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileEvent.scala index 5c26e72106..4f8bf7f9ea 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileEvent.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileEvent.scala @@ -94,7 +94,8 @@ object FileEvent { rev: Int, instant: Instant, subject: Subject, - tag: Option[UserTag] + tag: Option[UserTag], + sourceFile: Option[ResourceRef] ) extends FileEvent /** diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala index a55f658f1f..7115aeb9e2 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala @@ -13,6 +13,9 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} final case class FileId(id: IdSegmentRef, project: ProjectRef) { def expandIri(fetchContext: ProjectRef => IO[ProjectContext]): IO[(Iri, ProjectContext)] = fetchContext(project).flatMap(pc => iriExpander(id.value, pc).map(iri => (iri, pc))) + + def toResourceRef(fetchContext: ProjectRef => IO[ProjectContext]): IO[ResourceRef] = + fetchContext(project).flatMap(pc => iriExpander(id, pc)) } object FileId { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileState.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileState.scala index 5b98b15b0d..9b275cbde9 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileState.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileState.scala @@ -6,14 +6,15 @@ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.model.{ResourceF, ResourceUris, Tags} import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.instances._ +import ch.epfl.bluebrain.nexus.delta.sdk.circe.dropNullValues import ch.epfl.bluebrain.nexus.delta.sourcing.Serializer import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} import ch.epfl.bluebrain.nexus.delta.sourcing.state.State.ScopedState -import io.circe.Codec +import io.circe.{Codec, Encoder} import io.circe.generic.extras.Configuration -import io.circe.generic.extras.semiauto.deriveConfiguredCodec +import io.circe.generic.extras.semiauto.{deriveConfiguredCodec, deriveConfiguredDecoder, deriveConfiguredEncoder} import java.time.Instant import scala.annotation.nowarn @@ -52,6 +53,7 @@ final case class FileState( storage: ResourceRef.Revision, storageType: StorageType, attributes: FileAttributes, + sourceFile: Option[ResourceRef], tags: Tags, rev: Int, deprecated: Boolean, @@ -73,7 +75,7 @@ final case class FileState( */ def types: Set[Iri] = Set(nxvFile) - private def file: File = File(id, project, storage, storageType, attributes, tags) + private def file: File = File(id, project, storage, storageType, attributes, tags, sourceFile) def toResource: FileResource = ResourceF( @@ -103,7 +105,8 @@ object FileState { deriveConfiguredCodec[Digest] implicit val fileAttributesCodec: Codec.AsObject[FileAttributes] = deriveConfiguredCodec[FileAttributes] - implicit val codec: Codec.AsObject[FileState] = deriveConfiguredCodec[FileState] + implicit val enc: Encoder.AsObject[FileState] = deriveConfiguredEncoder[FileState].mapJsonObject(dropNullValues) + implicit val codec: Codec.AsObject[FileState] = Codec.AsObject.from(deriveConfiguredDecoder[FileState], enc) Serializer.dropNullsInjectType() } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala index c259fa68e8..ef47737660 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala @@ -1,42 +1,15 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes import cats.data.NonEmptyList -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId -import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag -import io.circe.{Decoder, DecodingFailure, Json} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import io.circe.Decoder final case class CopyFileSource( project: ProjectRef, - files: NonEmptyList[FileId] + files: NonEmptyList[ResourceRef] ) object CopyFileSource { - - implicit val dec: Decoder[CopyFileSource] = Decoder.instance { cur => - def parseSingle(j: Json, proj: ProjectRef): Decoder.Result[FileId] = - for { - sourceFile <- j.hcursor.get[String]("sourceFileId").map(IdSegment(_)) - sourceTag <- j.hcursor.get[Option[UserTag]]("sourceTag") - sourceRev <- j.hcursor.get[Option[Int]]("sourceRev") - fileId <- parseFileId(sourceFile, proj, sourceTag, sourceRev) - } yield fileId - - def parseFileId(id: IdSegment, proj: ProjectRef, sourceTag: Option[UserTag], sourceRev: Option[Int]) = - (sourceTag, sourceRev) match { - case (Some(tag), None) => Right(FileId(id, tag, proj)) - case (None, Some(rev)) => Right(FileId(id, rev, proj)) - case (None, None) => Right(FileId(id, proj)) - case (Some(_), Some(_)) => - Left( - DecodingFailure("Tag and revision cannot be simultaneously present for source file lookup", Nil) - ) - } - - for { - sourceProj <- cur.get[ProjectRef]("sourceProjectRef") - files <- cur.get[NonEmptyList[Json]]("files").flatMap(_.traverse(parseSingle(_, sourceProj))) - } yield CopyFileSource(sourceProj, files) - } + implicit val dec: Decoder[CopyFileSource] = + Decoder.forProduct2("sourceProject", "files")(CopyFileSource.apply) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala index 49b8d03841..84afa1448a 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala @@ -50,10 +50,10 @@ class FilesStmSpec extends CatsEffectSpec with FileFixtures with StorageFixtures "evaluating an incoming command" should { "create a new event from a CreateFile command" in { - val createCmd = CreateFile(id, projectRef, storageRef, DiskStorageType, attributes, bob, Some(myTag)) + val createCmd = CreateFile(id, projectRef, storageRef, DiskStorageType, attributes, bob, Some(myTag), None) evaluate(clock)(None, createCmd).accepted shouldEqual - FileCreated(id, projectRef, storageRef, DiskStorageType, attributes, 1, epoch, bob, Some(myTag)) + FileCreated(id, projectRef, storageRef, DiskStorageType, attributes, 1, epoch, bob, Some(myTag), None) } "create a new event from a UpdateFile command" in { @@ -121,7 +121,10 @@ class FilesStmSpec extends CatsEffectSpec with FileFixtures with StorageFixtures "reject with ResourceAlreadyExists when file already exists" in { val current = FileGen.state(id, projectRef, storageRef, attributes) - evaluate(clock)(Some(current), CreateFile(id, projectRef, storageRef, DiskStorageType, attributes, bob, None)) + evaluate(clock)( + Some(current), + CreateFile(id, projectRef, storageRef, DiskStorageType, attributes, bob, None, None) + ) .rejectedWith[ResourceAlreadyExists] } @@ -179,7 +182,7 @@ class FilesStmSpec extends CatsEffectSpec with FileFixtures with StorageFixtures "producing next state" should { "from a new FileCreated event" in { - val event = FileCreated(id, projectRef, storageRef, DiskStorageType, attributes, 1, epoch, bob, None) + val event = FileCreated(id, projectRef, storageRef, DiskStorageType, attributes, 1, epoch, bob, None, None) val nextState = FileGen.state(id, projectRef, storageRef, attributes, createdBy = bob, updatedBy = bob) next(None, event).value shouldEqual nextState diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala index 014ad46d98..c9f1c9c90e 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala @@ -36,14 +36,14 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit implicit private val fixedUUidF: UUIDF = UUIDF.fixed(sourceFileDescUuid) private val sourceProj = genProject() - private val sourceFileId = genFileId(sourceProj.ref) + private val sourceFileId = genResourceRef() private val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId)) private val storageStatEntry = StorageStatEntry(files = 10L, spaceUsed = 5L) private val stubbedFileAttr = NonEmptyList.of(attributes(genString())) test("successfully perform disk copy") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceProj.ref, sourceFileId, diskVal) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -58,7 +58,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => val obtainedEvents = events.toList assertEquals(obtained, stubbedFileAttr) - sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceFileWasFetched(obtainedEvents, sourceFileId, sourceProj.ref) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) diskCopyWasPerformed(obtainedEvents, destStorage, sourceFileRes.value.attributes) @@ -67,7 +67,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit test("successfully perform remote disk copy") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, remoteVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceProj.ref, sourceFileId, remoteVal) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -82,7 +82,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => val obtainedEvents = events.toList assertEquals(obtained, stubbedFileAttr) - sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceFileWasFetched(obtainedEvents, sourceFileId, sourceProj.ref) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) destinationRemoteStorageStatsWereNotFetched(obtainedEvents) remoteDiskCopyWasPerformed(obtainedEvents, destStorage, sourceFileRes.value.attributes) @@ -98,7 +98,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit test("fail if a source storage is different to destination storage") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceProj.ref, sourceFileId, diskVal) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -110,14 +110,14 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit batchCopy.copyFiles(source, genRemoteStorage())(caller(user)).interceptEquals(expectedError).map { _ => val obtainedEvents = events.toList - sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceFileWasFetched(obtainedEvents, sourceFileId, sourceProj.ref) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) } } test("fail if user does not have read access on a source file's storage") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceProj.ref, sourceFileId, diskVal) val user = genUser() val aclCheck = AclSimpleCheck((user, AclAddress.fromProject(sourceProj.ref), Set())).accepted @@ -129,14 +129,14 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit batchCopy.copyFiles(source, genDiskStorage())(caller(user)).intercept[AuthorizationFailed].map { _ => val obtainedEvents = events.toList - sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceFileWasFetched(obtainedEvents, sourceFileId, sourceProj.ref) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) } } test("fail if a single source file exceeds max size for destination storage") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, 1000L) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceProj.ref, sourceFileId, diskVal, 1000L) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -149,7 +149,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit batchCopy.copyFiles(source, destStorage)(caller(user)).interceptEquals(error).map { _ => val obtainedEvents = events.toList - sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceFileWasFetched(obtainedEvents, sourceFileId, sourceProj.ref) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) } } @@ -160,7 +160,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit val capacity = 10L val statEntry = StorageStatEntry(files = 10L, spaceUsed = 1L) val spaceLeft = capacity - statEntry.spaceUsed - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, fileSize) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceProj.ref, sourceFileId, diskVal, fileSize) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -176,7 +176,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit batchCopy.copyFiles(twoFileSource, destStorage)(caller(user)).interceptEquals(error).map { _ => val obtainedEvents = events.toList - sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceFileWasFetched(obtainedEvents, sourceFileId, sourceProj.ref) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) } @@ -197,9 +197,9 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit (user, AclSimpleCheck((user, AclAddress.fromProject(storage.project), permissions)).accepted) } - private def sourceFileWasFetched(events: List[Event], id: FileId) = { + private def sourceFileWasFetched(events: List[Event], ref: ResourceRef, proj: ProjectRef) = { val obtained = events.collectFirst { case f: FetchFileCalled => f } - assertEquals(obtained, Some(FetchFileCalled(id))) + assertEquals(obtained, Some(FetchFileCalled(FileId(ref, proj)))) } private def sourceStorageWasFetched(events: List[Event], storageRef: ResourceRef.Revision, proj: ProjectRef) = { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala index 30d1ee6b82..9d7730cd28 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch import cats.data.NonEmptyList import cats.effect.IO +import cats.implicits.toFunctorOps import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen @@ -21,11 +22,18 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, FetchContextDum import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} import ch.epfl.bluebrain.nexus.testkit.Generators import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite +import org.scalatest.OptionValues import java.util.UUID import scala.collection.mutable.ListBuffer -class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { +class BatchFilesSuite + extends NexusSuite + with StorageFixtures + with Generators + with FileFixtures + with FileGen + with OptionValues { private val destProj: Project = genProject() private val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) @@ -38,12 +46,13 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi val stubbedDestAttributes = genAttributes() val batchCopy = BatchCopyMock.withStubbedCopyFiles(events, stubbedDestAttributes) - val batchFiles: BatchFiles = mkBatchFiles(events, destProj, destFileUUId, fetchFileStorage, batchCopy) + val sourceProj = genProject() + val batchFiles: BatchFiles = mkBatchFiles(events, destProj, sourceProj, destFileUUId, fetchFileStorage, batchCopy) implicit val c: Caller = Caller(genUser(), Set()) - val source = genCopyFileSource() + val source = genCopyFileSource(sourceProj.ref) batchFiles.copyFiles(source, destination).map { obtained => - val expectedCommands = createCommandsFromFileAttributes(stubbedDestAttributes) + val expectedCommands = createCommandsFromFileAttributes(stubbedDestAttributes, source.files) val expectedResources = expectedCommands.map(genFileResourceFromCmd) val expectedCommandCalls = expectedCommands.toList.map(FileCommandEvaluated) val expectedEvents = activeStorageFetchedAndBatchCopyCalled(source) ++ expectedCommandCalls @@ -53,13 +62,34 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi } } + test("batch copying should return source file iris including tags and revisions") { + val events = ListBuffer.empty[Event] + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val sourceProj = genProject() + val (byRevFile, byTagFile, latestFile) = + (genResourceRefWithRev(), genResourceRefWithTag(), genResourceRef()) + val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(byTagFile, byRevFile, latestFile)) + val attr = source.files.as(attributes()) + val batchCopy = BatchCopyMock.withStubbedCopyFiles(events, attr) + + val batchFiles: BatchFiles = mkBatchFiles(events, destProj, sourceProj, destFileUUId, fetchFileStorage, batchCopy) + implicit val c: Caller = Caller(genUser(), Set()) + + batchFiles.copyFiles(source, destination).map { obtained => + val expectedSourceFiles = NonEmptyList.of(byTagFile, byRevFile, latestFile) + + assertEquals(obtained.map(_.value.sourceFile.value), expectedSourceFiles) + } + } + test("copy rejections should be mapped to a file rejection") { val events = ListBuffer.empty[Event] val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) val error = TotalCopySizeTooLarge(1L, 2L, genIri()) val batchCopy = BatchCopyMock.withError(error, events) - val batchFiles: BatchFiles = mkBatchFiles(events, destProj, UUID.randomUUID(), fetchFileStorage, batchCopy) + val batchFiles: BatchFiles = + mkBatchFiles(events, destProj, genProject(), UUID.randomUUID(), fetchFileStorage, batchCopy) implicit val c: Caller = Caller(genUser(), Set()) val source = genCopyFileSource() val expectedError = CopyRejection(source.project, destProj.ref, destStorage.id, error) @@ -83,6 +113,7 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi def mkBatchFiles( events: ListBuffer[Event], proj: Project, + sourceProj: Project, fixedUuid: UUID, fetchFileStorage: FetchFileStorage, batchCopy: BatchCopy @@ -91,7 +122,8 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi val evalFileCmd: CreateFile => IO[FileResource] = cmd => IO(events.addOne(FileCommandEvaluated(cmd))).as(genFileResourceFromCmd(cmd)) val fetchContext: FetchContext[FileRejection] = - FetchContextDummy(Map(proj.ref -> proj.context)).mapRejection(FileRejection.ProjectContextRejection) + FetchContextDummy(Map(proj.ref -> proj.context, sourceProj.ref -> sourceProj.context)) + .mapRejection(FileRejection.ProjectContextRejection) BatchFiles.mk(fetchFileStorage, fetchContext, evalFileCmd, batchCopy) } @@ -101,19 +133,23 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi List(expectedActiveStorageFetched, expectedBatchCopyCalled) } - def createCommandsFromFileAttributes(stubbedDestAttributes: NonEmptyList[FileAttributes])(implicit + def createCommandsFromFileAttributes( + stubbedDestAttributes: NonEmptyList[FileAttributes], + sourceFiles: NonEmptyList[ResourceRef] + )(implicit c: Caller - ): NonEmptyList[CreateFile] = stubbedDestAttributes.map( + ): NonEmptyList[CreateFile] = stubbedDestAttributes.zip(sourceFiles).map { case (destAttr, source) => CreateFile( destProj.base.iri / destFileUUId.toString, destProj.ref, destStorageRef, destStorage.value.tpe, - _, + destAttr, c.subject, - destination.tag + destination.tag, + Some(source) ) - ) + } } object BatchFilesSuite { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala index 35ff172471..f9f0bcbb7e 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -3,19 +3,21 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.Uri import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileState} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileFixtures, FileResource, schemas} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageGen, StorageResource} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, Tags} -import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, Project} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} @@ -41,61 +43,73 @@ trait FileGen { self: Generators with FileFixtures => def genFilesIdsInProject(projRef: ProjectRef): NonEmptyList[FileId] = NonEmptyList.of(genFileId(projRef), genFileId(projRef)) + def genResourceRefsInProject(): NonEmptyList[ResourceRef] = + NonEmptyList.of(genResourceRef(), genResourceRef()) + def genFileId(projRef: ProjectRef) = FileId(genString(), projRef) def genFileIdWithRev(projRef: ProjectRef): FileId = FileId(genString(), 4, projRef) def genFileIdWithTag(projRef: ProjectRef): FileId = FileId(genString(), UserTag.unsafe(genString()), projRef) + def genResourceRef() = ResourceRef(iri"https://bbp.epfl.ch/${genString()}") + + def genResourceRefWithRev() = ResourceRef(iri"https://bbp.epfl.ch/${genString()}?rev=4") + + def genResourceRefWithTag() = ResourceRef(iri"https://bbp.epfl.ch/${genString()}?tag=${genString()}") + def genAttributes(): NonEmptyList[FileAttributes] = { val proj = genProject() - genFilesIdsInProject(proj.ref).map(genFileResource(_, proj.context)).map(_.value.attributes) + genResourceRefsInProject().map(genFileResource(_, proj.ref, None)).map(_.value.attributes) } def genCopyFileSource(): CopyFileSource = genCopyFileSource(genProjectRef()) - def genCopyFileSource(proj: ProjectRef) = CopyFileSource(proj, genFilesIdsInProject(proj)) + def genCopyFileSource(proj: ProjectRef) = CopyFileSource(proj, genResourceRefsInProject()) def genCopyFileDestination(proj: ProjectRef, storage: Storage): CopyFileDestination = CopyFileDestination(proj, genOption(IdSegment(storage.id.toString)), genOption(genUserTag)) def genUserTag: UserTag = UserTag.unsafe(genString()) def genOption[A](genA: => A): Option[A] = if (Random.nextInt(2) % 2 == 0) Some(genA) else None - def genFileResource(fileId: FileId, context: ProjectContext): FileResource = - genFileResourceWithStorage(fileId, context, genRevision(), 1L) + def genFileResource(ref: ResourceRef, proj: ProjectRef, sourceFile: Option[ResourceRef]): FileResource = + genFileResourceWithStorage(proj, ref, genRevision(), 1L, sourceFile) def genFileResourceWithStorage( - fileId: FileId, - context: ProjectContext, + proj: ProjectRef, + ref: ResourceRef, storageRef: ResourceRef.Revision, - fileSize: Long + fileSize: Long, + sourceFile: Option[ResourceRef] = None ): FileResource = genFileResourceWithIri( - fileId.id.value.toIri(context.apiMappings, context.base).getOrElse(throw new Exception(s"Bad file $fileId")), - fileId.project, + iri"${UrlUtils.encode(ref.toString)}", + proj, storageRef, - attributes(genString(), size = fileSize) + attributes(genString(), size = fileSize), + sourceFile ) def genFileResourceAndStorage( - fileId: FileId, - context: ProjectContext, + proj: ProjectRef, + ref: ResourceRef, storageVal: StorageValue, fileSize: Long = 1L ): (FileResource, StorageResource) = { - val storageRes = StorageGen.resourceFor(genIri(), fileId.project, storageVal) + val storageRes = StorageGen.resourceFor(genIri(), proj, storageVal) val storageRef = ResourceRef.Revision(storageRes.id, storageRes.id, storageRes.rev) - (genFileResourceWithStorage(fileId, context, storageRef, fileSize), storageRes) + (genFileResourceWithStorage(proj, ref, storageRef, fileSize), storageRes) } def genFileResourceWithIri( iri: Iri, projRef: ProjectRef, storageRef: ResourceRef.Revision, - attr: FileAttributes + attr: FileAttributes, + sourceFile: Option[ResourceRef] ): FileResource = - FileGen.resourceFor(iri, projRef, storageRef, attr) + FileGen.resourceFor(iri, projRef, storageRef, attr, sourceFile = sourceFile) def genFileResourceFromCmd(cmd: CreateFile): FileResource = - genFileResourceWithIri(cmd.id, cmd.project, cmd.storage, cmd.attributes) + genFileResourceWithIri(cmd.id, cmd.project, cmd.storage, cmd.attributes, cmd.sourceFile) def genIri(): Iri = Iri.unsafe(genString()) def genStorage(proj: ProjectRef, storageValue: StorageValue): StorageState = StorageGen.storageState(genIri(), proj, storageValue) @@ -117,14 +131,16 @@ object FileGen { deprecated: Boolean = false, tags: Tags = Tags.empty, createdBy: Subject = Anonymous, - updatedBy: Subject = Anonymous - ): FileState = { + updatedBy: Subject = Anonymous, + sourceFile: Option[ResourceRef] = None + ): FileState = FileState( id, project, storage, storageType, attributes, + sourceFile, tags, rev, deprecated, @@ -133,7 +149,6 @@ object FileGen { Instant.EPOCH, updatedBy ) - } def resourceFor( id: Iri, @@ -145,9 +160,22 @@ object FileGen { deprecated: Boolean = false, tags: Tags = Tags.empty, createdBy: Subject = Anonymous, - updatedBy: Subject = Anonymous + updatedBy: Subject = Anonymous, + sourceFile: Option[ResourceRef] = None ): FileResource = - state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource + state( + id, + project, + storage, + attributes, + storageType, + rev, + deprecated, + tags, + createdBy, + updatedBy, + sourceFile + ).toResource def mkTempDir(prefix: String) = AbsolutePath(JavaFiles.createTempDirectory(prefix)).fold(e => throw new Exception(e), identity) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala index f487252f3e..5b9227b0c5 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala @@ -4,13 +4,13 @@ import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FileResource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileId} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.CopyFileDestination import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} import scala.collection.mutable.ListBuffer @@ -44,12 +44,12 @@ object BatchFilesMock { object BatchFilesCopyFilesCalled { def fromTestData( - destProj: ProjectRef, - sourceProj: ProjectRef, - sourceFiles: NonEmptyList[FileId], - user: User, - destStorage: Option[IdSegment] = None, - destTag: Option[UserTag] = None + destProj: ProjectRef, + sourceProj: ProjectRef, + sourceFiles: NonEmptyList[ResourceRef], + user: User, + destStorage: Option[IdSegment] = None, + destTag: Option[UserTag] = None ): BatchFilesCopyFilesCalled = { val expectedCopyFileSource = CopyFileSource(sourceProj, sourceFiles) val expectedCopyFileDestination = CopyFileDestination(destProj, destStorage, destTag) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileSerializationSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileSerializationSuite.scala index 680e5481eb..91c4e85127 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileSerializationSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileSerializationSuite.scala @@ -48,7 +48,7 @@ class FileSerializationSuite extends SerializationSuite with StorageFixtures { ) // format: off - private val created = FileCreated(fileId, projectRef, storageRef, DiskStorageType, attributes.copy(digest = NotComputedDigest), 1, instant, subject, None) + private val created = FileCreated(fileId, projectRef, storageRef, DiskStorageType, attributes.copy(digest = NotComputedDigest), 1, instant, subject, None, None) private val createdTagged = created.copy(tag = Some(tag)) private val updated = FileUpdated(fileId, projectRef, storageRef, DiskStorageType, attributes, 2, instant, subject, Some(tag)) private val updatedAttr = FileAttributesUpdated(fileId, projectRef, storageRef, DiskStorageType, Some(`text/plain(UTF-8)`), 12, digest, 3, instant, subject) @@ -169,6 +169,7 @@ class FileSerializationSuite extends SerializationSuite with StorageFixtures { storageRef, DiskStorageType, attributes, + None, Tags(UserTag.unsafe("mytag") -> 3), 5, false, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala index c79c937e2e..89762d4a57 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala @@ -10,9 +10,9 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock.BatchFilesCopyFilesCalled +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{CopyRejection, FileNotFound, WrappedStorageRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileId, FileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileFixtures, FileResource, contexts => fileContexts} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, StorageNotFound} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType @@ -24,15 +24,15 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller -import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef} +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.Project import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} import io.circe.Json -import io.circe.syntax.KeyOps +import io.circe.syntax.{EncoderOps, KeyOps} import org.scalatest.Assertion import scala.collection.mutable.ListBuffer @@ -51,39 +51,39 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF "succeed for source files looked up by latest" in { val sourceProj = genProject() - val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val sourceFileIds = genResourceRefsInProject() testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) } "succeed for source files looked up by tag" in { val sourceProj = genProject() - val sourceFileIds = NonEmptyList.of(genFileIdWithTag(sourceProj.ref), genFileIdWithTag(sourceProj.ref)) + val sourceFileIds = NonEmptyList.of(genResourceRefWithTag(), genResourceRefWithTag()) testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) } "succeed for source files looked up by rev" in { val sourceProj = genProject() - val sourceFileIds = NonEmptyList.of(genFileIdWithRev(sourceProj.ref), genFileIdWithRev(sourceProj.ref)) + val sourceFileIds = NonEmptyList.of(genResourceRefWithRev(), genResourceRefWithRev()) testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) } "succeed with a specific destination storage" in { val sourceProj = genProject() - val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val sourceFileIds = genResourceRefsInProject() val destStorageId = IdSegment(genString()) testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destStorageId = Some(destStorageId)) } "succeed with a user tag applied to destination files" in { val sourceProj = genProject() - val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val sourceFileIds = genResourceRefsInProject() val destTag = UserTag.unsafe(genString()) testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destTag = Some(destTag)) } "return 403 for a user without read permission on the source project" in { val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) - val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val sourceFileIds = genResourceRefsInProject() val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) @@ -93,18 +93,6 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF } } - "return 400 if tag and rev are present simultaneously for a source file" in { - val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) - - val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) - val invalidFilePayload = BatchFilesRoutesSpec.mkSourceFilePayload(genString(), Some(3), Some(genString())) - val payload = Json.obj("sourceProjectRef" := sourceProj.ref, "files" := List(invalidFilePayload)) - - callBulkCopyEndpoint(route, destProj.ref, payload, user) { - response.status shouldBe StatusCodes.BadRequest - } - } - "return 400 for copy errors raised by batch file logic" in { val unsupportedStorageType = UnsupportedOperation(StorageType.S3Storage) val fileTooLarge = SourceFileTooLarge(12, genIri()) @@ -115,7 +103,7 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF forAll(errors) { error => val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) - val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val sourceFileIds = genResourceRefsInProject() val events = ListBuffer.empty[BatchFilesCopyFilesCalled] val batchFiles = BatchFilesMock.withError(error, events) @@ -143,7 +131,7 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF forAll(fileRejections) { case (error, expectedStatus, expectedJson) => val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) - val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val sourceFileIds = genResourceRefsInProject() val events = ListBuffer.empty[BatchFilesCopyFilesCalled] val batchFiles = BatchFilesMock.withError(error, events) @@ -196,15 +184,17 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF } def testBulkCopySucceedsForStubbedFiles( - sourceProj: Project, - sourceFileIds: NonEmptyList[FileId], - destStorageId: Option[IdSegment] = None, - destTag: Option[UserTag] = None + sourceProj: Project, + sourceFileIds: NonEmptyList[ResourceRef], + destStorageId: Option[IdSegment] = None, + destTag: Option[UserTag] = None ): Assertion = { val (destProj, user) = (genProject(), genUser(realm)) - val sourceFileResources = sourceFileIds.map(genFileResource(_, destProj.context)) + val createdFileResources = sourceFileIds.map(f => + genFileResource(genResourceRef(), destProj.ref, Some(f)) + ) val events = ListBuffer.empty[BatchFilesCopyFilesCalled] - val stubbedBatchFiles = BatchFilesMock.withStubbedCopyFiles(sourceFileResources, events) + val stubbedBatchFiles = BatchFilesMock.withStubbedCopyFiles(createdFileResources, events) val route = mkRoute(stubbedBatchFiles, sourceProj, user, Set(files.permissions.read)) val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) @@ -221,7 +211,7 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF destTag ) events.toList shouldBe List(expectedBatchFilesCall) - response.asJson shouldBe expectedBulkCopyJson(sourceFileResources) + response.asJson.spaces2SortKeys shouldBe expectedBulkCopyJson(createdFileResources).spaces2SortKeys } } @@ -246,21 +236,10 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF ) .accepted .mapObject(_.remove("@context")) + .deepMerge(res.value.sourceFile.fold(Json.obj())(s => Json.obj("_sourceFile" := s.toString))) } object BatchFilesRoutesSpec { - def mkBulkCopyPayload(sourceProj: ProjectRef, sourceFileIds: NonEmptyList[FileId]): Json = - Json.obj("sourceProjectRef" := sourceProj.toString, "files" := mkSourceFilesPayload(sourceFileIds)) - - def mkSourceFilesPayload(sourceFileIds: NonEmptyList[FileId]): NonEmptyList[Json] = - sourceFileIds.map(id => mkSourceFilePayloadFromIdSegmentRef(id.id)) - - def mkSourceFilePayloadFromIdSegmentRef(id: IdSegmentRef): Json = id match { - case IdSegmentRef.Latest(value) => mkSourceFilePayload(value.asString, None, None) - case IdSegmentRef.Revision(value, rev) => mkSourceFilePayload(value.asString, Some(rev), None) - case IdSegmentRef.Tag(value, tag) => mkSourceFilePayload(value.asString, None, Some(tag.value)) - } - - def mkSourceFilePayload(id: String, rev: Option[Int], tag: Option[String]): Json = - Json.obj("sourceFileId" := id, "sourceRev" := rev, "sourceTag" := tag) + def mkBulkCopyPayload(sourceProj: ProjectRef, sourceFileIds: NonEmptyList[ResourceRef]): Json = + Json.obj("sourceProject" := sourceProj.toString, "files" := sourceFileIds.map(_.asJson)) } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala index 7f655571c5..6acb392b85 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala @@ -182,4 +182,8 @@ object Optics { val allProjects = root.projections.each.metadata.project } + object files { + val _sourceFile = root._sourceFile.string + } + } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/BatchCopySpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/BatchCopySpec.scala index fc7ece7068..0cbb079ebd 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/BatchCopySpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/BatchCopySpec.scala @@ -88,8 +88,8 @@ class BatchCopySpec extends BaseIntegrationSpec { def genTextFileInput(): FileInput = FileInput(genId(), genString(), ContentTypes.`text/plain(UTF-8)`, genString()) def mkPayload(sourceProjRef: String, sourceFiles: List[FileInput]): Json = { - val sourcePayloads = sourceFiles.map(f => Json.obj("sourceFileId" := f.fileId)) - Json.obj("sourceProjectRef" := sourceProjRef, "files" := sourcePayloads) + val sourcePayloads = sourceFiles.map(sourceFileId(_, sourceProjRef)) + Json.obj("sourceProject" := sourceProjRef, "files" := sourcePayloads) } def uploadFile(file: FileInput, storage: StorageDetails): IO[Assertion] = @@ -105,14 +105,18 @@ class BatchCopySpec extends BaseIntegrationSpec { val uri = s"/bulk/files/$destProjRef?storage=nxv:${destStorage.storageId}" for { - response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => - (json, expectCreated(json, response)) - } - _ <- checkFileResourcesExist(destProjRef, response) - assertions <- checkFileContentsAreCopiedCorrectly(destProjRef, sourceFiles, response) + response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => + (json, expectCreated(json, response)) + } + expectedSourceFileIds = sourceFiles.map(sourceFileId(_, sourceProjRef)) + _ <- checkFileResourcesExist(destProjRef, expectedSourceFileIds.zip(response.ids)) + assertions <- checkFileContentsAreCopiedCorrectly(destProjRef, sourceFiles, response) } yield assertions.head } + def sourceFileId(input: FileInput, sourceProjRef: String): String = + s"http://delta:8080/v1/resources/$sourceProjRef/_/${input.fileId}" + def checkFileContentsAreCopiedCorrectly(destProjRef: String, sourceFiles: List[FileInput], response: Response) = response.ids.zip(sourceFiles).traverse { case (destId, FileInput(_, filename, contentType, contents)) => deltaClient @@ -121,11 +125,12 @@ class BatchCopySpec extends BaseIntegrationSpec { } } - def checkFileResourcesExist(destProjRef: String, response: Response) = - response.ids.traverse { id => - deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(id)}", Coyote) { (json, response) => + def checkFileResourcesExist(destProjRef: String, fullResourceUrls: List[(String, String)]) = + fullResourceUrls.traverse { case (source, dest) => + deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(dest)}", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK - Optics.`@id`.getOption(json) shouldEqual Some(id) + Optics.`@id`.getOption(json) shouldEqual Some(dest) + Optics.files._sourceFile.getOption(json) shouldEqual Some(source) } }