Restructure file system (#196)

- Restructure file system to save study related files under a study/deployment specific repository - Introducing a new parameter when uploading files: `deploymentID`
cph-cachet · Feb 6, 2025 · 7e082e6 · 7e082e6
1 parent 416b0c7
commit 7e082e6
Show file tree

Hide file tree

Showing 25 changed files with 667 additions and 273 deletions.
diff --git a/src/main/kotlin/dk/cachet/carp/webservices/datastream/service/impl/DataStreamService.kt b/src/main/kotlin/dk/cachet/carp/webservices/datastream/service/impl/DataStreamService.kt
@@ -164,6 +164,8 @@ class DataStreamService(
                 LOGGER.info("A new file is created for zipping with name ${path.fileName}.")
             } catch (e: IOException) {
                 LOGGER.error("An error occurred while storing the file ${path.fileName}", e)
+            } catch (e: IllegalArgumentException) {
+                LOGGER.error("An error occurred while storing the file (empty dataStreamList) ${path.fileName}", e)
             }
         }
 

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/export/command/ExportCommand.kt b/src/main/kotlin/dk/cachet/carp/webservices/export/command/ExportCommand.kt
@@ -17,6 +17,7 @@ import kotlinx.datetime.Clock
 import kotlinx.datetime.toJavaInstant
 import org.springframework.stereotype.Service
 import java.nio.ByteBuffer
+import java.nio.file.Path
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import java.time.temporal.ChronoUnit
@@ -64,13 +65,21 @@ class ExportCommandFactory(
                 ExportType.DEPLOYMENT_DATA
             }
 
+        val relativePath =
+            if (exportType == ExportType.STUDY_DATA) {
+                Path.of("studies", studyId.toString(), "exports")
+            } else {
+                Path.of("studies", studyId.toString(), "deployments", deploymentIds!!.first().toString(), "exports")
+            }
+
         val entry =
             Export(
                 id = id.toString(),
                 fileName = getDefaultFileName(studyId, exportType, "zip"),
-                studyId = studyId.toString(),
                 status = ExportStatus.IN_PROGRESS,
+                studyId = studyId.toString(),
                 type = exportType,
+                relativePath = relativePath.toString(),
             )
 
         return ExportSummary(entry, deploymentIds, resourceExporter, fileUtil)
@@ -86,13 +95,16 @@ class ExportCommandFactory(
                 Clock.System.now().toJavaInstant().truncatedTo(ChronoUnit.SECONDS).toString(),
             )
 
+        val relativePath = Path.of("studies", studyId.toString(), "anonymous-participants-exports")
+
         val entry =
             Export(
                 id = id.toString(),
                 fileName = getDefaultFileName(studyId, ExportType.ANONYMOUS_PARTICIPANTS, "csv"),
-                studyId = studyId.toString(),
                 status = ExportStatus.IN_PROGRESS,
+                studyId = studyId.toString(),
                 type = ExportType.ANONYMOUS_PARTICIPANTS,
+                relativePath = relativePath.toString(),
             )
 
         return ExportAnonymousParticipants(

diff --git a/...main/kotlin/dk/cachet/carp/webservices/export/command/impl/ExportAnonymousParticipants.kt b/...main/kotlin/dk/cachet/carp/webservices/export/command/impl/ExportAnonymousParticipants.kt
@@ -17,6 +17,7 @@ import dk.cachet.carp.webservices.study.domain.AnonymousParticipantRequest
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
 import kotlinx.datetime.Clock
+import java.nio.file.Path
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration
 
@@ -90,7 +91,11 @@ class ExportAnonymousParticipants(
                 "${it.username},${it.studyDeploymentId},\"${it.magicLink}\",${it.expiryDate}"
             }
 
-        val csvPath = fileUtil.resolveFileStorage(entry.fileName)
+        val csvPath =
+            fileUtil.resolveFileStoragePathForFilenameAndRelativePath(
+                entry.fileName,
+                Path.of(entry.relativePath),
+            )
         resourceExporter.exportCSV(CSV_HEADER, csvBody, csvPath, logger)
     }
 

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/export/command/impl/ExportSummary.kt b/src/main/kotlin/dk/cachet/carp/webservices/export/command/impl/ExportSummary.kt
@@ -7,6 +7,7 @@ import dk.cachet.carp.webservices.export.service.ResourceExporterService
 import dk.cachet.carp.webservices.file.util.FileUtil
 import org.apache.logging.log4j.LogManager
 import java.io.IOException
+import java.nio.file.Path
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.createTempDirectory
 import kotlin.io.path.deleteRecursively
@@ -26,7 +27,11 @@ class ExportSummary(
     @OptIn(ExperimentalPathApi::class)
     override suspend fun execute() {
         val workingDir = createTempDirectory()
-        val zipPath = fileUtil.resolveFileStorage(entry.fileName)
+        val zipPath =
+            fileUtil.resolveFileStoragePathForFilenameAndRelativePath(
+                entry.fileName,
+                Path.of(entry.relativePath),
+            )
 
         resourceExporter.exportStudyData(UUID(entry.studyId), deploymentIds, workingDir, logger)
 

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/export/controller/ExportController.kt b/src/main/kotlin/dk/cachet/carp/webservices/export/controller/ExportController.kt
@@ -16,6 +16,7 @@ import org.springframework.http.HttpStatus
 import org.springframework.http.ResponseEntity
 import org.springframework.security.access.prepost.PreAuthorize
 import org.springframework.web.bind.annotation.*
+import org.springframework.web.server.ResponseStatusException
 
 @RestController
 @RequestMapping(EXPORT_BASE)
@@ -43,6 +44,15 @@ class ExportController(
     ): Export {
         LOGGER.info("Start POST: $EXPORT_BASE$SUMMARIES")
 
+        try {
+            require(request.deploymentIds.isNullOrEmpty() || request.deploymentIds.size == 1) {
+                "We only support exporting an entire study or a single deployment," +
+                    " (deploymentsIds.size should be less than 2)."
+            }
+        } catch (e: IllegalArgumentException) {
+            throw ResponseStatusException(HttpStatus.BAD_REQUEST, e.message, e)
+        }
+
         val command = exportCommandFactory.createExportSummary(studyId, request.deploymentIds)
 
         return exportService.createExport(command)

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/export/domain/Export.kt b/src/main/kotlin/dk/cachet/carp/webservices/export/domain/Export.kt
@@ -19,6 +19,7 @@ data class Export(
     var status: ExportStatus = ExportStatus.UNKNOWN,
     var studyId: String = "",
     var type: ExportType = ExportType.UNKNOWN,
+    var relativePath: String = "",
 ) : Auditable()
 
 enum class ExportStatus {

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/export/service/impl/ExportServiceImpl.kt b/src/main/kotlin/dk/cachet/carp/webservices/export/service/impl/ExportServiceImpl.kt
@@ -13,6 +13,7 @@ import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
 import org.springframework.core.io.Resource
 import org.springframework.stereotype.Service
+import java.nio.file.Path
 
 @Service
 class ExportServiceImpl(
@@ -39,8 +40,7 @@ class ExportServiceImpl(
         exportId: UUID,
     ): Resource {
         val export = getExportOrThrow(exportId, studyId)
-
-        val file = fileStorage.getFile(export.fileName)
+        val file = fileStorage.getFileAtPath(export.fileName, Path.of(export.relativePath))
 
         LOGGER.info("Summary with id $studyId is being downloaded.")
 
@@ -59,9 +59,11 @@ class ExportServiceImpl(
             throw ConflictException("The export creation is still in progress.")
         }
 
-        fileStorage.deleteFile(export.fileName)
+        fileStorage.deleteFileAtPath(export.fileName, Path.of(export.relativePath))
         exportRepository.delete(export)
+
         LOGGER.info("Export with id $exportId has been successfully deleted.")
+
         return studyId
     }
 

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/file/authorization/FileControllerAuthorizer.kt b/src/main/kotlin/dk/cachet/carp/webservices/file/authorization/FileControllerAuthorizer.kt
@@ -0,0 +1,19 @@
+package dk.cachet.carp.webservices.file.authorization
+
+import dk.cachet.carp.webservices.file.repository.FileRepository
+import dk.cachet.carp.webservices.security.authentication.service.AuthenticationService
+import org.springframework.stereotype.Component
+
+@Component
+class FileControllerAuthorizer(
+    private val fileRepository: FileRepository,
+    private val authenticationService: AuthenticationService,
+) {
+    fun isFileOwner(fileId: Int): Boolean {
+        val file = fileRepository.findById(fileId)
+
+        if (file.isEmpty) return false
+
+        return authenticationService.getId().stringRepresentation == file.get().ownerId
+    }
+}
diff --git a/src/main/kotlin/dk/cachet/carp/webservices/file/controller/FileController.kt b/src/main/kotlin/dk/cachet/carp/webservices/file/controller/FileController.kt
@@ -5,7 +5,7 @@ import dk.cachet.carp.webservices.common.constants.PathVariableName
 import dk.cachet.carp.webservices.common.constants.RequestParamName
 import dk.cachet.carp.webservices.file.domain.File
 import dk.cachet.carp.webservices.file.service.FileService
-import dk.cachet.carp.webservices.file.service.FileStorage
+import dk.cachet.carp.webservices.security.authentication.service.AuthenticationService
 import io.swagger.v3.oas.annotations.Operation
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
@@ -19,7 +19,7 @@ import org.springframework.web.bind.annotation.*
 import org.springframework.web.multipart.MultipartFile
 
 @RestController
-class FileController(private val fileStorage: FileStorage, private val fileService: FileService) {
+class FileController(private val fileService: FileService, private val authenticationService: AuthenticationService) {
     companion object {
         private val LOGGER: Logger = LogManager.getLogger()
 
@@ -28,6 +28,18 @@ class FileController(private val fileStorage: FileStorage, private val fileServi
         const val UPLOAD_IMAGE = "/api/studies/{${PathVariableName.STUDY_ID}}/images"
         const val DOWNLOAD = "$FILE_BASE/{${PathVariableName.FILE_ID}}/download"
         const val FILE_ID = "$FILE_BASE/{${PathVariableName.FILE_ID}}"
+        const val CREATE = "$FILE_BASE/{${PathVariableName.DEPLOYMENT_ID}}"
+    }
+
+    @GetMapping(FILE_ID)
+    @Operation(tags = ["file/getOne.json"])
+    @PreAuthorize("canManageStudy(#studyId) or @fileControllerAuthorizer.isFileOwner(#fileId)")
+    fun getOne(
+        @PathVariable(PathVariableName.STUDY_ID) studyId: UUID,
+        @PathVariable(PathVariableName.FILE_ID) fileId: Int,
+    ): File {
+        LOGGER.info("Start GET: /api/studies/$studyId/files/$fileId")
+        return fileService.getOne(fileId)
     }
 
     @GetMapping(FILE_BASE)
@@ -41,17 +53,6 @@ class FileController(private val fileStorage: FileStorage, private val fileServi
         return fileService.getAll(query, studyId.stringRepresentation)
     }
 
-    @GetMapping(FILE_ID)
-    @Operation(tags = ["file/getOne.json"])
-    @PreAuthorize("canManageStudy(#studyId) or isFileOwner(#fileId)")
-    fun getOne(
-        @PathVariable(PathVariableName.STUDY_ID) studyId: UUID,
-        @PathVariable(PathVariableName.FILE_ID) fileId: Int,
-    ): File {
-        LOGGER.info("Start GET: /api/studies/$studyId/files/$fileId")
-        return fileService.getOne(fileId)
-    }
-
     @GetMapping(
         produces = [
             MediaType.MULTIPART_FORM_DATA_VALUE,
@@ -61,18 +62,18 @@ class FileController(private val fileStorage: FileStorage, private val fileServi
     )
     @ResponseBody
     @Operation(tags = ["file/download.json"])
-    @PreAuthorize("canManageStudy(#studyId) or isFileOwner(#id)")
+    @PreAuthorize("canManageStudy(#studyId) or @fileControllerAuthorizer.isFileOwner(#id)")
     fun download(
         @PathVariable(PathVariableName.STUDY_ID) studyId: UUID,
         @PathVariable(PathVariableName.FILE_ID) id: Int,
     ): ResponseEntity<Resource> {
         LOGGER.info("Start GET: /api/studies/$studyId/files/$id/download")
-        val file = fileService.getOne(id)
-        val fileToDownload = fileStorage.getFile(file.storageName)
+        val (fileToDownload, originalFilename) = fileService.download(id, studyId)
+
         return ResponseEntity.ok().header(
             HttpHeaders.CONTENT_DISPOSITION,
-            "attachment; filename=\"" + file.originalName + "\"",
-        ).body<Resource>(fileToDownload)
+            "attachment; filename=\"$originalFilename\"",
+        ).body(fileToDownload)
     }
 
     @PostMapping(
@@ -83,28 +84,54 @@ class FileController(private val fileStorage: FileStorage, private val fileServi
         produces = [MediaType.APPLICATION_JSON_VALUE],
         value = [FILE_BASE],
     )
+    @Operation(tags = ["file/create_deprecated.json"])
+    @ResponseStatus(HttpStatus.CREATED)
+    @PreAuthorize("canManageStudy(#studyId) or isInDeploymentOfStudy(#studyId)")
+    @Deprecated("Use the other -create- method instead.")
+    fun createDEPRICATED(
+        @PathVariable(PathVariableName.STUDY_ID) studyId: UUID,
+        @RequestParam(RequestParamName.METADATA, required = false) metadata: String?,
+        @RequestPart file: MultipartFile,
+    ): File {
+        LOGGER.info("Start POST: /api/studies/$studyId/files")
+        val ownerId = authenticationService.getId()
+
+        return fileService.createDEPRECATED(studyId.stringRepresentation, file, metadata, ownerId)
+    }
+
+    @PostMapping(
+        consumes = [
+            MediaType.MULTIPART_FORM_DATA_VALUE,
+            MediaType.APPLICATION_OCTET_STREAM_VALUE,
+        ],
+        produces = [MediaType.APPLICATION_JSON_VALUE],
+        value = [CREATE],
+    )
     @Operation(tags = ["file/create.json"])
     @ResponseStatus(HttpStatus.CREATED)
     @PreAuthorize("canManageStudy(#studyId) or isInDeploymentOfStudy(#studyId)")
     fun create(
         @PathVariable(PathVariableName.STUDY_ID) studyId: UUID,
+        @PathVariable(PathVariableName.DEPLOYMENT_ID) deploymentId: UUID,
         @RequestParam(RequestParamName.METADATA, required = false) metadata: String?,
         @RequestPart file: MultipartFile,
     ): File {
-        LOGGER.info("Start POST: /api/studies/$studyId/files")
-        return fileService.create(studyId.stringRepresentation, file, metadata)
+        LOGGER.info("Start POST: /api/studies/$studyId/files/$deploymentId")
+        val ownerId = authenticationService.getId()
+
+        return fileService.create(studyId, deploymentId, ownerId, file, metadata)
     }
 
     @DeleteMapping(FILE_ID)
     @Operation(tags = ["file/delete.json"])
     @ResponseStatus(HttpStatus.NO_CONTENT)
-    @PreAuthorize("canManageStudy(#studyId) or isFileOwner(#fileId)")
+    @PreAuthorize("canManageStudy(#studyId) or @fileControllerAuthorizer.isFileOwner(#fileId)")
     fun delete(
         @PathVariable(PathVariableName.STUDY_ID) studyId: UUID,
         @PathVariable(PathVariableName.FILE_ID) fileId: Int,
     ) {
         LOGGER.info("Start DELETE: /api/studies/$studyId/files/$fileId")
-        fileService.delete(fileId)
+        fileService.delete(fileId, studyId)
     }
 
     @PostMapping(UPLOAD_IMAGE)

diff --git a/src/main/kotlin/dk/cachet/carp/webservices/file/domain/File.kt b/src/main/kotlin/dk/cachet/carp/webservices/file/domain/File.kt
@@ -18,7 +18,9 @@ data class File(
     @GeneratedValue(strategy = GenerationType.IDENTITY)
     val id: Int = 0,
     @field:NotNull
-    var storageName: String = "",
+    var fileName: String = "", // (storageName)
+    @field:NotNull
+    val relativePath: String = "", // relative path e.g. .../local/{relativePath}/{fileName}
     @field:NotNull
     var originalName: String = "",
     @JdbcTypeCode(SqlTypes.JSON)
@@ -27,4 +29,6 @@ data class File(
     var metadata: JsonNode? = null,
     @field:NotNull
     var studyId: String = "",
+    var ownerId: String? = null,
+    var deploymentId: String? = null,
 ) : Auditable()
diff --git a/src/main/kotlin/dk/cachet/carp/webservices/file/repository/FileRepository.kt b/src/main/kotlin/dk/cachet/carp/webservices/file/repository/FileRepository.kt
@@ -27,12 +27,19 @@ interface FileRepositoryCustom {
      * @param uploadedFile The [uploadedFile] in a multipart request.
      * @param fileName The [fileName] of the file.
      * @param metadata The [metadata] of the file.
+     * @param ownerId The [ownerId] of the file.
+     * @param deploymentId The [deploymentId] of the file.
+     * @param relativePath The [relativePath] of the file.
      */
+    @Suppress("LongParameterList")
     fun save(
         studyId: String,
         uploadedFile: MultipartFile,
         fileName: String,
         metadata: JsonNode?,
+        ownerId: String?,
+        deploymentId: String?,
+        relativePath: String,
     ): File
 }
 
@@ -46,19 +53,29 @@ class FileRepositoryImpl(
      * @param studyId The [studyId] of the study to save the file.
      * @param uploadedFile The [uploadedFile] to be saved to the filesystem.
      * @param fileName The [fileName] of the file.
+     * @param metadata The [metadata] of the file.
+     * @param ownerId The [ownerId] of the file.
+     * @param deploymentId The [deploymentId] of the file.
+     * @param relativePath The [relativePath] of the file.
      */
     override fun save(
         studyId: String,
         uploadedFile: MultipartFile,
         fileName: String,
         metadata: JsonNode?,
+        ownerId: String?,
+        deploymentId: String?,
+        relativePath: String,
     ): File {
         val file =
             File(
-                studyId = studyId,
-                storageName = fileName,
+                fileName = fileName,
                 originalName = uploadedFile.originalFilename!!,
                 metadata = metadata,
+                studyId = studyId,
+                ownerId = ownerId,
+                deploymentId = deploymentId,
+                relativePath = relativePath,
             )
         return fileRepository.save(file)
     }