Feature/1693 api v3 schema etc

data-model/src/main/scala/za/co/absa/enceladus/model/versionedModel/VersionedSummary.scala

@@ -16,3 +16,5 @@
 package za.co.absa.enceladus.model.versionedModel
 
 case class VersionedSummary(_id: String, latestVersion: Int)
+
+

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/SpringFoxConfig.scala

@@ -39,11 +39,18 @@ class SpringFoxConfig extends ProjectMetadata {
   }
 
   private def filteredPaths: Predicate[String] =
-    or[String](regex("/api/dataset.*"), regex("/api/schema.*"),
+    or[String](
+      // api v2
+      regex("/api/dataset.*"), regex("/api/schema.*"),
       regex("/api/mappingTable.*"), regex("/api/properties.*"),
-      regex("/api/monitoring.*"),regex("/api/runs.*"),
+      regex("/api/monitoring.*"), regex("/api/runs.*"),
       regex("/api/user.*"), regex("/api/spark.*"),
-      regex("/api/configuration.*")
+      regex("/api/configuration.*"),
+
+      // api v3
+      regex("/api-v3/datasets.*"), regex("/api-v3/schemas.*"),
+      regex("/api-v3/mapping-tables.*"), regex("/api-v3/property-definitions.*")
+
     )
 
   private def apiInfo =

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/DatasetController.scala

@@ -64,7 +64,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
       latestVersion <- datasetService.getLatestVersionValue(datasetName)
       res <- latestVersion match {
         case Some(version) => datasetService.addConformanceRule(user.getUsername, datasetName, version, rule).map {
-          case Some(ds) => ds
+          case Some((ds, validation)) => ds // v2 disregarding validation
           case _        => throw notFound()
         }
         case _ => throw notFound()
@@ -113,7 +113,7 @@ class DatasetController @Autowired()(datasetService: DatasetService)
   def replaceProperties(@AuthenticationPrincipal principal: UserDetails,
                         @PathVariable datasetName: String,
                         @RequestBody newProperties: Optional[Map[String, String]]): CompletableFuture[ResponseEntity[Option[Dataset]]] = {
-    datasetService.replaceProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map {
+    datasetService.updateProperties(principal.getUsername, datasetName, newProperties.toScalaOption).map {
       case None => throw notFound()
       case Some(dataset) =>
         val location: URI = new URI(s"/api/dataset/${dataset.name}/${dataset.version}")

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/MappingTableController.scala

@@ -40,7 +40,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ
     @RequestBody upd: MenasObject[Array[DefaultValue]]): CompletableFuture[MappingTable] = {
     mappingTableService.updateDefaults(user.getUsername, upd.id.name,
       upd.id.version, upd.value.toList).map {
-        case Some(entity) => entity
+        case Some(entity) => entity._1 // v2 disregarding validation
         case None         => throw notFound()
       }
   }
@@ -51,7 +51,7 @@ class MappingTableController @Autowired() (mappingTableService: MappingTableServ
     @RequestBody newDefault: MenasObject[DefaultValue]): CompletableFuture[MappingTable] = {
     mappingTableService.addDefault(user.getUsername, newDefault.id.name,
       newDefault.id.version, newDefault.value).map {
-        case Some(entity) => entity
+        case Some(entity) => entity._1 // v2 disregarding validation
         case None         => throw notFound()
       }
   }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RestExceptionHandler.scala

@@ -44,6 +44,11 @@ class RestExceptionHandler {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
+  @ExceptionHandler(value = Array(classOf[IllegalArgumentException]))
+  def handleIllegalArgumentException(exception: IllegalArgumentException): ResponseEntity[Any] = {
+    ResponseEntity.badRequest().body(exception.getMessage)
+  }
+
   @ExceptionHandler(value = Array(classOf[AsyncRequestTimeoutException]))
   def handleAsyncRequestTimeoutException(exception: AsyncRequestTimeoutException): ResponseEntity[Any] = {
     val message = Option(exception.getMessage).getOrElse("Request timeout expired.")

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/SchemaController.scala

@@ -146,9 +146,9 @@ class SchemaController @Autowired()(
     try {
       for {
         // the parsing of sparkStruct can fail, therefore we try to save it first before saving the attachment
-        update <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct)
+        (update, validation) <- schemaService.schemaUpload(username, menasAttachment.refName, menasAttachment.refVersion - 1, sparkStruct)
         _ <- attachmentService.uploadAttachment(menasAttachment)
-      } yield update
+      } yield Some(update) // v2 disregarding the validation; conforming to V2 Option[Entity] signature
     } catch {
       case e: SchemaParsingException => throw e.copy(schemaType = schemaType) //adding schema type
     }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/VersionedModelController.scala

@@ -17,7 +17,6 @@ package za.co.absa.enceladus.rest_api.controllers
 
 import java.util.Optional
 import java.util.concurrent.CompletableFuture
-
 import com.mongodb.client.result.UpdateResult
 import org.springframework.http.HttpStatus
 import org.springframework.security.core.annotation.AuthenticationPrincipal
@@ -29,6 +28,7 @@ import za.co.absa.enceladus.rest_api.exceptions.NotFoundException
 import za.co.absa.enceladus.rest_api.services.VersionedModelService
 import za.co.absa.enceladus.model.menas.audit._
 
+
 abstract class VersionedModelController[C <: VersionedModel with Product with Auditable[C]](versionedModelService: VersionedModelService[C])
   extends BaseController {
 
@@ -39,7 +39,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
   @GetMapping(Array("/list", "/list/{searchQuery}"))
   @ResponseStatus(HttpStatus.OK)
   def getList(@PathVariable searchQuery: Optional[String]): CompletableFuture[Seq[VersionedSummary]] = {
-    versionedModelService.getLatestVersionsSummary(searchQuery.toScalaOption)
+    versionedModelService.getLatestVersionsSummarySearch(searchQuery.toScalaOption)
   }
 
   @GetMapping(Array("/searchSuggestions"))
@@ -114,7 +114,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
   @ResponseStatus(HttpStatus.CREATED)
   def importSingleEntity(@AuthenticationPrincipal principal: UserDetails,
                          @RequestBody importObject: ExportableObject[C]): CompletableFuture[C] = {
-    versionedModelService.importSingleItem(importObject.item, principal.getUsername, importObject.metadata).map {
+    versionedModelService.importSingleItemV2(importObject.item, principal.getUsername, importObject.metadata).map {
       case Some(entity) => entity
       case None         => throw notFound()
     }
@@ -130,7 +130,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
         versionedModelService.create(item, principal.getUsername)
       }
     }.map {
-      case Some(entity) => entity
+      case Some((entity, validation)) => entity // v2 does not support validation-warnings on create
       case None         => throw notFound()
     }
   }
@@ -140,7 +140,7 @@ abstract class VersionedModelController[C <: VersionedModel with Product with Au
   def edit(@AuthenticationPrincipal user: UserDetails,
       @RequestBody item: C): CompletableFuture[C] = {
     versionedModelService.update(user.getUsername, item).map {
-      case Some(entity) => entity
+      case Some((entity, validation)) => entity // v2 disregarding validation on edit
       case None         => throw notFound()
     }
   }

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/DatasetControllerV3.scala

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.rest_api.controllers.v3
+
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.http.{HttpStatus, ResponseEntity}
+import org.springframework.security.access.prepost.PreAuthorize
+import org.springframework.security.core.annotation.AuthenticationPrincipal
+import org.springframework.security.core.userdetails.UserDetails
+import org.springframework.web.bind.annotation._
+import za.co.absa.enceladus.model.Validation
+import za.co.absa.enceladus.model.conformanceRule.ConformanceRule
+import za.co.absa.enceladus.rest_api.services.v3.DatasetServiceV3
+import za.co.absa.enceladus.rest_api.utils.implicits._
+
+import java.util.concurrent.CompletableFuture
+import javax.servlet.http.HttpServletRequest
+import scala.concurrent.ExecutionContext.Implicits.global
+
+@RestController
+@RequestMapping(path = Array("/api-v3/datasets"))
+class DatasetControllerV3 @Autowired()(datasetService: DatasetServiceV3)
+  extends VersionedModelControllerV3(datasetService) {
+
+  @GetMapping(Array("/{name}/{version}/properties"))
+  @ResponseStatus(HttpStatus.OK)
+  def getAllPropertiesForVersion(@PathVariable name: String,
+                                 @PathVariable version: String): CompletableFuture[Map[String, String]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).map {
+      case Some(entity) => entity.propertiesAsMap
+      case None => throw notFound()
+    }
+  }
+
+  @PutMapping(Array("/{name}/{version}/properties"))
+  @ResponseStatus(HttpStatus.OK)
+  @PreAuthorize("@authConstants.hasAdminRole(authentication)")
+  def updateProperties(@AuthenticationPrincipal principal: UserDetails,
+                       @PathVariable name: String,
+                       @PathVariable version: String,
+                       @RequestBody newProperties: java.util.Map[String, String],
+                       request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = {
+    forVersionExpression(name, version) { case (dsName, dsVersion) =>
+      datasetService.updateProperties(principal.getUsername, dsName, dsVersion, newProperties.toScalaMap).map {
+
+        case Some((entity, validation)) =>
+          // stripping last 3 segments (/dsName/dsVersion/properties), instead of /api-v3/dastasets/dsName/dsVersion/properties we want /api-v3/dastasets/dsName/dsVersion/properties
+          createdWithNameVersionLocationBuilder(entity.name, entity.version, request, stripLastSegments = 3, suffix = "/properties")
+            .body(validation) // todo include in tests
+        case None => throw notFound()
+      }
+    }
+  }
+
+  // todo putIntoInfoFile switch needed?
+
+  @GetMapping(Array("/{name}/{version}/rules"))
+  @ResponseStatus(HttpStatus.OK)
+  def getConformanceRules(@PathVariable name: String,
+                          @PathVariable version: String): CompletableFuture[Seq[ConformanceRule]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).map {
+      case Some(entity) => entity.conformance
+      case None => throw notFound()
+    }
+  }
+
+  @PostMapping(Array("/{name}/{version}/rules"))
+  @ResponseStatus(HttpStatus.CREATED)
+  @PreAuthorize("@authConstants.hasAdminRole(authentication)")
+  def addConformanceRule(@AuthenticationPrincipal user: UserDetails,
+                         @PathVariable name: String,
+                         @PathVariable version: String,
+                         @RequestBody rule: ConformanceRule,
+                         request: HttpServletRequest): CompletableFuture[ResponseEntity[Validation]] = {
+    forVersionExpression(name, version)(datasetService.getVersion).flatMap {
+      case Some(entity) => datasetService.addConformanceRule(user.getUsername, name, entity.version, rule).map {
+        case Some((updatedDs, validation)) =>
+          val addedRuleOrder = updatedDs.conformance.last.order
+          createdWithNameVersionLocationBuilder(name, updatedDs.version, request, stripLastSegments = 3, // strip: /{name}/{version}/rules
+            suffix = s"/rules/$addedRuleOrder").body(validation)
+        case _ => throw notFound()
+      }
+      case None => throw notFound()
+    }
+  }
+
+  @GetMapping(Array("/{name}/{version}/rules/{order}"))
+  @ResponseStatus(HttpStatus.OK)
+  def getConformanceRuleByOrder(@PathVariable name: String,
+                                @PathVariable version: String,
+                                @PathVariable order: Int): CompletableFuture[ConformanceRule] = {
+    for {
+      optDs <- forVersionExpression(name, version)(datasetService.getVersion)
+      ds = optDs.getOrElse(throw notFound())
+      rule = ds.conformance.find(_.order == order).getOrElse(throw notFound())
+    } yield rule
+  }
+
+}
+
+

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/MappingTableControllerV3.scala

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.rest_api.controllers.v3
+
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.http.{HttpStatus, ResponseEntity}
+import org.springframework.security.access.prepost.PreAuthorize
+import org.springframework.security.core.annotation.AuthenticationPrincipal
+import org.springframework.security.core.userdetails.UserDetails
+import org.springframework.web.bind.annotation._
+import za.co.absa.enceladus.model._
+import za.co.absa.enceladus.rest_api.services.v3.MappingTableServiceV3
+
+import java.util.concurrent.CompletableFuture
+import javax.servlet.http.HttpServletRequest
+
+@RestController
+@RequestMapping(Array("/api-v3/mapping-tables"))
+class MappingTableControllerV3 @Autowired()(mappingTableService: MappingTableServiceV3)
+  extends VersionedModelControllerV3(mappingTableService) {
+
+  import za.co.absa.enceladus.rest_api.utils.implicits._
+
+  import scala.concurrent.ExecutionContext.Implicits.global
+
+  @GetMapping(path = Array("/{name}/{version}/defaults"))
+  @ResponseStatus(HttpStatus.OK)
+  def getDefaults(@PathVariable name: String,
+                  @PathVariable version: String): CompletableFuture[Seq[DefaultValue]] = {
+
+    forVersionExpression(name, version)(mappingTableService.getVersion).map { // "latest" version is accepted
+      case Some(entity) => entity.defaultMappingValue
+      case None => throw notFound()
+    }
+  }
+
+  @PutMapping(path = Array("/{name}/{version}/defaults"))
+  @ResponseStatus(HttpStatus.CREATED)
+  @PreAuthorize("@authConstants.hasAdminRole(authentication)")
+  def updateDefault(@AuthenticationPrincipal user: UserDetails,
+                 @PathVariable name: String,
+                 @PathVariable version: String,
+                 @RequestBody newDefaults: Array[DefaultValue],
+                 request: HttpServletRequest
+                ): CompletableFuture[ResponseEntity[Validation]] = {
+    for {
+      existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion)
+      existingMt = existingMtOpt.getOrElse(throw notFound())
+      updatedMtAndValidationOpt <- mappingTableService.updateDefaults(user.getUsername, name, existingMt.version, newDefaults.toList)
+      (updatedMt, validation) = updatedMtAndValidationOpt.getOrElse(throw notFound())
+      response = createdWithNameVersionLocationBuilder(name, updatedMt.version, request,
+        stripLastSegments = 3, suffix = s"/defaults").body(validation)  // stripping: /{name}/{version}/defaults
+    } yield response
+  }
+
+  @PostMapping(path = Array("/{name}/{version}/defaults"))
+  @ResponseStatus(HttpStatus.CREATED)
+  @PreAuthorize("@authConstants.hasAdminRole(authentication)")
+  def addDefault(@AuthenticationPrincipal user: UserDetails,
+                 @PathVariable name: String,
+                 @PathVariable version: String,
+                 @RequestBody newDefault: DefaultValue,
+                 request: HttpServletRequest
+                ): CompletableFuture[ResponseEntity[Validation]] = {
+    for {
+      existingMtOpt <- forVersionExpression(name, version)(mappingTableService.getVersion)
+      existingMt = existingMtOpt.getOrElse(throw notFound())
+      updatedMtAndValidationOpt <- mappingTableService.addDefault(user.getUsername, name, existingMt.version, newDefault)
+      (updatedMt, validation) = updatedMtAndValidationOpt.getOrElse(throw notFound())
+      response = createdWithNameVersionLocationBuilder(name, updatedMt.version, request,
+        stripLastSegments = 3, suffix = s"/defaults").body(validation)  // stripping: /{name}/{version}/defaults
+    } yield response
+  }
+
+}

rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/PropertyDefinitionControllerV3.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.rest_api.controllers.v3
+
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.web.bind.annotation._
+import za.co.absa.enceladus.rest_api.services.PropertyDefinitionService
+
+@RestController
+@RequestMapping(path = Array("/api-v3/property-definitions/datasets"), produces = Array("application/json"))
+class PropertyDefinitionControllerV3 @Autowired()(propertyDefService: PropertyDefinitionService)
+  extends VersionedModelControllerV3(propertyDefService)
+
+// super-class implementation is sufficient