diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 0aeb698ccc..5a0f2eb36d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib import org.apache.pekko.actor.{Cancellable, Scheduler} -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime @@ -14,7 +14,7 @@ import scala.concurrent.duration._ import scala.util.control.NonFatal -abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) +abstract class BaseStore[TStoreKey, TStoreVal](bucket: S3Bucket, config: CommonConfig)(implicit ec: ExecutionContext) extends GridLogging { val s3 = new S3(config) @@ -25,15 +25,14 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key) protected def getLatestS3Stream: Option[InputStream] = { - val objects = s3.client - .listObjects(bucket).getObjectSummaries.asScala + val objects = s3.listObjects(bucket).getObjectSummaries.asScala .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION") if (objects.nonEmpty) { val obj = objects.maxBy(_.getLastModified) logger.info(s"Latest key ${obj.getKey} in bucket $bucket") - val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent + val stream = s3.getObject(bucket, obj).getObjectContent Some(stream) } else { logger.error(s"Bucket $bucket is empty") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 96c9412556..c158008cfd 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,10 +1,10 @@ package com.gu.mediaservice.lib -import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException} +import com.amazonaws.services.s3.model.MultiObjectDeleteException import java.io.File import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{Instance, MimeType, Png} import com.typesafe.scalalogging.StrictLogging @@ -21,7 +21,7 @@ object ImageIngestOperations { private def snippetForId(id: String) = id.take(6).mkString("/") + "/" + id } -class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config: CommonConfig, isVersionedS3: Boolean = false) +class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) extends S3ImageStorage(config) with StrictLogging { import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} @@ -44,7 +44,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config private def storeThumbnailImage(storableImage: StorableThumbImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage) - logger.info(s"Storing thumbnail to instance specific key: $thumbnailBucket / $instanceSpecificKey") + logger.info(s"Storing thumbnail to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey") storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) } @@ -57,26 +57,45 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config overwrite = true) } - private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { + private def bulkDelete(bucket: S3Bucket, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) - case _ => Future { - try { - logger.info(s"Creating S3 bulkDelete request for $bucket / keys: " + keys.mkString(",")) - client.deleteObjects( - new DeleteObjectsRequest(bucket).withKeys(keys: _*) - ) - keys.map { key => - key -> true - }.toMap - } catch { - case partialFailure: MultiObjectDeleteException => - logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") - val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet + case _ => + val bulkDeleteImplemented = bucket.endpoint != "storage.googleapis.com" + if (bulkDeleteImplemented) { + Future { + try { + logger.info(s"Bulk deleting S3 objects from ${bucket.bucket}: " + keys.mkString(",")) + deleteObjects(bucket, keys) + keys.map { key => + key -> true + }.toMap + } catch { + case partialFailure: MultiObjectDeleteException => + logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") + val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet + keys.map { key => + key -> !errorKeys.contains(key) + }.toMap + } + } + + } else { + Future.sequence { keys.map { key => - key -> !errorKeys.contains(key) - }.toMap + Future { + logger.info(s"Deleting S3 objects from ${bucket.bucket}: " + key) + try { + deleteObject(bucket, key) + (key, true) + } catch { + case e: Exception => + logger.debug(s"Failure when deleting images from $bucket: $key, ${e.getMessage}") + (key, false) + } + } + } + }.map(_.toMap) } - } } def deleteOriginal(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id)) @@ -87,7 +106,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config def deletePNGs(ids: Set[String])(implicit instance: Instance) = bulkDelete(imageBucket, ids.map(id => optimisedPngKeyFromId(id)).toList) def doesOriginalExist(id: String)(implicit instance: Instance): Boolean = - client.doesObjectExist(imageBucket, fileKeyFromId(id)) + doesObjectExist(imageBucket, fileKeyFromId(id)) private def instanceAwareOriginalImageKey(storableImage: StorableOriginalImage) = { fileKeyFromId(storableImage.id)(storableImage.instance) @@ -107,7 +126,7 @@ sealed trait ImageWrapper { val instance: Instance } sealed trait StorableImage extends ImageWrapper { - def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( + def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( thumbBucket, ImageIngestOperations.fileKeyFromId(id)(instance), file, @@ -119,7 +138,7 @@ sealed trait StorableImage extends ImageWrapper { case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { - override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( + override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( thumbBucket, ImageIngestOperations.fileKeyFromId(id)(instance), file, @@ -129,7 +148,7 @@ case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, las ) } case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { - override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( + override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( thumbBucket, ImageIngestOperations.optimisedPngKeyFromId(id)(instance), file, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala index 0cc3a146a0..eaf6b82979 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala @@ -2,13 +2,13 @@ package com.gu.mediaservice.lib import java.io.File import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{Instance, MimeType} import scala.concurrent.Future -class ImageQuarantineOperations(quarantineBucket: String, config: CommonConfig, isVersionedS3: Boolean = false) +class ImageQuarantineOperations(quarantineBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) extends S3ImageStorage(config) { def storeQuarantineImage(id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala index 1e57a3513a..010a3c0744 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala @@ -2,11 +2,10 @@ package com.gu.mediaservice.lib import java.util.concurrent.Executors import java.io.File - import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration._ import scala.language.postfixOps -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.MimeType @@ -37,9 +36,9 @@ trait ImageStorage { /** Store a copy of the given file and return the URI of that copy. * The file can safely be deleted afterwards. */ - def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], + def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker): Future[S3Object] - def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit] + def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit] } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 30b5d4b21c..a4d603f1a6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType @@ -14,10 +14,10 @@ import scala.concurrent.Future class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage with GridLogging { private val cacheSetting = Some(cacheForever) - def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], + def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker) = { - logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta") + logger.info(logMarker, s"storeImage to bucket: ${bucket.bucket}, id: $id, meta: $meta") val eventualObject = if (overwrite) { store(bucket, id, file, mimeType, meta, cacheSetting) } else { @@ -27,21 +27,21 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage eventualObject } - def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - client.deleteObject(bucket, id) + def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { + deleteObject(bucket, id) logger.info(logMarker, s"Deleted image $id from bucket $bucket") } - def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val objectVersion = client.getObjectMetadata(bucket, id).getVersionId - client.deleteVersion(bucket, id, objectVersion) + def deleteVersionedImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { + val objectVersion = getObjectMetadata(bucket, id).getVersionId + deleteVersion(bucket, id, objectVersion) logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } - def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val files = client.listObjects(bucket, id).getObjectSummaries.asScala + def deleteFolder(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { + val files = listObjects(bucket, id).getObjectSummaries.asScala logger.info(s"Found ${files.size} files to delete in folder $id") - files.foreach(file => client.deleteObject(bucket, file.getKey)) + files.foreach(file => deleteObject(bucket, file.getKey)) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index bbb9da0ca1..ebc43bf09c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -1,13 +1,13 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.model.Instance -import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext -class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) +class KeyStore(bucket: S3Bucket, config: CommonConfig)(implicit ec: ExecutionContext) extends BaseStore[String, ApiAccessor](bucket, config)(ec) { def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = store.get().get(instance.id + "/" + key) @@ -17,7 +17,6 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte } private def fetchAll: Map[String, ApiAccessor] = { - val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey) - keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap + s3.listObjectKeys(bucket).flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 816c644f95..07a4d77f51 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -1,5 +1,7 @@ package com.gu.mediaservice.lib.aws +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration import com.amazonaws.services.s3.model._ import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder, model} import com.amazonaws.util.IOUtils @@ -7,25 +9,21 @@ import com.amazonaws.{AmazonServiceException, ClientConfiguration} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ -import org.joda.time.{DateTime, Duration} +import org.joda.time.DateTime import java.io.File import java.net.{URI, URL} -import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ case class S3Object(uri: URI, size: Long, metadata: S3Metadata) object S3Object { - def objectUrl(bucket: String, key: String): URI = { - val bucketUrl = s"$bucket.${S3Ops.s3Endpoint}" - new URI("http", bucketUrl, s"/$key", null) - } - def apply(bucket: String, key: String, size: Long, metadata: S3Metadata): S3Object = - apply(objectUrl(bucket, key), size, metadata) + def apply(bucket: S3Bucket, key: String, size: Long, metadata: S3Metadata): S3Object = + apply(bucket.objectUrl(key), size, metadata) - def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], + def apply(bucket: S3Bucket, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { S3Object( bucket, @@ -61,40 +59,62 @@ object S3Metadata { case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[DateTime]) class S3(config: CommonConfig) extends GridLogging with ContentDisposition with RoundedExpiration { - type Bucket = String type Key = String type UserMetadata = Map[String, String] - lazy val client: AmazonS3 = S3Ops.buildS3Client(config) - - def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + val AmazonAwsS3Endpoint: String = S3.AmazonAwsS3Endpoint + def signUrl(bucket: S3Bucket, key: String, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition) - val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) - client.generatePresignedUrl(request).toExternalForm + val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) + bucket.client.generatePresignedUrl(request).toExternalForm } - def signUrlTony(bucket: Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + def signUrlTony(bucket: S3Bucket, key: String, expiration: DateTime = cachableExpiration()): URL = { + val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate) + bucket.client.generatePresignedUrl(request) + } - val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate) - client.generatePresignedUrl(request) + def copyObject(sourceBucket: S3Bucket, destinationBucket: S3Bucket, key: String): CopyObjectResult = { + // TODO check that source and destination share the same client + sourceBucket.client.copyObject(sourceBucket.bucket, key, destinationBucket.bucket, key) } - def getObject(bucket: Bucket, url: URI): model.S3Object = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) - client.getObject(new GetObjectRequest(bucket, key)) + def generatePresignedRequest(request: GeneratePresignedUrlRequest, bucket: S3Bucket): URL = { + bucket.client.generatePresignedUrl(request) } - def getObjectAsString(bucket: Bucket, key: String): Option[String] = { - val content = client.getObject(new GetObjectRequest(bucket, key)) + def deleteObject(bucket: S3Bucket, key: String): Unit = { + bucket.client.deleteObject(bucket.bucket, key) + } + + def deleteObjects(bucket: S3Bucket, keys: Seq[String]): DeleteObjectsResult = { + bucket.client.deleteObjects( + new DeleteObjectsRequest(bucket.bucket).withKeys(keys: _*) + ) + } + + def deleteVersion(bucket: S3Bucket, id: String, objectVersion: String): Unit = { + bucket.client.deleteVersion(bucket.bucket, id, objectVersion) + } + + def doesObjectExist(bucket: S3Bucket, key: String) = { + bucket.client.doesObjectExist(bucket.bucket, key) + } + + def getObject(bucket: S3Bucket, key: String): model.S3Object = { + bucket.client.getObject(new GetObjectRequest(bucket.bucket, key)) + } + + def getObject(bucket: S3Bucket, obj: S3ObjectSummary): model.S3Object = { + bucket.client.getObject(bucket.bucket, obj.getKey) + } + + def getObjectAsString(bucket: S3Bucket, key: String): Option[String] = { + val content = bucket.client.getObject(new GetObjectRequest(bucket.bucket, key)) val stream = content.getObjectContent try { Some(IOUtils.toString(stream).trim) @@ -108,7 +128,27 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def getObjectMetadata(bucket: S3Bucket, id: String): ObjectMetadata = { + bucket.client.getObjectMetadata(bucket.bucket, id) + } + + def listObjects(bucket: S3Bucket): ObjectListing = { + bucket.client.listObjects(bucket.bucket) + } + + def listObjects(bucket: S3Bucket, prefix: String): ObjectListing = { + bucket.client.listObjects(bucket.bucket, prefix) + } + + def listObjectKeys(bucket: S3Bucket): Seq[String] = { + bucket.client.listObjects(bucket.bucket).getObjectSummaries.asScala.map(_.getKey).toSeq + } + + def putObject(bucket: S3Bucket, key: String, content: String): Unit = { + bucket.client.putObject(bucket.bucket, key, content) + } + + def store(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = Future { val metadata = new ObjectMetadata @@ -117,25 +157,26 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with metadata.setUserMetadata(meta.asJava) val fileMarkers = Map( - "bucket" -> bucket, + "bucket" -> bucket.bucket, "fileName" -> id, "mimeType" -> mimeType.getOrElse("none"), ) val markers = logMarker ++ fileMarkers - val req = new PutObjectRequest(bucket, id, file).withMetadata(metadata) + val req = new PutObjectRequest(bucket.bucket, id, file).withMetadata(metadata) Stopwatch(s"S3 client.putObject ($req)"){ + val client = bucket.client client.putObject(req) // once we've completed the PUT read back to ensure that we are returning reality - val metadata = client.getObjectMetadata(bucket, id) + val metadata = client.getObjectMetadata(bucket.bucket, id) S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata)) }(markers) } - def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def storeIfNotPresent(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { Future{ - Some(client.getObjectMetadata(bucket, id)) + Some(bucket.client.getObjectMetadata(bucket.bucket, id)) }.recover { // translate this exception into the object not existing case as3e:AmazonS3Exception if as3e.getStatusCode == 404 => None @@ -148,11 +189,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def list(bucket: Bucket, prefixDir: String) + def list(bucket: S3Bucket, prefixDir: String) (implicit ex: ExecutionContext): Future[List[S3Object]] = Future { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixDir/") - val listing = client.listObjects(req) + val req = new ListObjectsRequest().withBucketName(bucket.bucket).withPrefix(s"$prefixDir/") + val listing = bucket.client.listObjects(req) val summaries = listing.getObjectSummaries.asScala summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) { case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) => @@ -160,17 +201,17 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def getMetadata(bucket: Bucket, key: Key): S3Metadata = { - val meta = client.getObjectMetadata(bucket, key) + def getMetadata(bucket: S3Bucket, key: Key): S3Metadata = { + val meta = bucket.client.getObjectMetadata(bucket.bucket, key) S3Metadata(meta) } - def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] = - client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap + def getUserMetadata(bucket: S3Bucket, key: Key): Map[String, String] = + bucket.client.getObjectMetadata(bucket.bucket, key).getUserMetadata.asScala.toMap - def syncFindKey(bucket: Bucket, prefixName: String): Option[Key] = { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixName-") - val listing = client.listObjects(req) + def syncFindKey(bucket: S3Bucket, prefixName: String): Option[Key] = { + val req = new ListObjectsRequest().withBucketName(bucket.bucket).withPrefix(s"$prefixName-") + val listing = bucket.client.listObjects(req) val summaries = listing.getObjectSummaries.asScala summaries.headOption.map(_.getKey) } @@ -178,20 +219,61 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } object S3Ops { - // TODO make this localstack friendly - // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX) - val s3Endpoint = "s3.amazonaws.com" + def buildGoogleS3Client(config: CommonConfig): Option[AmazonS3] = { + config.googleS3AccessKey.flatMap { accessKey => + config.googleS3SecretKey.map { secretKey => + val endpointConfig = new EndpointConfiguration("https://storage.googleapis.com", null) + // create credentials provider + val credentials = new BasicAWSCredentials(accessKey, secretKey) + val credentialsProvider = new AWSStaticCredentialsProvider(credentials) + // create a client config + val clientConfig = new ClientConfiguration() + + val clientBuilder = AmazonS3ClientBuilder.standard() + clientBuilder.setEndpointConfiguration(endpointConfig) + clientBuilder.withCredentials(credentialsProvider) + clientBuilder.withClientConfiguration(clientConfig) + clientBuilder.build() + } + } + } + + def buildLocalS3Client(config: CommonConfig): Option[AmazonS3] = { + config.googleS3AccessKey.flatMap { accessKey => + config.googleS3SecretKey.map { secretKey => + val endpointConfig = new EndpointConfiguration("https://minio.griddev.eelpieconsulting.co.uk", null) + // create credentials provider + val credentials = new BasicAWSCredentials(accessKey, secretKey) + val credentialsProvider = new AWSStaticCredentialsProvider(credentials) + // create a client config + val clientConfig = new ClientConfiguration() + + val clientBuilder = AmazonS3ClientBuilder.standard() + clientBuilder.setEndpointConfiguration(endpointConfig) + clientBuilder.withCredentials(credentialsProvider) + clientBuilder.withClientConfiguration(clientConfig) + clientBuilder.withPathStyleAccessEnabled(true) + clientBuilder.build() + } + } + } def buildS3Client(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = { val builder = config.awsLocalEndpoint match { - case Some(_) if config.isDev => + case Some(_) if config.isDev => { // TODO revise closer to the time of deprecation https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/ // `withPathStyleAccessEnabled` for localstack // see https://github.com/localstack/localstack/issues/1512 AmazonS3ClientBuilder.standard().withPathStyleAccessEnabled(true) + } case _ => AmazonS3ClientBuilder.standard() } config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build() } + +} + +object S3 { + val AmazonAwsS3Endpoint: String = "s3.amazonaws.com" } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala new file mode 100644 index 0000000000..9f21737558 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -0,0 +1,29 @@ +package com.gu.mediaservice.lib.aws + +import com.amazonaws.services.s3.AmazonS3 + +import java.net.URI + +case class S3Bucket(bucket: String, endpoint: String, usesPathStyleURLs: Boolean, client: AmazonS3) { + def objectUrl(key: String): URI = { + val bucketBaseURL = bucketURL() + new URI("http", bucketBaseURL.getHost, bucketBaseURL.getPath + key, null) + } + + def keyFromS3URL(url: URI): String = { + if (usesPathStyleURLs) { + url.getPath.drop(bucket.length + 2) + } else { + url.getPath.drop(1) + } + } + + def bucketURL(): URI = { + if (usesPathStyleURLs) { + new URI("https", endpoint, s"/$bucket/", null) + } else { + new URI("https", s"$bucket.$endpoint", "/", null) + } + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 83539c3fc0..48853eaf2b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -1,11 +1,11 @@ package com.gu.mediaservice.lib.config -import com.gu.mediaservice.lib.aws.{AwsClientV1BuilderUtils, AwsClientV2BuilderUtils, KinesisSenderConfig} +import com.amazonaws.services.s3.AmazonS3 +import com.gu.mediaservice.lib.aws._ import com.gu.mediaservice.model.UsageRightsSpec import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import play.api.{ConfigLoader, Configuration} -import scalaz.NonEmptyList import java.net.URI import java.util.UUID @@ -53,13 +53,43 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") + val googleS3AccessKey: Option[String] = stringOpt("s3.accessKey") + val googleS3SecretKey: Option[String] = stringOpt("s3.secretKey") + + private val amazonS3: AmazonS3 = S3Ops.buildS3Client(this) + private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(this) + private val localS3: Option[AmazonS3] = S3Ops.buildLocalS3Client(this) + + def clientFor(bucketEndpoint: String): AmazonS3 = { + (bucketEndpoint match { + case "storage.googleapis.com" => + googleS3 + case "minio.griddev.eelpieconsulting.co.uk" => + localS3 + case _ => + Some(amazonS3) + }).getOrElse { + amazonS3 + } + } + val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") - val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") - val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket") + val maybeIngestBucket: Option[S3Bucket] = for { + ingestBucket <- stringOpt("s3.ingest.bucket.name") + ingestBucketEndpoint <- stringOpt("s3.ingest.bucket.endpoint") + } yield { + S3Bucket(ingestBucket, ingestBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.ingest.bucket.pathStyleURLs").getOrElse(false), clientFor(ingestBucketEndpoint)) + } + val maybeFailBucket: Option[S3Bucket] = for { + failBucket <- stringOpt("s3.fail.bucket.name") + failBucketEndpoint <- stringOpt("s3.fail.bucket.endpoint") + } yield { + S3Bucket(failBucket, failBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.fail.bucket.pathStyleURLs").getOrElse(false), clientFor(failBucketEndpoint)) + } - val maybeQuarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket") + val maybeQuarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket.name").map(S3Bucket(_, S3.AmazonAwsS3Endpoint, booleanOpt("s3.quarantine.bucket.pathStyleURLs").getOrElse(false), clientFor(S3.AmazonAwsS3Endpoint))) - val maybeBucketForUIUploads: Option[String] = maybeQuarantineBucket orElse maybeIngestBucket + val maybeBucketForUIUploads: Option[S3Bucket] = maybeQuarantineBucket orElse maybeIngestBucket val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024) @@ -74,6 +104,11 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B val services = new SingleHostServices(domainRoot) + private val imageBucketEndpoint = string("s3.image.bucket.endpoint") + val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), imageBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.image.bucket.pathStyleURLs").getOrElse(false), clientFor(imageBucketEndpoint)) + private val thumbBucketEndpoint = string("s3.thumb.bucket.endpoint") + val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket.name"), thumbBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.thumb.bucket.pathStyleURLs").getOrElse(false), clientFor(thumbBucketEndpoint)) + /** * Load in a list of domain metadata specifications from configuration. For example: * {{{ diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf index 05d1db98e6..b5c3e1aee2 100644 --- a/common-lib/src/test/resources/application.conf +++ b/common-lib/src/test/resources/application.conf @@ -61,3 +61,10 @@ instance.service.my="" instance.service.instances="" usageEvents.queue.name="" + +s3.image.bucket.name="images" +s3.image.bucket.endpoint="some-providers-s3-endpoint" + +s3.thumb.bucket.name="thumbs" +s3.thumb.bucket.endpoint="some-providers-s3-endpoint" + diff --git a/cropper/app/CropperComponents.scala b/cropper/app/CropperComponents.scala index 509192dedc..d570859ade 100644 --- a/cropper/app/CropperComponents.scala +++ b/cropper/app/CropperComponents.scala @@ -1,4 +1,5 @@ import com.gu.mediaservice.GridClient +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.management.Management import com.gu.mediaservice.lib.play.GridComponents @@ -12,8 +13,9 @@ class CropperComponents(context: Context) extends GridComponents(context, new Cr val store = new CropStore(config) val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) + val s3 = new S3(config) - val crops = new Crops(config, store, imageOperations, config.imageBucket) + val crops = new Crops(config, store, imageOperations, config.imageBucket, s3) val notifications = new Notifications(config) private val gridClient = GridClient(config.services)(wsClient) diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala index 1cc815d2cb..034790420f 100644 --- a/cropper/app/lib/CropStore.scala +++ b/cropper/app/lib/CropStore.scala @@ -49,9 +49,11 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS date = userMetadata.get("date").flatMap(parseDateTime) dimensions = Dimensions(width, height) + key = config.imgPublishingBucket.keyFromS3URL(s3Object.uri) + sizing = Asset( - signedCropAssetUrl(s3Object.uri), + signedCropAssetUrl(key), Some(s3Object.size), objectMetadata.contentType, Some(dimensions), @@ -78,12 +80,12 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS def translateImgHost(uri: URI): URI = new URI("https", config.imgPublishingHost, uri.getPath, uri.getFragment) - private def folderForImagesCrops(id: Bucket, instance: Instance) = { + private def folderForImagesCrops(id: String, instance: Instance) = { instance.id + "/" + id } - private def signedCropAssetUrl(uri: URI): URI = { - signUrlTony(config.imgPublishingBucket, uri).toURI + private def signedCropAssetUrl(key: String): URI = { + signUrlTony(config.imgPublishingBucket, key).toURI } } diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala index 96d043d60a..5e082585bb 100644 --- a/cropper/app/lib/CropperConfig.scala +++ b/cropper/app/lib/CropperConfig.scala @@ -1,5 +1,6 @@ package lib +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.model.Instance @@ -7,10 +8,13 @@ import java.io.File class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) { - val imageBucket: String = string("s3.image.bucket") - - val imgPublishingBucket = string("publishing.image.bucket") - + // TODO this is common with media-api download exports + val imgPublishingBucket: S3Bucket = S3Bucket( + string("publishing.image.bucket.name"), + string("publishing.image.bucket.endpoint"), + boolean("publishing.image.bucket.pathStyleURLs"), + clientFor(string("publishing.image.bucket.endpoint")) + ) val canDownloadCrop: Boolean = boolean("canDownloadCrop") val imgPublishingHost = string("publishing.image.host") diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala index 6c2be74d21..d7b3a7eda7 100644 --- a/cropper/app/lib/Crops.scala +++ b/cropper/app/lib/Crops.scala @@ -3,7 +3,7 @@ package lib import java.io.File import com.gu.mediaservice.lib.metadata.FileMetadataHelper import com.gu.mediaservice.lib.Files -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.imaging.{ExportResult, ImageOperations} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ @@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float) -class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: String)(implicit ec: ExecutionContext) extends GridLogging { +class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { import Files._ private val cropQuality = 75d @@ -26,8 +26,6 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera // We don't overly care about output crop file sizes here, but prefer a fast output, so turn it right down. private val pngCropQuality = 1d - private val s3 = new S3(config) - def outputFilename(source: SourceImage, bounds: Bounds, outputWidth: Int, fileType: MimeType, isMaster: Boolean = false)(implicit instance: Instance): String = { val masterString: String = if (isMaster) "master/" else "" instance.id + "/" + s"${source.id}/${Crop.getCropId(bounds)}/$masterString$outputWidth${fileType.fileExtension}" @@ -116,7 +114,8 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera val hasAlpha = apiImage.fileMetadata.colourModelInformation.get("hasAlpha").flatMap(a => Try(a.toBoolean).toOption).getOrElse(true) val cropType = Crops.cropType(mimeType, colourType, hasAlpha) - val secureUrl = s3.signUrlTony(imageBucket, secureFile) + val key = imageBucket.keyFromS3URL(secureFile) + val secureUrl = s3.signUrlTony(imageBucket, key) Stopwatch.async(s"making crop assets for ${apiImage.id} ${Crop.getCropId(source.bounds)}") { for { diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala index e348b9a4a6..83fdeb8590 100644 --- a/cropper/test/lib/CropsTest.scala +++ b/cropper/test/lib/CropsTest.scala @@ -1,7 +1,10 @@ package lib +import com.amazonaws.services.s3.AmazonS3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.model._ +import org.mockito.Mockito.when import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatestplus.mockito.MockitoSugar @@ -44,31 +47,39 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar { Crops.cropType(Tiff, "TrueColor", hasAlpha = false) shouldBe Jpeg } - private val config = mock[CropperConfig] + private val config = { + val mockConfig = mock[CropperConfig] + when(mockConfig.awsRegion).thenReturn("eu-west-1") + when(mockConfig.googleS3AccessKey).thenReturn(None) + when(mockConfig.googleS3SecretKey).thenReturn(None) + mockConfig + } private val store = mock[CropStore] private val imageOperations: ImageOperations = mock[ImageOperations] private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata]) private val bounds: Bounds = Bounds(10, 20, 30, 40) private val outputWidth = 1234 - private val imageBucket = "crops-bucket" + private val mockS3Client = mock[AmazonS3] + private val imageBucket = S3Bucket("crops-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client) + private val s3 = new S3(config) it("should should construct a correct address for a master jpg") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Jpeg, isMaster = true) outputFilename shouldBe "an-instance/test/10_20_30_40/master/1234.jpg" } it("should should construct a correct address for a non-master jpg") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Jpeg) outputFilename shouldBe "an-instance/test/10_20_30_40/1234.jpg" } it("should should construct a correct address for a non-master tiff") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Tiff) outputFilename shouldBe "an-instance/test/10_20_30_40/1234.tiff" } it("should should construct a correct address for a non-master png") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Png) outputFilename shouldBe "an-instance/test/10_20_30_40/1234.png" } diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 24bc85d0e7..6f8b255498 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,5 +1,6 @@ import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3Vectors, SimpleSqsMessageConsumer} +import com.gu.mediaservice.lib.aws.{Bedrock, S3, S3Vectors, SimpleSqsMessageConsumer, Embedder} import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.lib.play.GridComponents @@ -35,7 +36,8 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne } val uploader = new Uploader(store, config, imageOperations, notifications, maybeEmbedder, imageProcessor, gridClient, auth) - val projector = Projector(config, imageOperations, imageProcessor, auth, maybeEmbedder) + val s3 = new S3(config) + val projector = Projector(config, imageOperations, imageProcessor, auth, maybeEmbedder, s3) val quarantineUploader: Option[QuarantineUploader] = config.maybeQuarantineBucket.map(_ => new QuarantineUploader(new QuarantineStore(config), config) ) diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index a350a200b2..e2c546da4c 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -141,8 +141,6 @@ class ImageLoaderController(auth: Authentication, } private def handleMessageFromIngestBucket(sqsMessage: SQSMessage)(basicLogMarker: LogMarker): Future[Unit] = { - logger.info(basicLogMarker, sqsMessage.toString) - extractS3KeyFromSqsMessage(sqsMessage) match { case Failure(exception) => metrics.failedIngestsFromQueue.increment() diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index 37d43aa078..6e8bfde3a6 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -1,5 +1,7 @@ package lib +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} + import java.io.File import com.gu.mediaservice.lib.cleanup.{ComposedImageProcessor, ImageProcessor, ImageProcessorResources} import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources, ImageProcessorLoader} @@ -10,11 +12,13 @@ import play.api.inject.ApplicationLifecycle import scala.concurrent.duration.FiniteDuration class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging { - val imageBucket: String = string("s3.image.bucket") val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket") - val thumbnailBucket: String = string("s3.thumb.bucket") + private val quarantineBucketEndpoint = S3.AmazonAwsS3Endpoint + val quarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket").map { bucket => + S3Bucket(bucket, quarantineBucketEndpoint, usesPathStyleURLs = false, clientFor(quarantineBucketEndpoint)) + } val lowerEnvironmentSamplingPercentageAsDecimal = intOpt("s3.sampling.percentage").getOrElse(1) / 100.0 val maybeLowerEnvironmentQueueBucketToSampleInto = stringOpt("s3.sampling.targetBucket") diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index e2355ff0b0..0f7975399b 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -31,7 +31,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) { - client.getObject(config.maybeIngestBucket.get, key) + getObject(config.maybeIngestBucket.get, key) } { logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.") } @@ -42,13 +42,13 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati s"${instance.id}/$uploader/$filename", file, mimeType = None, // we don't care as this is just the queue bucket - meta = s3Meta, + meta = s3Meta ) } def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String)(implicit instance: Instance): String = { val request = new GeneratePresignedUrlRequest( - config.maybeBucketForUIUploads.get, // bucket + config.maybeBucketForUIUploads.get.bucket, // bucket s"${instance.id}/$uploadedBy/$filename", // key ) .withMethod(HttpMethod.PUT) @@ -57,18 +57,18 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati // sent by the client in manager.js request.putCustomRequestHeader("x-amz-meta-media-id", mediaId) - client.generatePresignedUrl(request).toString + generatePresignedRequest(request, config.maybeIngestBucket.get).toString } def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){ - client.copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key) // TODO Naked get - make optional + copyObject(config.maybeIngestBucket.get, config.maybeFailBucket.get, key) // TODO Naked get - make optional deleteObjectFromIngestBucket(key) } { logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.") } def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) { - client.deleteObject(config.maybeIngestBucket.get,key) + deleteObject(config.maybeIngestBucket.get, key) } { logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.") } diff --git a/image-loader/app/lib/QuarantineStore.scala b/image-loader/app/lib/QuarantineStore.scala index 4fc8581672..0c7e5f515f 100644 --- a/image-loader/app/lib/QuarantineStore.scala +++ b/image-loader/app/lib/QuarantineStore.scala @@ -3,4 +3,4 @@ package lib.storage import lib.ImageLoaderConfig import com.gu.mediaservice.lib -class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.maybeQuarantineBucket.get, config) +class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.quarantineBucket.get, config) diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala index f81b9ce3a7..40b9ad413e 100644 --- a/image-loader/app/lib/imaging/FileMetadataReader.scala +++ b/image-loader/app/lib/imaging/FileMetadataReader.scala @@ -125,7 +125,7 @@ object FileMetadataReader extends GridLogging { val redactionReplacementValue = s"REDACTED (value longer than $redactionThreshold characters, please refer to the metadata stored in the file itself)" private def redactLongFieldValues(imageId: String, metadataType: String, exceptions: List[String] = Nil)(props: Map[String, String]) = props.map { case (fieldName, value) if value.length > redactionThreshold && !exceptions.exists(fieldName.contains) => - logger.warn(s"Redacting '$fieldName' $metadataType field for image $imageId, as it's problematically long (longer than $redactionThreshold characters") + logger.debug(s"Redacting '$fieldName' $metadataType field for image $imageId, as it's problematically long (longer than $redactionThreshold characters") fieldName -> redactionReplacementValue case keyValuePair => keyValuePair } diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index d1d5ac1e30..82bdf3ad09 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -1,39 +1,35 @@ package model -import java.io.{File, FileOutputStream} -import com.amazonaws.services.s3.AmazonS3 -import com.gu.mediaservice.{GridClient, ImageDataMerger} -import com.gu.mediaservice.lib.auth.Authentication -import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} +import com.amazonaws.services.s3.model.{ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} -import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Ops} +import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3, S3Bucket} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.lib.net.URI +import com.gu.mediaservice.lib._ import com.gu.mediaservice.model.{Image, Instance, MimeType, UploadInfo} +import com.gu.mediaservice.{GridClient, ImageDataMerger} import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3} import lib.{DigestedFile, ImageLoaderConfig} import model.upload.UploadRequest import org.apache.commons.io.IOUtils import org.joda.time.{DateTime, DateTimeZone} -import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse -import play.api.mvc.RequestHeader +import _root_.play.api.libs.ws.WSRequest -import java.nio.file.Path -import scala.jdk.CollectionConverters._ +import java.io.{File, FileOutputStream} import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} +import scala.jdk.CollectionConverters._ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbedder) + def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder], s3: S3)(implicit ec: ExecutionContext): Projector + = new Projector(toImageUploadOpsCfg(config), s3, imageOps, processor, auth, maybeEmbedder) } case class S3FileExtractedMetadata( @@ -86,7 +82,7 @@ object S3FileExtractedMetadata { } class Projector(config: ImageUploadOpsCfg, - s3: AmazonS3, + s3: S3, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, @@ -101,7 +97,7 @@ class Projector(config: ImageUploadOpsCfg, val s3Key = fileKeyFromId(imageId) if (!s3.doesObjectExist(config.originalFileBucket, s3Key)) - throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key) + throw new NoSuchImageExistsInS3(config.originalFileBucket.bucket, s3Key) val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){ s3.getObject(config.originalFileBucket, s3Key) @@ -165,11 +161,11 @@ class Projector(config: ImageUploadOpsCfg, class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, - s3: AmazonS3, + s3: S3, maybeEmbedder: Option[Embedder], ) extends GridLogging { - import Uploader.{fromUploadRequestShared, toMetaMap} + import Uploader.fromUploadRequestShared def projectImageFromUploadRequest(uploadRequest: UploadRequest) @@ -211,7 +207,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, } private def fetchFile( - bucket: String, key: String, outFile: File + bucket: S3Bucket, key: String, outFile: File )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = { logger.info(logMarker, s"Trying fetch existing image from S3 bucket - $bucket at key $key") val doesFileExist = Future { s3.doesObjectExist(bucket, key) } recover { case _ => false } @@ -220,7 +216,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key") Future.successful(None) // falls back to creating from original file case true => - val obj = s3.getObject(new GetObjectRequest(bucket, key)) + val obj = s3.getObject(bucket, key) val fos = new FileOutputStream(outFile) try { IOUtils.copy(obj.getObjectContent, fos) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index af5acf651d..7b22de8b41 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -8,6 +8,8 @@ import com.gu.mediaservice.lib._ import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, UpdateMessage} +import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} +import com.gu.mediaservice.lib.aws.{Embedder, S3Bucket, S3Object, S3Vectors, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -66,8 +68,8 @@ case class ImageUploadOpsCfg( thumbWidth: Int, thumbQuality: Double, transcodedMimeTypes: List[MimeType], - originalFileBucket: String, - thumbBucket: String + originalFileBucket: S3Bucket, + thumbBucket: S3Bucket, ) case class ImageUploadOpsDependencies( @@ -94,7 +96,7 @@ object Uploader extends GridLogging { config.thumbQuality, config.transcodedMimeTypes, config.imageBucket, - config.thumbnailBucket + config.thumbnailBucket, ) } @@ -472,6 +474,7 @@ class Uploader( )) // TODO: centralise where all these URLs are constructed } yield { + /* config.maybeLowerEnvironmentQueueBucketToSampleInto.foreach { lowerEnvironmentQueueBucket => if (math.random() < config.lowerEnvironmentSamplingPercentageAsDecimal) { val mediaId = imageUpload.image.id @@ -487,7 +490,7 @@ class Uploader( } } } - + */ UploadStatusUri(s"${config.rootUri(instance)}/uploadStatus/${uploadRequest.imageId}") } diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 9711519a0c..0d894a90fe 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -1,7 +1,8 @@ package model +import com.amazonaws.services.s3.AmazonS3 import com.drew.imaging.ImageProcessingException -import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata} +import com.gu.mediaservice.lib.aws.{S3, S3Bucket, S3Object} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.LogMarker @@ -17,7 +18,6 @@ import org.scalatestplus.mockito.MockitoSugar import test.lib.ResourceHelpers import java.io.File -import java.net.URI import java.util.UUID import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -29,10 +29,12 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { override def markerContents: Map[String, Any] = Map() } + private val mockS3Client = mock[AmazonS3] + private implicit val logMarker: MockLogMarker = new MockLogMarker() // For mime type info, see https://github.com/guardian/grid/pull/2568 val tempDir = new File("/tmp") - val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket") + val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client)) /** * @todo: I flailed about until I found a path that worked, but @@ -48,12 +50,9 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { val randomId = UUID.randomUUID().toString + fileName - val mockS3Meta = S3Metadata(Map.empty, S3ObjectMetadata(None, None, None)) - val mockS3Object = S3Object(new URI("innernets.com"), 12345, mockS3Meta) - def mockStore = (a: StorableImage) => Future.successful( - S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None) + S3Object(S3Bucket("madeupname", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), "madeupkey", a.file, Some(a.mimeType), None, a.meta, None) ) def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index 3c5d384863..e7eceb3fb0 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -1,12 +1,15 @@ package model +import com.amazonaws.services.s3.AmazonS3 + import java.io.File import java.net.URI -import java.util.{Date, UUID} -import com.amazonaws.services.s3.AmazonS3 +import java.util.Date import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Bucket, S3Vectors} +import com.gu.mediaservice.lib.aws.S3Ops import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} @@ -23,7 +26,6 @@ import org.scalatest.time.{Millis, Span} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsString} import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse -import play.api.mvc.RequestHeader import test.lib.ResourceHelpers import java.nio.file.Path @@ -41,11 +43,12 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val imageOperations = new ImageOperations(ctxPath) - private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket") + private val mockS3Client = mock[AmazonS3] + private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client)) private val maybeEmbedder = None - private val s3 = mock[AmazonS3] + private val s3 = mock[S3] private val auth = mock[Authentication] private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder) diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala index 525423ed05..8be882a61a 100644 --- a/kahuna/app/lib/KahunaConfig.scala +++ b/kahuna/app/lib/KahunaConfig.scala @@ -2,6 +2,7 @@ package lib import com.gu.mediaservice.lib.auth.Permissions.Pinboard import com.gu.mediaservice.lib.auth.SimplePermission +import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.model.Instance import play.api.libs.json._ @@ -49,9 +50,8 @@ class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resource val showSendToPhotoSales: Option[Boolean] = booleanOpt("showSendToPhotoSales") val frameAncestors: Set[String] = getStringSet("security.frameAncestors") - val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeBucketForUIUploads.map { bucket => - if (isDev) "https://localstack.media.local.dev-gutools.co.uk" - else s"https://$bucket.s3.$awsRegion.amazonaws.com" + val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeIngestBucket.map { ingestBucket => + ingestBucket.bucketURL().toURL.toExternalForm } ++ telemetryUri val fontSources: Set[String] = getStringSet("security.fontSources") val imageSources: Set[String] = getStringSet("security.imageSources") diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index b06b6fea5c..0a57532819 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -1,4 +1,4 @@ -import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3Vectors, ThrallMessageSender} +import com.gu.mediaservice.lib.aws._ import com.gu.mediaservice.lib.instances.InstancesClient import com.gu.mediaservice.lib.aws._ import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, Management} @@ -18,7 +18,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val messageSender = new ThrallMessageSender(config.thrallKinesisStreamConfig) val mediaApiMetrics = new MediaApiMetrics(config, actorSystem, applicationLifecycle) - val s3Client = new S3(config) + val s3 = new S3(config) val usageQuota = new UsageQuota(config, actorSystem.scheduler) usageQuota.quotaStore.update() @@ -28,12 +28,12 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val elasticSearch = new ElasticSearch(config, mediaApiMetrics, config.esConfig, () => usageQuota.usageStore.overQuotaAgencies, actorSystem.scheduler, new InstancesClient(config, wsClient)) // TODO needs to move somewhere more instance aware elasticSearch.ensureIndexExistsAndAliasAssigned() - val imageResponse = new ImageResponse(config, s3Client, usageQuota) + val imageResponse = new ImageResponse(config, s3, usageQuota) val softDeletedMetadataTable = new SoftDeletedMetadataTable(config) val embedder = new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(config.queueUrl, config)) - val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder, usageEvents) + val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3, mediaApiMetrics, wsClient, authorisation, embedder, usageEvents) val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents) val aggController = new AggregationController(auth, elasticSearch, controllerComponents) val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 3f268a1dd8..8509c07c79 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -9,6 +9,10 @@ import com.gu.mediaservice.lib.auth.Permissions.{ArchiveImages, DeleteCropsOrUsa import com.gu.mediaservice.lib.auth._ import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider import com.gu.mediaservice.lib.aws._ +import com.gu.mediaservice.lib.aws.{ContentDisposition, Embedder, ThrallMessageSender, UpdateMessage} +import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, ThrallMessageSender, UpdateMessage} +import com.gu.mediaservice.lib.aws.{ContentDisposition, ThrallMessageSender, UpdateMessage} +import com.gu.mediaservice.lib.aws.{ContentDisposition, S3, ThrallMessageSender, UpdateMessage} import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.lib.formatting.printDateTime @@ -42,13 +46,13 @@ class MediaApi( imageResponse: ImageResponse, config: MediaApiConfig, override val controllerComponents: ControllerComponents, - s3Client: S3, + s3: S3, mediaApiMetrics: MediaApiMetrics, ws: WSClient, authorisation: Authorisation, embedder: Embedder, events: UsageEvents, - )(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest { +)(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest { private val gridClient: GridClient = GridClient(config.services)(ws) @@ -321,7 +325,8 @@ class MediaApi( val maybeResult = for { export <- source.exports.find(_.id.contains(exportId)) asset <- export.assets.find(_.dimensions.exists(_.width == width)) - s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, asset.file)).toOption + key = config.imgPublishingBucket.keyFromS3URL(asset.file) + s3Object <- Try(s3.getObject(config.imgPublishingBucket, key)).toOption file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name)) result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename)) @@ -450,7 +455,8 @@ class MediaApi( val apiKey = request.user.accessor logger.info(logMarker, s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType) - val s3Object = s3Client.getObject(config.imageBucket, image.source.file) + val key = config.imageBucket.keyFromS3URL(image.source.file) + val s3Object = s3.getObject(config.imageBucket, key) val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name)) @@ -513,8 +519,9 @@ class MediaApi( logger.info(logMarker, s"Download optimised image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OptimisedDownloadType) + val key = config.imageBucket.keyFromS3URL(image.optimisedPng.getOrElse(image.source).file) val sourceImageUri = - new URI(s3Client.signUrl(config.imageBucket, image.optimisedPng.getOrElse(image.source).file, image, imageType = image.optimisedPng match { + new URI(s3.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match { case Some(_) => OptimisedPng case _ => Source })) diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala index a8cd9df3bd..a7ab167770 100644 --- a/media-api/app/lib/ImageResponse.scala +++ b/media-api/app/lib/ImageResponse.scala @@ -12,12 +12,10 @@ import lib.ImageResponse.extractAliasFieldValues import lib.elasticsearch.SourceWrapper import lib.usagerights.CostCalculator import org.apache.commons.codec.binary.Base64 -import org.joda.time.DateTime -import play.api.libs.functional.syntax._ import play.api.libs.json._ import play.utils.UriEncoding -import java.net.{URI, URLEncoder} +import java.net.URI import scala.annotation.tailrec import scala.util.{Failure, Try} @@ -79,11 +77,12 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3, usageQuota: UsageQuota val fileUri = image.source.file - val imageUrl = s3Client.signUrl(config.imageBucket, fileUri, image, imageType = Source) + val key = config.imageBucket.keyFromS3URL(fileUri) + val imageUrl = s3Client.signUrl(config.imageBucket, key, image, imageType = Source) val pngUrl: Option[String] = pngFileUri - .map(s3Client.signUrl(config.imageBucket, _, image, imageType = OptimisedPng)) + .map(uri => s3Client.signUrl(config.imageBucket, config.imageBucket.keyFromS3URL(uri), image, imageType = OptimisedPng)) - def s3SignedThumbUrl = s3Client.signUrl(config.thumbBucket, fileUri, image, imageType = Thumbnail) + def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, key, image, imageType = Thumbnail) val thumbUrl = config.cloudFrontDomainThumbBucket .map(domain => s"https://$domain${fileUri.getPath}") diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 6258b1faf8..b506508fba 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -1,5 +1,6 @@ package lib +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources} import com.gu.mediaservice.lib.elasticsearch.filters import com.sksamuel.elastic4s.ElasticApi.{matchPhraseQuery, should} @@ -14,25 +15,30 @@ import scala.collection.immutable import scala.util.Try case class StoreConfig( - storeBucket: String, - storeKey: String + storeBucket: S3Bucket, + storeKey: String, ) class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val configBucket: String = string("s3.config.bucket") - val usageMailBucket: String = string("s3.usagemail.bucket") + private val configBucketEndpoint = S3.AmazonAwsS3Endpoint + val configBucket: S3Bucket = S3Bucket(string("s3.config.bucket"), configBucketEndpoint, usesPathStyleURLs = false, clientFor(configBucketEndpoint)) + private val usageMailBucketEndpoint = S3.AmazonAwsS3Endpoint + val usageMailBucket: S3Bucket = S3Bucket(string("s3.usagemail.bucket"), S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, clientFor(usageMailBucketEndpoint)) val quotaStoreKey: String = string("quota.store.key") val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey) - //Lazy allows this to be empty and not break things unless used somewhere - lazy val imgPublishingBucket = string("publishing.image.bucket") - - val imageBucket: String = string("s3.image.bucket") - val thumbBucket: String = string("s3.thumb.bucket") + // TODO this needs to be the same as the crops bucket? Is downloadExports broken? + // TODO can be optional at BBC? + val imgPublishingBucket: S3Bucket = S3Bucket( + string("publishing.image.bucket.name"), + string("publishing.image.bucket.endpoint"), + boolean("publishing.image.bucket.pathStyleURLs"), + clientFor(string("publishing.image.bucket.endpoint")) + ) val cloudFrontDomainThumbBucket: Option[String] = stringOpt("cloudfront.domain.thumbbucket") - val cloudFrontPrivateKeyBucket: Option[String] = stringOpt("cloudfront.private-key.bucket") + val cloudFrontPrivateKeyBucket: Option[S3Bucket] = stringOpt("cloudfront.private-key.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, clientFor(S3.AmazonAwsS3Endpoint))) val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key") val cloudFrontKeyPairId: Option[String] = stringOpt("cloudfront.keypair.id") diff --git a/media-api/app/lib/QuotaStore.scala b/media-api/app/lib/QuotaStore.scala index c79486fbd1..1606586554 100644 --- a/media-api/app/lib/QuotaStore.scala +++ b/media-api/app/lib/QuotaStore.scala @@ -1,14 +1,15 @@ package lib import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3Bucket import play.api.libs.json.Json import scala.concurrent.ExecutionContext class QuotaStore( quotaFile: String, - bucket: String, - config: MediaApiConfig + bucket: S3Bucket, + config: MediaApiConfig, )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config)(ec) { def getQuota: Map[String, SupplierUsageQuota] = store.get() diff --git a/media-api/app/lib/UsageQuota.scala b/media-api/app/lib/UsageQuota.scala index bcd4e93e21..0dc049fc77 100644 --- a/media-api/app/lib/UsageQuota.scala +++ b/media-api/app/lib/UsageQuota.scala @@ -2,6 +2,7 @@ package lib import org.apache.pekko.actor.Scheduler import com.gu.mediaservice.lib.FeatureToggle +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.model.UsageRights import scala.concurrent.Await @@ -16,13 +17,13 @@ class UsageQuota(config: MediaApiConfig, scheduler: Scheduler) { val quotaStore = new QuotaStore( config.quotaStoreConfig.storeKey, config.quotaStoreConfig.storeBucket, - config + config, ) val usageStore = new UsageStore( config.usageMailBucket, config, - quotaStore + quotaStore, ) def scheduleUpdates(): Unit = { diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala index 2c19044131..c8b55defa0 100644 --- a/media-api/app/lib/UsageStore.scala +++ b/media-api/app/lib/UsageStore.scala @@ -1,6 +1,7 @@ package lib import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{Agencies, Agency, UsageRights} import org.joda.time.DateTime @@ -57,9 +58,9 @@ object UsageStore extends GridLogging { } class UsageStore( - bucket: String, + bucket: S3Bucket, config: MediaApiConfig, - quotaStore: QuotaStore + quotaStore: QuotaStore, )(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config) with GridLogging { import UsageStore._ diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index d38c28d497..d6e23a8e0d 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -35,13 +35,17 @@ trait Fixtures { "es.index.aliases.current", "es.index.aliases.migration", "es6.url", - "s3.image.bucket", - "s3.thumb.bucket", + "s3.image.bucket.name", + "s3.image.bucket.endpoint", + "s3.thumb.bucket.name", + "s3.thumb.bucket.endpoint", "grid.stage", "grid.appName", "instance.service.my", "instance.service.instances", - "usageEvents.queue.name" + "usageEvents.queue.name", + "publishing.image.bucket.name", + "publishing.image.bucket.endpoint" ) def deletionData(deletedBy: String): SoftDeletedMetadata = SoftDeletedMetadata( diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala index e42c25c718..c7b0c0f7b8 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala @@ -1,9 +1,9 @@ package com.gu.mediaservice.lib.auth.provider import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, Principal} -import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider.{ApiKeyInstance, KindeIdKey} +import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider.ApiKeyInstance import com.gu.mediaservice.lib.auth.{ApiAccessor, KeyStore} +import com.gu.mediaservice.lib.aws.{S3, S3Bucket, S3Ops} import com.gu.mediaservice.lib.config.InstanceForRequest -import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.model.Instance import com.typesafe.scalalogging.StrictLogging import play.api.Configuration @@ -25,7 +25,10 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth var keyStorePlaceholder: Option[KeyStore] = _ override def initialise(): Unit = { - val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig) + val authBucketEndPoint = S3.AmazonAwsS3Endpoint + val authBucketS3Client = S3Ops.buildS3Client(resources.commonConfig) + val authKeyStoreBucket = S3Bucket(configuration.get[String]("authKeyStoreBucket"), authBucketEndPoint, usesPathStyleURLs = false, authBucketS3Client) + val store = new KeyStore(authKeyStoreBucket, resources.commonConfig) store.scheduleUpdates(resources.actorSystem.scheduler) keyStorePlaceholder = Some(store) } diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf index 930c6a00d5..7e55f72bb2 100644 --- a/rest-lib/src/test/resources/application.conf +++ b/rest-lib/src/test/resources/application.conf @@ -3,3 +3,7 @@ grid.appName: "test" thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" +s3.image.bucket.name: images +s3.image.bucket.endpoint: some-providers-s3-endpoint +s3.thumb.bucket.name: thumbs +s3.thumb.bucket.endpoint: some-providers-s3-endpoint diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala index 3ba9231943..8fcc8ef54c 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala @@ -1,8 +1,10 @@ package com.gu.mediaservice.lib.auth +import com.amazonaws.services.s3.AmazonS3 import org.apache.pekko.actor.ActorSystem import com.gu.mediaservice.lib.auth.Authentication.MachinePrincipal import com.gu.mediaservice.lib.auth.provider.{ApiKeyAuthenticationProvider, Authenticated, AuthenticationProviderResources, Invalid, NotAuthenticated, NotAuthorised} +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.model.Instance @@ -43,7 +45,8 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with Future.successful(()) } - override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig) { + val mockS3Client: AmazonS3 = mock[AmazonS3] + override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), resources.commonConfig) { override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = { key match { case "key-chuckle" => Some(ApiAccessor("brothers", Internal)) diff --git a/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala b/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala index da73d03db6..c6ad7b03f8 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/lib/JsonValueCodecJsValue.scala @@ -98,6 +98,7 @@ object JsonValueCodecJsValue { case JsNull => out.writeNull() case _ => + out.writeNull() } } diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index fb8a121b42..e0a615fd82 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -127,7 +127,7 @@ class ReaperController( case Some(reaperBucket) => val now = DateTime.now(DateTimeZone.UTC) val key = s"$deleteType/${s3DirNameFromDate(now)}/$deleteType-${now.toString()}.json" - store.client.putObject(reaperBucket, key, json.toString()) + store.putObject(reaperBucket, key, json.toString()) json } } @@ -213,8 +213,8 @@ class ReaperController( case (Some(reaperBucket), Some(countOfImagesToReap)) => val recentRecords = List(now, now.minusDays(1), now.minusDays(2)).flatMap { day => val s3DirName = s3DirNameFromDate(day) - store.client.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++ - store.client.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList + store.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++ + store.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList } val recentRecordKeys = recentRecords @@ -229,9 +229,11 @@ class ReaperController( def reaperRecord(key: String) = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - Ok( - store.client.getObjectAsString(reaperBucket, key) - ).as(JSON) + store.getObjectAsString(reaperBucket, key).map { record => + Ok(record).as(JSON) + }.getOrElse{ + NotFound + } }} def conf() = Action.async { diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala index 25743ddf67..c6a816136d 100644 --- a/thrall/app/lib/ThrallConfig.scala +++ b/thrall/app/lib/ThrallConfig.scala @@ -1,6 +1,6 @@ package lib -import com.gu.mediaservice.lib.aws.AwsClientV2BuilderUtils +import com.gu.mediaservice.lib.aws.{AwsClientV2BuilderUtils, S3, S3Bucket} import com.gu.mediaservice.lib.cleanup.ReapableEligibiltyResources import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources, ReapableEligibilityLoader} import com.gu.mediaservice.lib.elasticsearch.ReapableEligibility @@ -56,11 +56,7 @@ object KinesisReceiverConfig { } class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val imageBucket: String = string("s3.image.bucket") - - val thumbnailBucket: String = string("s3.thumb.bucket") - - val maybeReaperBucket: Option[String] = stringOpt("s3.reaper.bucket") + val maybeReaperBucket: Option[S3Bucket] = stringOpt("s3.reaper.bucket").map(S3Bucket(_, S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, clientFor(S3.AmazonAwsS3Endpoint))) val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun") val metadataTopicArn: String = string("indexed.image.sns.topic.arn")