diff --git a/README.md b/README.md index 3030a31..abd5bb3 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ but without hitting real S3 endpoints. Implemented API methods: * list buckets -* list objects (all & by prefix) +* list objects (all, by prefix, and with marker) * create bucket * delete bucket * put object (via PUT, POST, multipart and chunked uploads are also supported) diff --git a/src/main/scala/io/findify/s3mock/provider/FileProvider.scala b/src/main/scala/io/findify/s3mock/provider/FileProvider.scala index f9a3341..fccbfb3 100644 --- a/src/main/scala/io/findify/s3mock/provider/FileProvider.scala +++ b/src/main/scala/io/findify/s3mock/provider/FileProvider.scala @@ -32,7 +32,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging { ListAllMyBuckets("root", UUID.randomUUID().toString, buckets) } - override def listBucket(bucket: String, prefix: Option[String], delimiter: Option[String], maxkeys: Option[Int]) = { + override def listBucket(bucket: String, prefix: Option[String], delimiter: Option[String], marker: Option[String], maxkeys: Option[Int]) = { def commonPrefix(dir: String, p: String, d: String): Option[String] = { dir.indexOf(d, p.length) match { case -1 => None @@ -61,7 +61,10 @@ class FileProvider(dir:String) extends Provider with LazyLogging { case Some(del) => files.flatMap(f => commonPrefix(f.key, prefixNoLeadingSlash, del)).distinct.sorted case None => Nil } - val filteredFiles = files.filterNot(f => commonPrefixes.exists(p => f.key.startsWith(p))) + val filteredFiles = marker.fold(files) { m => + files.filter(_.key > m) + } + .filterNot(f => commonPrefixes.exists(p => f.key.startsWith(p))) val count = maxkeys.getOrElse(Int.MaxValue) val result = filteredFiles.sortBy(_.key) ListBucket(bucket, prefix, delimiter, commonPrefixes, result.take(count), isTruncated = result.size>count) diff --git a/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala b/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala index a9f4ecd..de5e473 100644 --- a/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala +++ b/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala @@ -37,7 +37,7 @@ class InMemoryProvider extends Provider with LazyLogging { ListAllMyBuckets("root", UUID.randomUUID().toString, buckets.toList) } - override def listBucket(bucket: String, prefix: Option[String], delimiter: Option[String], maxkeys: Option[Int]): ListBucket = { + override def listBucket(bucket: String, prefix: Option[String], delimiter: Option[String], marker: Option[String], maxkeys: Option[Int]): ListBucket = { def commonPrefix(dir: String, p: String, d: String): Option[String] = { dir.indexOf(d, p.length) match { case -1 => None @@ -57,7 +57,10 @@ class InMemoryProvider extends Provider with LazyLogging { case Some(del) => matchResults.flatMap(f => commonPrefix(f.key, prefix2, del)).toList.sorted.distinct case None => Nil } - val filteredFiles: List[Content] = matchResults.filterNot(f => commonPrefixes.exists(p => f.key.startsWith(p))).toList + val filteredFiles: List[Content] = marker.fold(matchResults) { m => + matchResults.filter(_.key > m) + } + .filterNot(f => commonPrefixes.exists(p => f.key.startsWith(p))).toList val count = maxkeys.getOrElse(Int.MaxValue) val result = filteredFiles.sortBy(_.key) ListBucket(bucket, prefix, delimiter, commonPrefixes, result.take(count).take(count), isTruncated = result.size>count) diff --git a/src/main/scala/io/findify/s3mock/provider/Provider.scala b/src/main/scala/io/findify/s3mock/provider/Provider.scala index cef2e68..3d95858 100644 --- a/src/main/scala/io/findify/s3mock/provider/Provider.scala +++ b/src/main/scala/io/findify/s3mock/provider/Provider.scala @@ -14,7 +14,7 @@ case class GetObjectData(bytes: Array[Byte], metadata: Option[ObjectMetadata]) trait Provider { def metadataStore: MetadataStore def listBuckets:ListAllMyBuckets - def listBucket(bucket:String, prefix:Option[String], delimiter: Option[String], maxkeys: Option[Int]):ListBucket + def listBucket(bucket:String, prefix:Option[String], delimiter: Option[String], marker: Option[String], maxkeys: Option[Int]):ListBucket def createBucket(name:String, bucketConfig:CreateBucketConfiguration):CreateBucket def putObject(bucket:String, key:String, data:Array[Byte], metadata: ObjectMetadata):Unit def getObject(bucket:String, key:String): GetObjectData diff --git a/src/main/scala/io/findify/s3mock/route/ListBucket.scala b/src/main/scala/io/findify/s3mock/route/ListBucket.scala index 1bd7a2d..093e147 100644 --- a/src/main/scala/io/findify/s3mock/route/ListBucket.scala +++ b/src/main/scala/io/findify/s3mock/route/ListBucket.scala @@ -14,10 +14,10 @@ import scala.language.postfixOps */ case class ListBucket()(implicit provider:Provider) extends LazyLogging { def route(bucket:String) = get { - parameter('prefix?, 'delimiter?, Symbol("max-keys")?) { (prefix, delimiter, maxkeys) => + parameter('prefix?, 'delimiter?, 'marker?, Symbol("max-keys")?) { (prefix, delimiter, marker, maxkeys) => complete { - logger.info(s"listing bucket $bucket with prefix=$prefix, delimiter=$delimiter") - Try(provider.listBucket(bucket, prefix, delimiter, maxkeys.map(_.toInt))) match { + logger.info(s"listing bucket $bucket with prefix=$prefix, delimiter=$delimiter, marker=$marker") + Try(provider.listBucket(bucket, prefix, delimiter, marker, maxkeys.map(_.toInt))) match { case Success(l) => HttpResponse( StatusCodes.OK, entity = HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), l.toXML.toString) diff --git a/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala b/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala index 4fd034b..3dec107 100644 --- a/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala +++ b/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala @@ -111,10 +111,10 @@ class GetPutObjectTest extends S3MockTest { it should "work with = in path" in { s3.createBucket("urlencoded") - s3.listBuckets().exists(_.getName == "urlencoded") shouldBe true + s3.listBuckets().asScala.exists(_.getName == "urlencoded") shouldBe true s3.putObject("urlencoded", "path/with=123/foo", "bar=") s3.putObject("urlencoded", "path/withoutequals/foo", "bar") - val result = s3.listObjects("urlencoded").getObjectSummaries.toList.map(_.getKey) + val result = s3.listObjects("urlencoded").getObjectSummaries.asScala.toList.map(_.getKey) result shouldBe List("path/with=123/foo", "path/withoutequals/foo") getContent(s3.getObject("urlencoded", "path/with=123/foo")) shouldBe "bar=" getContent(s3.getObject("urlencoded", "path/withoutequals/foo")) shouldBe "bar" diff --git a/src/test/scala/io/findify/s3mock/ListBucketTest.scala b/src/test/scala/io/findify/s3mock/ListBucketTest.scala index 2b0a013..f48ee2c 100644 --- a/src/test/scala/io/findify/s3mock/ListBucketTest.scala +++ b/src/test/scala/io/findify/s3mock/ListBucketTest.scala @@ -26,6 +26,18 @@ class ListBucketTest extends S3MockTest { val list = s3.listObjects("list", "foo").getObjectSummaries.asScala.toList list.map(_.getKey).forall(_.startsWith("foo")) shouldBe true } + it should "list bucket with marker" in { + s3.createBucket("list") + s3.putObject("list", "foo1", "xxx") + s3.putObject("list", "foo2", "xxx") + s3.putObject("list", "xfoo3", "xxx") + val request = new ListObjectsRequest() + request.setBucketName("list") + request.setMarker("foo1") + val list = s3.listObjects(request).getObjectSummaries.asScala.toList + val expectedResult = List("foo2", "xfoo3") + list.map(_.getKey) should contain theSameElementsAs(expectedResult) + } it should "list objects in subfolders with prefix" in { s3.createBucket("list2") s3.putObject("list2", "one/foo1/1", "xxx")