Merge pull request #35 from SwiftPackageIndex/update-soto-s3-file-transfer

finestructure · web-flow · commit ce95e7462396 · 2025-03-21T14:33:37.000+01:00
Bump soto-s3-file-transfer to 2.x
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ DerivedData/
 .netrc
 /.lambda/
 /.vscode/
+.swiftpm/
diff --git a/Package.resolved b/Package.resolved
@@ -5,8 +5,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/swift-server/async-http-client.git",
       "state" : {
-        "revision" : "291438696abdd48d2a83b52465c176efbd94512b",
-        "version" : "1.20.1"
+        "revision" : "333f51104b75d1a5b94cb3b99e4c58a3b442c9f7",
+        "version" : "1.25.2"
       }
     },
     {
@@ -32,26 +32,26 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/soto-project/soto.git",
       "state" : {
-        "revision" : "c28c3efa72e6beceae6f38f8c8fe18e7c57e3418",
-        "version" : "6.8.0"
+        "revision" : "141b3c944f3f4a9ad428280e8ad94f6f437e0f00",
+        "version" : "7.5.0"
       }
     },
     {
       "identity" : "soto-core",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/soto-project/soto-core.git",
       "state" : {
-        "revision" : "3ed66ec536bc372360d7e3ec39a5e8b1870747ea",
-        "version" : "6.5.2"
+        "revision" : "a61299aa18626e462312984765d34146286b29d3",
+        "version" : "7.4.0"
       }
     },
     {
       "identity" : "soto-s3-file-transfer",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/soto-project/soto-s3-file-transfer.git",
       "state" : {
-        "revision" : "da375d7544080e8f00dba3bb573d7a412f3c4b41",
-        "version" : "1.3.0"
+        "revision" : "5104491d1a8f76a2839e4476e9644b3885ba758c",
+        "version" : "2.1.0"
       }
     },
     {
@@ -135,6 +135,15 @@
         "version" : "1.2.2"
       }
     },
+    {
+      "identity" : "swift-distributed-tracing",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-distributed-tracing.git",
+      "state" : {
+        "revision" : "a64a0abc2530f767af15dd88dda7f64d5f1ff9de",
+        "version" : "1.2.0"
+      }
+    },
     {
       "identity" : "swift-http-types",
       "kind" : "remoteSourceControl",
@@ -167,8 +176,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio.git",
       "state" : {
-        "revision" : "fc63f0cf4e55a4597407a9fc95b16a2bc44b4982",
-        "version" : "2.64.0"
+        "revision" : "c51907a839e63ebf0ba2076bba73dd96436bd1b9",
+        "version" : "2.81.0"
       }
     },
     {
@@ -194,8 +203,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio-ssl.git",
       "state" : {
-        "revision" : "7c381eb6083542b124a6c18fae742f55001dc2b5",
-        "version" : "2.26.0"
+        "revision" : "0cc3528ff48129d64ab9cab0b1cd621634edfc6b",
+        "version" : "2.29.3"
       }
     },
     {
@@ -216,6 +225,15 @@
         "version" : "1.0.2"
       }
     },
+    {
+      "identity" : "swift-service-context",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-service-context.git",
+      "state" : {
+        "revision" : "8946c930cae601452149e45d31d8ddfac973c3c7",
+        "version" : "1.2.0"
+      }
+    },
     {
       "identity" : "swift-syntax",
       "kind" : "remoteSourceControl",
@@ -230,8 +248,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-system.git",
       "state" : {
-        "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496",
-        "version" : "1.2.1"
+        "revision" : "a34201439c74b53f0fd71ef11741af7e7caf01e1",
+        "version" : "1.4.2"
       }
     },
     {
diff --git a/Package.swift b/Package.swift
@@ -39,7 +39,7 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-crypto.git", from: "3.0.0"),
         .package(url: "https://github.com/swift-server/swift-aws-lambda-events.git", from: "0.1.0"),
         .package(url: "https://github.com/swift-server/swift-aws-lambda-runtime.git", from: "1.0.0-alpha.1"),
-        .package(url: "https://github.com/soto-project/soto-s3-file-transfer.git", from: "1.2.0"),
+        .package(url: "https://github.com/soto-project/soto-s3-file-transfer.git", from: "2.0.0"),
         .package(url: "https://github.com/vapor-community/Zip.git", from: "2.0.0"),
         .package(url: "https://github.com/pointfreeco/swift-dependencies.git", from: "1.0.0")
     ],
diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ Instead, use the `dev` environment to validate a new release as follows:
 - Run the tests
 
     ```
-    docker run --rm -v "$PWD":/host -w /host swift:5.10.0-amazonlinux2 swift test
+    docker run --rm -v "$PWD":/host -w /host swift:6.0.3-amazonlinux2 swift test
     ```
 
 - Deploy the new version to the "test" lambda
diff --git a/Sources/DocUploader/DocUploader.swift b/Sources/DocUploader/DocUploader.swift
@@ -48,16 +48,11 @@ public struct DocUploader: LambdaHandler {
         }
         self.httpClient = httpClient
 
-        let awsClient = AWSClient(httpClientProvider: .shared(httpClient))
+        let awsClient = AWSClient(httpClient: httpClient)
         context.terminator.register(name: "awsclient") { eventLoop in
             let promise = eventLoop.makePromise(of: Void.self)
-            awsClient.shutdown() { error in
-                switch error {
-                case .none:
-                    promise.succeed(())
-                case .some(let error):
-                    promise.fail(error)
-                }
+            promise.completeWithTask {
+                try await awsClient.shutdown()
             }
             return promise.futureResult
         }
diff --git a/Sources/DocUploader/LiveS3Client.swift b/Sources/DocUploader/LiveS3Client.swift
@@ -20,7 +20,7 @@ import SotoS3FileTransfer
 struct LiveS3Client: S3Client {
     func deleteFile(client: AWSClient, logger: Logger, key: S3StoreKey) async throws {
         let s3 = S3(client: client, region: .useast2)
-        let s3FileTransfer = S3FileTransferManager(s3: s3, threadPoolProvider: .singleton)
+        let s3FileTransfer = S3FileTransferManager(s3: s3)
 
         guard let file = S3File(key: key) else {
             throw Error(message: "Invalid key: \(key)")
@@ -30,7 +30,7 @@ struct LiveS3Client: S3Client {
 
     func loadFile(client: AWSClient, logger: Logger, from key: S3StoreKey, to path: String) async throws {
         let s3 = S3(client: client, region: .useast2)
-        let s3FileTransfer = S3FileTransferManager(s3: s3, threadPoolProvider: .singleton)
+        let s3FileTransfer = S3FileTransferManager(s3: s3)
 
         guard let file = S3File(key: key) else {
             throw Error(message: "Invalid key: \(key)")
@@ -44,26 +44,17 @@ struct LiveS3Client: S3Client {
                     timeout: .seconds(60),
                     options: .s3DisableChunkedUploads)
 
-        let threadPool = NIOThreadPool(numberOfThreads: 8)
-        defer { threadPool.shutdownGracefully { _ in } }
-        threadPool.start()
-
-        let s3FileTransfer = S3FileTransferManager(s3: s3,
-                                                   threadPoolProvider: .shared(threadPool),
-                                                   configuration: .init(maxConcurrentTasks: 60))
-        defer {
-            try? s3FileTransfer.syncShutdown()
-        }
+        let s3FileTransfer = S3FileTransferManager(s3: s3, configuration: .init(maxConcurrentTasks: 60))
 
         guard let s3Folder = S3Folder(url: key.url) else {
             throw Error(message: "Invalid key: \(key)")
         }
 
-        var nextProgressTick = 0.1
+        let nextProgressTick = QueueIsolated(0.1)
         try await s3FileTransfer.sync(from: folder, to: s3Folder, delete: true) { progress in
-            if progress >= nextProgressTick {
+            if progress >= nextProgressTick.value {
                 logger.info("Syncing... [\(percent: progress)]")
-                nextProgressTick += 0.1
+                nextProgressTick.withValue { $0 += 0.1 }
             }
         }
     }
diff --git a/Sources/DocUploader/QueueIsolated.swift b/Sources/DocUploader/QueueIsolated.swift
@@ -0,0 +1,69 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+
+// Modeled after ActorIsolated with synchronisation via a queue instead of an actor, allowing
+// sync access where async isn't possible.
+
+@dynamicMemberLookup
+public final class QueueIsolated<Value: Sendable>: @unchecked Sendable {
+    private let _queue = DispatchQueue(label: "queue-isolated")
+
+    private var _value: Value
+
+    public init(_ value: Value) {
+        self._value = value
+    }
+
+    public var value: Value {
+        get {
+            _queue.sync { self._value }
+        }
+    }
+
+    public subscript<Subject>(dynamicMember keyPath: KeyPath<Value, Subject>) -> Subject {
+        _queue.sync {
+            self._value[keyPath: keyPath]
+        }
+    }
+
+    public func withValue<T>(
+        _ operation: (inout Value) throws -> T
+    ) rethrows -> T {
+        try _queue.sync {
+            var value = self._value
+            defer { self._value = value }
+            return try operation(&value)
+        }
+    }
+
+    public func setValue(_ newValue: Value) {
+        _queue.async {
+            self._value = newValue
+        }
+    }
+}
+
+
+extension QueueIsolated where Value == Int {
+    public func increment(by delta: Int = 1) {
+        withValue { $0 += delta }
+    }
+
+    public func decrement(by delta: Int = 1) {
+        withValue { $0 -= delta }
+    }
+}
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -22,5 +22,5 @@ docker run \
   --rm \
   --volume "$(pwd):/src" \
   --workdir "/src" \
-  swift:5.10.0-amazonlinux2 \
+  swift:6.0.3-amazonlinux2 \
   swift build --disable-automatic-resolution --product "$executable" -c release --static-swift-stdlib #-Xswiftc -cross-module-optimization 

Original file line number	Diff line number	Diff line change
`@@ -48,16 +48,11 @@ public struct DocUploader: LambdaHandler {`
`48`	`48`	`}`
`49`	`49`	`self.httpClient = httpClient`
`50`	`50`
`51`		`- let awsClient = AWSClient(httpClientProvider: .shared(httpClient))`
	`51`	`+ let awsClient = AWSClient(httpClient: httpClient)`
`52`	`52`	`context.terminator.register(name: "awsclient") { eventLoop in`
`53`	`53`	`let promise = eventLoop.makePromise(of: Void.self)`
`54`		`- awsClient.shutdown() { error in`
`55`		`- switch error {`
`56`		`- case .none:`
`57`		`- promise.succeed(())`
`58`		`- case .some(let error):`
`59`		`- promise.fail(error)`
`60`		`- }`
	`54`	`+ promise.completeWithTask {`
	`55`	`+ try await awsClient.shutdown()`
`61`	`56`	`}`
`62`	`57`	`return promise.futureResult`
`63`	`58`	`}`