From 8610af1ef8935d4071151f8d1ae275eccd914b9d Mon Sep 17 00:00:00 2001 From: Adam Fowler Date: Mon, 13 Jan 2025 17:04:32 +0000 Subject: [PATCH 1/4] Replace all uses of NonBlockingFileIO --- Package.swift | 3 +- Sources/Hummingbird/Files/FileIO.swift | 80 +++++++++---------- .../Hummingbird/Files/LocalFileSystem.swift | 12 +-- 3 files changed, 42 insertions(+), 53 deletions(-) diff --git a/Package.swift b/Package.swift index 1810bb32..73d1c950 100644 --- a/Package.swift +++ b/Package.swift @@ -25,7 +25,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-http-types.git", from: "1.0.0"), .package(url: "https://github.com/apple/swift-metrics.git", from: "2.5.0"), .package(url: "https://github.com/apple/swift-distributed-tracing.git", from: "1.1.0"), - .package(url: "https://github.com/apple/swift-nio.git", from: "2.63.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.78.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.20.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.34.1"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.0"), @@ -46,6 +46,7 @@ let package = Package( .product(name: "Metrics", package: "swift-metrics"), .product(name: "Tracing", package: "swift-distributed-tracing"), .product(name: "NIOCore", package: "swift-nio"), + .product(name: "_NIOFileSystem", package: "swift-nio"), .product(name: "NIOPosix", package: "swift-nio"), ], swiftSettings: swiftSettings diff --git a/Sources/Hummingbird/Files/FileIO.swift b/Sources/Hummingbird/Files/FileIO.swift index d0c727ff..701b4d2e 100644 --- a/Sources/Hummingbird/Files/FileIO.swift +++ b/Sources/Hummingbird/Files/FileIO.swift @@ -16,15 +16,24 @@ import HummingbirdCore import Logging import NIOCore import NIOPosix +import _NIOFileSystem /// Manages File reading and writing. public struct FileIO: Sendable { - let fileIO: NonBlockingFileIO + struct FileError: Error { + internal enum Value { + case fileDoesNotExist + } + internal let value: Value + + static var fileDoesNotExist: Self { .init(value: .fileDoesNotExist) } + } + let fileSystem: FileSystem /// Initialize FileIO /// - Parameter threadPool: ThreadPool to use for file operations public init(threadPool: NIOThreadPool = .singleton) { - self.fileIO = .init(threadPool: threadPool) + self.fileSystem = .init(threadPool: threadPool) } /// Load file and return response body @@ -39,12 +48,12 @@ public struct FileIO: Sendable { public func loadFile( path: String, context: some RequestContext, - chunkLength: Int = NonBlockingFileIO.defaultChunkSize + chunkLength: Int = 128 * 1024 ) async throws -> ResponseBody { do { - let stat = try await fileIO.stat(path: path) - guard stat.st_size > 0 else { return .init() } - return self.readFile(path: path, range: 0...numericCast(stat.st_size - 1), context: context, chunkLength: chunkLength) + guard let info = try await self.fileSystem.info(forFileAt: .init(path)) else { throw FileError.fileDoesNotExist } + guard info.size > 0 else { return .init() } + return self.readFile(path: path, range: 0...numericCast(info.size - 1), context: context, chunkLength: chunkLength) } catch { throw HTTPError(.notFound) } @@ -64,12 +73,12 @@ public struct FileIO: Sendable { path: String, range: ClosedRange, context: some RequestContext, - chunkLength: Int = NonBlockingFileIO.defaultChunkSize + chunkLength: Int = 128 * 1024 ) async throws -> ResponseBody { do { - let stat = try await fileIO.stat(path: path) - guard stat.st_size > 0 else { return .init() } - let fileRange: ClosedRange = 0...numericCast(stat.st_size - 1) + guard let info = try await self.fileSystem.info(forFileAt: .init(path)) else { throw FileError.fileDoesNotExist } + guard info.size > 0 else { return .init() } + let fileRange: ClosedRange = 0...numericCast(info.size - 1) let range = range.clamped(to: fileRange) return self.readFile(path: path, range: range, context: context, chunkLength: chunkLength) } catch { @@ -89,9 +98,12 @@ public struct FileIO: Sendable { context: some RequestContext ) async throws where AS.Element == ByteBuffer { context.logger.debug("[FileIO] PUT", metadata: ["hb.file.path": .string(path)]) - try await self.fileIO.withFileHandle(path: path, mode: .write, flags: .allowFileCreation()) { handle in - for try await buffer in contents { - try await self.fileIO.write(fileHandle: handle, buffer: buffer) + try await self.fileSystem.withFileHandle( + forWritingAt: .init(path), + options: .newFile(replaceExisting: true) + ) { fileHandle in + try await fileHandle.withBufferedWriter { writer in + _ = try await writer.write(contentsOf: contents) } } } @@ -108,8 +120,11 @@ public struct FileIO: Sendable { context: some RequestContext ) async throws { context.logger.debug("[FileIO] PUT", metadata: ["hb.file.path": .string(path)]) - try await self.fileIO.withFileHandle(path: path, mode: .write, flags: .allowFileCreation()) { handle in - try await self.fileIO.write(fileHandle: handle, buffer: buffer) + try await self.fileSystem.withFileHandle( + forWritingAt: .init(path), + options: .newFile(replaceExisting: true) + ) { fileHandle in + _ = try await fileHandle.write(contentsOf: buffer, toAbsoluteOffset: 0) } } @@ -118,41 +133,18 @@ public struct FileIO: Sendable { path: String, range: ClosedRange, context: some RequestContext, - chunkLength: Int = NonBlockingFileIO.defaultChunkSize + chunkLength: Int ) -> ResponseBody { ResponseBody(contentLength: range.count) { writer in - try await self.fileIO.withFileHandle(path: path, mode: .read) { handle in - let endOffset = range.endIndex - let chunkLength = chunkLength - var fileOffset = range.startIndex - let allocator = ByteBufferAllocator() + try await self.fileSystem.withFileHandle(forReadingAt: .init(path)) { fileHandle in + let startOffset: Int64 = numericCast(range.lowerBound) + let endOffset: Int64 = numericCast(range.upperBound) - while case .inRange(let offset) = fileOffset { - let bytesLeft = range.distance(from: fileOffset, to: endOffset) - let bytesToRead = Swift.min(chunkLength, bytesLeft) - let buffer = try await self.fileIO.read( - fileHandle: handle, - fromOffset: numericCast(offset), - byteCount: bytesToRead, - allocator: allocator - ) - fileOffset = range.index(fileOffset, offsetBy: bytesToRead) - try await writer.write(buffer) + for try await chunk in fileHandle.readChunks(in: startOffset...endOffset, chunkLength: .bytes(numericCast(chunkLength))) { + try await writer.write(chunk) } try await writer.finish(nil) } } } } - -extension NonBlockingFileIO { - func stat(path: String) async throws -> stat { - let stat = try await self.lstat(path: path) - if stat.st_mode & S_IFMT == S_IFLNK { - let realPath = try await self.readlink(path: path) - return try await self.lstat(path: realPath) - } else { - return stat - } - } -} diff --git a/Sources/Hummingbird/Files/LocalFileSystem.swift b/Sources/Hummingbird/Files/LocalFileSystem.swift index d0099fb0..fa95d8cf 100644 --- a/Sources/Hummingbird/Files/LocalFileSystem.swift +++ b/Sources/Hummingbird/Files/LocalFileSystem.swift @@ -89,16 +89,12 @@ public struct LocalFileSystem: FileProvider { /// - Returns: File attributes public func getAttributes(id path: FileIdentifier) async throws -> FileAttributes? { do { - let stat = try await self.fileIO.fileIO.stat(path: path) - let isFolder = (stat.st_mode & S_IFMT) == S_IFDIR - #if os(Linux) - let modificationDate = Double(stat.st_mtim.tv_sec) + (Double(stat.st_mtim.tv_nsec) / 1_000_000_000.0) - #else - let modificationDate = Double(stat.st_mtimespec.tv_sec) + (Double(stat.st_mtimespec.tv_nsec) / 1_000_000_000.0) - #endif + guard let info = try await self.fileIO.fileSystem.info(forFileAt: .init(path)) else { throw FileIO.FileError.fileDoesNotExist } + let isFolder = info.type == .directory + let modificationDate = Double(info.lastDataModificationTime.seconds) return .init( isFolder: isFolder, - size: numericCast(stat.st_size), + size: numericCast(info.size), modificationDate: Date(timeIntervalSince1970: modificationDate) ) } catch { From 4395e55a6fa57ef2cd3503ed1c443deae0f6b7ec Mon Sep 17 00:00:00 2001 From: Adam Fowler Date: Mon, 13 Jan 2025 17:04:39 +0000 Subject: [PATCH 2/4] Tidy up tests --- .../FileMiddlewareTests.swift | 341 +++++++++--------- 1 file changed, 169 insertions(+), 172 deletions(-) diff --git a/Tests/HummingbirdTests/FileMiddlewareTests.swift b/Tests/HummingbirdTests/FileMiddlewareTests.swift index dc890a69..accdee59 100644 --- a/Tests/HummingbirdTests/FileMiddlewareTests.swift +++ b/Tests/HummingbirdTests/FileMiddlewareTests.swift @@ -18,6 +18,7 @@ import Hummingbird import HummingbirdTesting import NIOPosix import XCTest +import _NIOFileSystem final class FileMiddlewareTests: XCTestCase { static func randomBuffer(size: Int) -> ByteBuffer { @@ -34,6 +35,25 @@ final class FileMiddlewareTests: XCTestCase { return formatter } + static func withFile( + _ path: String, + contents: Buffer, + process: () async throws -> ReturnValue + ) async throws -> ReturnValue where Buffer.Element == UInt8 { + let fileSystem = FileSystem(threadPool: .singleton) + try await fileSystem.withFileHandle(forWritingAt: .init(path)) { write in + _ = try await write.write(contentsOf: contents, toAbsoluteOffset: 0) + } + do { + let value = try await process() + _ = try? await fileSystem.removeItem(at: .init(path)) + return value + } catch { + _ = try? await fileSystem.removeItem(at: .init(path)) + throw error + } + } + func testRead() async throws { let router = Router() router.middlewares.add(FileMiddleware(".")) @@ -41,15 +61,13 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).jpg" let text = "Test file contents" - let data = Data(text.utf8) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - try await client.execute(uri: filename, method: .get) { response in - XCTAssertEqual(String(buffer: response.body), text) - XCTAssertEqual(response.headers[.contentType], "image/jpeg") + try await Self.withFile(filename, contents: text.utf8) { + try await app.test(.router) { client in + try await client.execute(uri: filename, method: .get) { response in + XCTAssertEqual(String(buffer: response.body), text) + XCTAssertEqual(response.headers[.contentType], "image/jpeg") + } } } } @@ -73,14 +91,12 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 380_000) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - try await client.execute(uri: filename, method: .get) { response in - XCTAssertEqual(response.body, buffer) + try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await app.test(.router) { client in + try await client.execute(uri: filename, method: .get) { response in + XCTAssertEqual(response.body, buffer) + } } } } @@ -92,40 +108,38 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 326_000) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=100-3999"]) { response in - let slice = buffer.getSlice(at: 100, length: 3900) - XCTAssertEqual(response.body, slice) - XCTAssertEqual(response.headers[.contentRange], "bytes 100-3999/326000") - XCTAssertEqual(response.headers[.contentLength], "3900") - XCTAssertEqual(response.headers[.contentType], "text/plain") - } + try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await app.test(.router) { client in + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=100-3999"]) { response in + let slice = buffer.getSlice(at: 100, length: 3900) + XCTAssertEqual(response.body, slice) + XCTAssertEqual(response.headers[.contentRange], "bytes 100-3999/326000") + XCTAssertEqual(response.headers[.contentLength], "3900") + XCTAssertEqual(response.headers[.contentType], "text/plain") + } - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=0-0"]) { response in - let slice = buffer.getSlice(at: 0, length: 1) - XCTAssertEqual(response.body, slice) - XCTAssertEqual(response.headers[.contentRange], "bytes 0-0/326000") - XCTAssertEqual(response.headers[.contentLength], "1") - XCTAssertEqual(response.headers[.contentType], "text/plain") - } + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=0-0"]) { response in + let slice = buffer.getSlice(at: 0, length: 1) + XCTAssertEqual(response.body, slice) + XCTAssertEqual(response.headers[.contentRange], "bytes 0-0/326000") + XCTAssertEqual(response.headers[.contentLength], "1") + XCTAssertEqual(response.headers[.contentType], "text/plain") + } - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=-3999"]) { response in - let slice = buffer.getSlice(at: 0, length: 4000) - XCTAssertEqual(response.body, slice) - XCTAssertEqual(response.headers[.contentLength], "4000") - XCTAssertEqual(response.headers[.contentRange], "bytes 0-3999/326000") - } + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=-3999"]) { response in + let slice = buffer.getSlice(at: 0, length: 4000) + XCTAssertEqual(response.body, slice) + XCTAssertEqual(response.headers[.contentLength], "4000") + XCTAssertEqual(response.headers[.contentRange], "bytes 0-3999/326000") + } - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=6000-"]) { response in - let slice = buffer.getSlice(at: 6000, length: 320_000) - XCTAssertEqual(response.body, slice) - XCTAssertEqual(response.headers[.contentLength], "320000") - XCTAssertEqual(response.headers[.contentRange], "bytes 6000-325999/326000") + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=6000-"]) { response in + let slice = buffer.getSlice(at: 6000, length: 320_000) + XCTAssertEqual(response.body, slice) + XCTAssertEqual(response.headers[.contentLength], "320000") + XCTAssertEqual(response.headers[.contentRange], "bytes 6000-325999/326000") + } } } } @@ -137,32 +151,30 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 10000) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - let (eTag, modificationDate) = try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=-3999"]) { - response -> (String, String) in - let eTag = try XCTUnwrap(response.headers[.eTag]) - let modificationDate = try XCTUnwrap(response.headers[.lastModified]) - let slice = buffer.getSlice(at: 0, length: 4000) - XCTAssertEqual(response.body, slice) - XCTAssertEqual(response.headers[.contentRange], "bytes 0-3999/10000") - return (eTag, modificationDate) - } + try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await app.test(.router) { client in + let (eTag, modificationDate) = try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=-3999"]) { + response -> (String, String) in + let eTag = try XCTUnwrap(response.headers[.eTag]) + let modificationDate = try XCTUnwrap(response.headers[.lastModified]) + let slice = buffer.getSlice(at: 0, length: 4000) + XCTAssertEqual(response.body, slice) + XCTAssertEqual(response.headers[.contentRange], "bytes 0-3999/10000") + return (eTag, modificationDate) + } - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=4000-", .ifRange: eTag]) { response in - XCTAssertEqual(response.headers[.contentRange], "bytes 4000-9999/10000") - } + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=4000-", .ifRange: eTag]) { response in + XCTAssertEqual(response.headers[.contentRange], "bytes 4000-9999/10000") + } - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=4000-", .ifRange: modificationDate]) { response in - XCTAssertEqual(response.headers[.contentRange], "bytes 4000-9999/10000") - } + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=4000-", .ifRange: modificationDate]) { response in + XCTAssertEqual(response.headers[.contentRange], "bytes 4000-9999/10000") + } - try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=4000-", .ifRange: "not valid"]) { response in - XCTAssertNil(response.headers[.contentRange]) + try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=4000-", .ifRange: "not valid"]) { response in + XCTAssertNil(response.headers[.contentRange]) + } } } } @@ -172,21 +184,21 @@ final class FileMiddlewareTests: XCTestCase { router.middlewares.add(FileMiddleware(".")) let app = Application(responder: router.buildResponder()) + let filename = "testHead.txt" let date = Date() let text = "Test file contents" - let data = Data(text.utf8) - let fileURL = URL(fileURLWithPath: "testHead.txt") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - try await client.execute(uri: "/testHead.txt", method: .head) { response in - XCTAssertEqual(response.body.readableBytes, 0) - XCTAssertEqual(response.headers[.contentLength], text.utf8.count.description) - XCTAssertEqual(response.headers[.contentType], "text/plain") - let responseDateString = try XCTUnwrap(response.headers[.lastModified]) - let responseDate = try XCTUnwrap(Self.rfc9110Formatter.date(from: responseDateString)) - XCTAssert(date < responseDate + 2 && date > responseDate - 2) + try await Self.withFile(filename, contents: text.utf8) { + try await app.test(.router) { client in + let filename = "testHead.txt" + try await client.execute(uri: "/\(filename)", method: .head) { response in + XCTAssertEqual(response.body.readableBytes, 0) + XCTAssertEqual(response.headers[.contentLength], text.utf8.count.description) + XCTAssertEqual(response.headers[.contentType], "text/plain") + let responseDateString = try XCTUnwrap(response.headers[.lastModified]) + let responseDate = try XCTUnwrap(Self.rfc9110Formatter.date(from: responseDateString)) + XCTAssert(date < responseDate + 2 && date > responseDate - 2) + } } } } @@ -198,18 +210,16 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 16200) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - var eTag: String? - try await client.execute(uri: filename, method: .head) { response in - eTag = try XCTUnwrap(response.headers[.eTag]) - } - try await client.execute(uri: filename, method: .head) { response in - XCTAssertEqual(response.headers[.eTag], eTag) + try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await app.test(.router) { client in + var eTag: String? + try await client.execute(uri: filename, method: .head) { response in + eTag = try XCTUnwrap(response.headers[.eTag]) + } + try await client.execute(uri: filename, method: .head) { response in + XCTAssertEqual(response.headers[.eTag], eTag) + } } } } @@ -221,25 +231,23 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 16200) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - let eTag = try await client.execute(uri: filename, method: .head) { response in - try XCTUnwrap(response.headers[.eTag]) - } - try await client.execute(uri: filename, method: .get, headers: [.ifNoneMatch: eTag]) { response in - XCTAssertEqual(response.status, .notModified) - } - var headers: HTTPFields = .init() - headers[values: .ifNoneMatch] = ["test", "\(eTag)"] - try await client.execute(uri: filename, method: .get, headers: headers) { response in - XCTAssertEqual(response.status, .notModified) - } - try await client.execute(uri: filename, method: .get, headers: [.ifNoneMatch: "dummyETag"]) { response in - XCTAssertEqual(response.status, .ok) + try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await app.test(.router) { client in + let eTag = try await client.execute(uri: filename, method: .head) { response in + try XCTUnwrap(response.headers[.eTag]) + } + try await client.execute(uri: filename, method: .get, headers: [.ifNoneMatch: eTag]) { response in + XCTAssertEqual(response.status, .notModified) + } + var headers: HTTPFields = .init() + headers[values: .ifNoneMatch] = ["test", "\(eTag)"] + try await client.execute(uri: filename, method: .get, headers: headers) { response in + XCTAssertEqual(response.status, .notModified) + } + try await client.execute(uri: filename, method: .get, headers: [.ifNoneMatch: "dummyETag"]) { response in + XCTAssertEqual(response.status, .ok) + } } } } @@ -251,22 +259,20 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 16200) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - let modifiedDate = try await client.execute(uri: filename, method: .head) { response in - try XCTUnwrap(response.headers[.lastModified]) - } - try await client.execute(uri: filename, method: .get, headers: [.ifModifiedSince: modifiedDate]) { response in - XCTAssertEqual(response.status, .notModified) - } - // one minute before current date - let date = try XCTUnwrap(Self.rfc9110Formatter.string(from: Date(timeIntervalSinceNow: -60))) - try await client.execute(uri: filename, method: .get, headers: [.ifModifiedSince: date]) { response in - XCTAssertEqual(response.status, .ok) + try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await app.test(.router) { client in + let modifiedDate = try await client.execute(uri: filename, method: .head) { response in + try XCTUnwrap(response.headers[.lastModified]) + } + try await client.execute(uri: filename, method: .get, headers: [.ifModifiedSince: modifiedDate]) { response in + XCTAssertEqual(response.status, .notModified) + } + // one minute before current date + let date = try XCTUnwrap(Self.rfc9110Formatter.string(from: Date(timeIntervalSinceNow: -60))) + try await client.execute(uri: filename, method: .get, headers: [.ifModifiedSince: date]) { response in + XCTAssertEqual(response.status, .ok) + } } } } @@ -282,20 +288,18 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let text = "Test file contents" - let data = Data(text.utf8) - let fileURL = URL(fileURLWithPath: filename) - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - let fileURL2 = URL(fileURLWithPath: "test.jpg") - XCTAssertNoThrow(try data.write(to: fileURL2)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL2)) } - - try await app.test(.router) { client in - try await client.execute(uri: filename, method: .get) { response in - XCTAssertEqual(response.headers[.cacheControl], "max-age=2592000") - } - try await client.execute(uri: "/test.jpg", method: .get) { response in - XCTAssertEqual(response.headers[.cacheControl], "max-age=2592000, private") + let filename2 = "\(#function).jpg" + + try await Self.withFile(filename, contents: text.utf8) { + try await Self.withFile(filename2, contents: text.utf8) { + try await app.test(.router) { client in + try await client.execute(uri: filename, method: .get) { response in + XCTAssertEqual(response.headers[.cacheControl], "max-age=2592000") + } + try await client.execute(uri: filename2, method: .get) { response in + XCTAssertEqual(response.headers[.cacheControl], "max-age=2592000, private") + } + } } } } @@ -306,14 +310,12 @@ final class FileMiddlewareTests: XCTestCase { let app = Application(responder: router.buildResponder()) let text = "Test file contents" - let data = Data(text.utf8) - let fileURL = URL(fileURLWithPath: "index.html") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - try await client.execute(uri: "/", method: .get) { response in - XCTAssertEqual(String(buffer: response.body), text) + try await Self.withFile("index.html", contents: text.utf8) { + try await app.test(.router) { client in + try await client.execute(uri: "/", method: .get) { response in + XCTAssertEqual(String(buffer: response.body), text) + } } } } @@ -324,33 +326,30 @@ final class FileMiddlewareTests: XCTestCase { let app = Application(responder: router.buildResponder()) let text = "Test file contents" - let data = Data(text.utf8) - let fileURL = URL(fileURLWithPath: "test.html") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - let fileIO = NonBlockingFileIO(threadPool: .singleton) + try await Self.withFile("test.html", contents: text.utf8) { + try await app.test(.router) { client in + try await client.execute(uri: "/test.html", method: .get) { response in + XCTAssertEqual(String(buffer: response.body), text) + } - try await app.test(.router) { client in - try await client.execute(uri: "/test.html", method: .get) { response in - XCTAssertEqual(String(buffer: response.body), text) - } + try await client.execute(uri: "/", method: .get) { response in + XCTAssertEqual(String(buffer: response.body), "") + } - try await client.execute(uri: "/", method: .get) { response in - XCTAssertEqual(String(buffer: response.body), "") - } + let fileSystem = FileSystem(threadPool: .singleton) + try await fileSystem.createSymbolicLink(at: .init("index.html"), withDestination: .init("test.html")) - try await fileIO.symlink(path: "index.html", to: "test.html") + do { + try await client.execute(uri: "/", method: .get) { response in + XCTAssertEqual(String(buffer: response.body), text) + } - do { - try await client.execute(uri: "/", method: .get) { response in - XCTAssertEqual(String(buffer: response.body), text) + try await fileSystem.removeItem(at: .init("index.html")) + } catch { + try await fileSystem.removeItem(at: .init("index.html")) + throw error } - - try await fileIO.unlink(path: "index.html") - } catch { - try await fileIO.unlink(path: "index.html") - throw error } } } @@ -371,14 +370,12 @@ final class FileMiddlewareTests: XCTestCase { let app = Application(responder: router.buildResponder()) let text = "Test file contents" - let data = Data(text.utf8) - let fileURL = URL(fileURLWithPath: "index.html") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - try await app.test(.router) { client in - try await client.execute(uri: "/", method: .get) { response in - XCTAssertEqual(String(buffer: response.body), text) + try await Self.withFile("index.html", contents: text.utf8) { + try await app.test(.router) { client in + try await client.execute(uri: "/", method: .get) { response in + XCTAssertEqual(String(buffer: response.body), text) + } } } } From 3fd40035b12e448e5648a85a5b257a0ff992026f Mon Sep 17 00:00:00 2001 From: Adam Fowler Date: Mon, 13 Jan 2025 17:22:17 +0000 Subject: [PATCH 3/4] Update tests --- Tests/HummingbirdTests/FileIOTests.swift | 101 ++++++++++-------- .../FileMiddlewareTests.swift | 45 +++----- 2 files changed, 72 insertions(+), 74 deletions(-) diff --git a/Tests/HummingbirdTests/FileIOTests.swift b/Tests/HummingbirdTests/FileIOTests.swift index 10dd2535..52f20da5 100644 --- a/Tests/HummingbirdTests/FileIOTests.swift +++ b/Tests/HummingbirdTests/FileIOTests.swift @@ -15,6 +15,7 @@ import Hummingbird import HummingbirdTesting import XCTest +import _NIOFileSystem final class FileIOTests: XCTestCase { static func randomBuffer(size: Int) -> ByteBuffer { @@ -23,6 +24,25 @@ final class FileIOTests: XCTestCase { return ByteBufferAllocator().buffer(bytes: data) } + static func withFile( + _ path: String, + contents: Buffer, + process: () async throws -> ReturnValue + ) async throws -> ReturnValue where Buffer.Element == UInt8 { + let fileSystem = FileSystem(threadPool: .singleton) + try await fileSystem.withFileHandle(forWritingAt: .init(path)) { write in + _ = try await write.write(contentsOf: contents, toAbsoluteOffset: 0) + } + do { + let value = try await process() + _ = try? await fileSystem.removeItem(at: .init(path)) + return value + } catch { + _ = try? await fileSystem.removeItem(at: .init(path)) + throw error + } + } + func testReadFileIO() async throws { let router = Router() router.get("test.jpg") { _, context -> Response in @@ -30,17 +50,15 @@ final class FileIOTests: XCTestCase { let body = try await fileIO.loadFile(path: "testReadFileIO.jpg", context: context) return .init(status: .ok, headers: [:], body: body) } - let buffer = Self.randomBuffer(size: 320_003) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: "testReadFileIO.jpg") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - let app = Application(responder: router.buildResponder()) - try await app.test(.router) { client in - try await client.execute(uri: "/test.jpg", method: .get) { response in - XCTAssertEqual(response.body, buffer) + let buffer = Self.randomBuffer(size: 320_003) + + try await FileIOTests.withFile("testReadFileIO.jpg", contents: buffer.readableBytesView) { + try await app.test(.router) { client in + try await client.execute(uri: "/test.jpg", method: .get) { response in + XCTAssertEqual(response.body, buffer) + } } } } @@ -53,19 +71,17 @@ final class FileIOTests: XCTestCase { return .init(status: .ok, headers: [:], body: body) } let buffer = Self.randomBuffer(size: 54003) - let data = Data(buffer: buffer) - let fileURL = URL(fileURLWithPath: "testReadMultipleFilesOnSameConnection.jpg") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } let app = Application(responder: router.buildResponder()) - try await app.test(.live) { client in - try await client.execute(uri: "/test.jpg", method: .get) { response in - XCTAssertEqual(response.body, buffer) - } - try await client.execute(uri: "/test.jpg", method: .get) { response in - XCTAssertEqual(response.body, buffer) + try await FileIOTests.withFile("testReadMultipleFilesOnSameConnection.jpg", contents: buffer.readableBytesView) { + try await app.test(.live) { client in + try await client.execute(uri: "/test.jpg", method: .get) { response in + XCTAssertEqual(response.body, buffer) + } + try await client.execute(uri: "/test.jpg", method: .get) { response in + XCTAssertEqual(response.body, buffer) + } } } } @@ -87,10 +103,11 @@ final class FileIOTests: XCTestCase { } } - let fileURL = URL(fileURLWithPath: filename) - let data = try Data(contentsOf: fileURL) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - XCTAssertEqual(String(decoding: data, as: Unicode.UTF8.self), "This is a test") + let contents = try await FileSystem.shared.withFileHandle(forReadingAt: .init(filename)) { read in + try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .megabytes(1000)) + } + try await FileSystem.shared.removeItem(at: .init(filename)) + XCTAssertEqual(String(buffer: contents), "This is a test") } func testWriteLargeFile() async throws { @@ -109,10 +126,11 @@ final class FileIOTests: XCTestCase { XCTAssertEqual(response.status, .ok) } - let fileURL = URL(fileURLWithPath: filename) - let data = try Data(contentsOf: fileURL) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - XCTAssertEqual(Data(buffer: buffer), data) + let contents = try await FileSystem.shared.withFileHandle(forReadingAt: .init(filename)) { read in + try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .megabytes(1000)) + } + try await FileSystem.shared.removeItem(at: .init(filename)) + XCTAssertEqual(contents, buffer) } } @@ -123,16 +141,15 @@ final class FileIOTests: XCTestCase { let body = try await fileIO.loadFile(path: "empty.txt", context: context) return .init(status: .ok, headers: [:], body: body) } - let data = Data() - let fileURL = URL(fileURLWithPath: "empty.txt") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } - let app = Application(responder: router.buildResponder()) - try await app.test(.router) { client in - try await client.execute(uri: "/empty.txt", method: .get) { response in - XCTAssertEqual(response.status, .ok) + let buffer = ByteBuffer() + + try await FileIOTests.withFile("empty.txt", contents: buffer.readableBytesView) { + try await app.test(.router) { client in + try await client.execute(uri: "/empty.txt", method: .get) { response in + XCTAssertEqual(response.status, .ok) + } } } } @@ -144,16 +161,16 @@ final class FileIOTests: XCTestCase { let body = try await fileIO.loadFile(path: "empty.txt", range: 0...10, context: context) return .init(status: .ok, headers: [:], body: body) } - let data = Data() - let fileURL = URL(fileURLWithPath: "empty.txt") - XCTAssertNoThrow(try data.write(to: fileURL)) - defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) } let app = Application(responder: router.buildResponder()) - try await app.test(.router) { client in - try await client.execute(uri: "/empty.txt", method: .get) { response in - XCTAssertEqual(response.status, .ok) + let buffer = ByteBuffer() + + try await FileIOTests.withFile("empty.txt", contents: buffer.readableBytesView) { + try await app.test(.router) { client in + try await client.execute(uri: "/empty.txt", method: .get) { response in + XCTAssertEqual(response.status, .ok) + } } } } diff --git a/Tests/HummingbirdTests/FileMiddlewareTests.swift b/Tests/HummingbirdTests/FileMiddlewareTests.swift index accdee59..7d95897a 100644 --- a/Tests/HummingbirdTests/FileMiddlewareTests.swift +++ b/Tests/HummingbirdTests/FileMiddlewareTests.swift @@ -35,25 +35,6 @@ final class FileMiddlewareTests: XCTestCase { return formatter } - static func withFile( - _ path: String, - contents: Buffer, - process: () async throws -> ReturnValue - ) async throws -> ReturnValue where Buffer.Element == UInt8 { - let fileSystem = FileSystem(threadPool: .singleton) - try await fileSystem.withFileHandle(forWritingAt: .init(path)) { write in - _ = try await write.write(contentsOf: contents, toAbsoluteOffset: 0) - } - do { - let value = try await process() - _ = try? await fileSystem.removeItem(at: .init(path)) - return value - } catch { - _ = try? await fileSystem.removeItem(at: .init(path)) - throw error - } - } - func testRead() async throws { let router = Router() router.middlewares.add(FileMiddleware(".")) @@ -62,7 +43,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).jpg" let text = "Test file contents" - try await Self.withFile(filename, contents: text.utf8) { + try await FileIOTests.withFile(filename, contents: text.utf8) { try await app.test(.router) { client in try await client.execute(uri: filename, method: .get) { response in XCTAssertEqual(String(buffer: response.body), text) @@ -92,7 +73,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 380_000) - try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await FileIOTests.withFile(filename, contents: buffer.readableBytesView) { try await app.test(.router) { client in try await client.execute(uri: filename, method: .get) { response in XCTAssertEqual(response.body, buffer) @@ -109,7 +90,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 326_000) - try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await FileIOTests.withFile(filename, contents: buffer.readableBytesView) { try await app.test(.router) { client in try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=100-3999"]) { response in let slice = buffer.getSlice(at: 100, length: 3900) @@ -152,7 +133,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 10000) - try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await FileIOTests.withFile(filename, contents: buffer.readableBytesView) { try await app.test(.router) { client in let (eTag, modificationDate) = try await client.execute(uri: filename, method: .get, headers: [.range: "bytes=-3999"]) { response -> (String, String) in @@ -188,7 +169,7 @@ final class FileMiddlewareTests: XCTestCase { let date = Date() let text = "Test file contents" - try await Self.withFile(filename, contents: text.utf8) { + try await FileIOTests.withFile(filename, contents: text.utf8) { try await app.test(.router) { client in let filename = "testHead.txt" try await client.execute(uri: "/\(filename)", method: .head) { response in @@ -211,7 +192,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 16200) - try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await FileIOTests.withFile(filename, contents: buffer.readableBytesView) { try await app.test(.router) { client in var eTag: String? try await client.execute(uri: filename, method: .head) { response in @@ -232,7 +213,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 16200) - try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await FileIOTests.withFile(filename, contents: buffer.readableBytesView) { try await app.test(.router) { client in let eTag = try await client.execute(uri: filename, method: .head) { response in try XCTUnwrap(response.headers[.eTag]) @@ -260,7 +241,7 @@ final class FileMiddlewareTests: XCTestCase { let filename = "\(#function).txt" let buffer = Self.randomBuffer(size: 16200) - try await Self.withFile(filename, contents: buffer.readableBytesView) { + try await FileIOTests.withFile(filename, contents: buffer.readableBytesView) { try await app.test(.router) { client in let modifiedDate = try await client.execute(uri: filename, method: .head) { response in try XCTUnwrap(response.headers[.lastModified]) @@ -290,8 +271,8 @@ final class FileMiddlewareTests: XCTestCase { let text = "Test file contents" let filename2 = "\(#function).jpg" - try await Self.withFile(filename, contents: text.utf8) { - try await Self.withFile(filename2, contents: text.utf8) { + try await FileIOTests.withFile(filename, contents: text.utf8) { + try await FileIOTests.withFile(filename2, contents: text.utf8) { try await app.test(.router) { client in try await client.execute(uri: filename, method: .get) { response in XCTAssertEqual(response.headers[.cacheControl], "max-age=2592000") @@ -311,7 +292,7 @@ final class FileMiddlewareTests: XCTestCase { let text = "Test file contents" - try await Self.withFile("index.html", contents: text.utf8) { + try await FileIOTests.withFile("index.html", contents: text.utf8) { try await app.test(.router) { client in try await client.execute(uri: "/", method: .get) { response in XCTAssertEqual(String(buffer: response.body), text) @@ -327,7 +308,7 @@ final class FileMiddlewareTests: XCTestCase { let text = "Test file contents" - try await Self.withFile("test.html", contents: text.utf8) { + try await FileIOTests.withFile("test.html", contents: text.utf8) { try await app.test(.router) { client in try await client.execute(uri: "/test.html", method: .get) { response in XCTAssertEqual(String(buffer: response.body), text) @@ -371,7 +352,7 @@ final class FileMiddlewareTests: XCTestCase { let text = "Test file contents" - try await Self.withFile("index.html", contents: text.utf8) { + try await FileIOTests.withFile("index.html", contents: text.utf8) { try await app.test(.router) { client in try await client.execute(uri: "/", method: .get) { response in XCTAssertEqual(String(buffer: response.body), text) From eaf04aa88301d12db21b7046270d3404da5e93b4 Mon Sep 17 00:00:00 2001 From: Adam Fowler Date: Tue, 14 Jan 2025 09:11:23 +0000 Subject: [PATCH 4/4] Use NIOFileSystem to load dotEnv --- Sources/Hummingbird/Environment.swift | 15 ++++----------- Tests/HummingbirdTests/FileIOTests.swift | 4 ++-- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Sources/Hummingbird/Environment.swift b/Sources/Hummingbird/Environment.swift index dabe9e0a..f68ab848 100644 --- a/Sources/Hummingbird/Environment.swift +++ b/Sources/Hummingbird/Environment.swift @@ -14,6 +14,7 @@ import HummingbirdCore import NIOCore +import _NIOFileSystem #if canImport(FoundationEssentials) import FoundationEssentials @@ -136,18 +137,10 @@ public struct Environment: Sendable, Decodable, ExpressibleByDictionaryLiteral { /// Load `.env` file into string internal static func loadDotEnv(_ dovEnvPath: String = ".env") async -> String? { do { - let fileHandle = try NIOFileHandle(path: dovEnvPath) - defer { - try? fileHandle.close() + return try await FileSystem.shared.withFileHandle(forReadingAt: .init(dovEnvPath)) { fileHandle in + let buffer = try await fileHandle.readToEnd(maximumSizeAllowed: .unlimited) + return String(buffer: buffer) } - let fileRegion = try FileRegion(fileHandle: fileHandle) - let contents = try fileHandle.withUnsafeFileDescriptor { descriptor in - [UInt8](unsafeUninitializedCapacity: fileRegion.readableBytes) { bytes, size in - size = fileRegion.readableBytes - read(descriptor, .init(bytes.baseAddress), size) - } - } - return String(bytes: contents, encoding: .utf8) } catch { return nil } diff --git a/Tests/HummingbirdTests/FileIOTests.swift b/Tests/HummingbirdTests/FileIOTests.swift index 52f20da5..331e53f8 100644 --- a/Tests/HummingbirdTests/FileIOTests.swift +++ b/Tests/HummingbirdTests/FileIOTests.swift @@ -104,7 +104,7 @@ final class FileIOTests: XCTestCase { } let contents = try await FileSystem.shared.withFileHandle(forReadingAt: .init(filename)) { read in - try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .megabytes(1000)) + try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .unlimited) } try await FileSystem.shared.removeItem(at: .init(filename)) XCTAssertEqual(String(buffer: contents), "This is a test") @@ -127,7 +127,7 @@ final class FileIOTests: XCTestCase { } let contents = try await FileSystem.shared.withFileHandle(forReadingAt: .init(filename)) { read in - try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .megabytes(1000)) + try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .unlimited) } try await FileSystem.shared.removeItem(at: .init(filename)) XCTAssertEqual(contents, buffer)