Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use NIOFileSystem #655

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

Use NIOFileSystem #655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8610af1
Replace all uses of NonBlockingFileIO
adam-fowler Jan 13, 2025
4395e55
Tidy up tests
adam-fowler Jan 13, 2025
3fd4003
Update tests
adam-fowler Jan 13, 2025
22fba04
Merge branch 'main' into nio-file-system
adam-fowler Jan 13, 2025
eaf04aa
Use NIOFileSystem to load dotEnv
adam-fowler Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Package.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ let package = Package(
.package(url: "https://github.com/apple/swift-http-types.git", from: "1.0.0"),
.package(url: "https://github.com/apple/swift-metrics.git", from: "2.5.0"),
.package(url: "https://github.com/apple/swift-distributed-tracing.git", from: "1.1.0"),
.package(url: "https://github.com/apple/swift-nio.git", from: "2.63.0"),
.package(url: "https://github.com/apple/swift-nio.git", from: "2.78.0"),
.package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.20.0"),
.package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.34.1"),
.package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.0"),
Expand All @@ -46,6 +46,7 @@ let package = Package(
.product(name: "Metrics", package: "swift-metrics"),
.product(name: "Tracing", package: "swift-distributed-tracing"),
.product(name: "NIOCore", package: "swift-nio"),
.product(name: "_NIOFileSystem", package: "swift-nio"),
.product(name: "NIOPosix", package: "swift-nio"),
],
swiftSettings: swiftSettings
Expand Down
15 changes: 4 additions & 11 deletions Sources/Hummingbird/Environment.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import HummingbirdCore
import NIOCore
import _NIOFileSystem

#if canImport(FoundationEssentials)
import FoundationEssentials
Expand Down Expand Up @@ -136,18 +137,10 @@ public struct Environment: Sendable, Decodable, ExpressibleByDictionaryLiteral {
/// Load `.env` file into string
internal static func loadDotEnv(_ dovEnvPath: String = ".env") async -> String? {
do {
let fileHandle = try NIOFileHandle(path: dovEnvPath)
defer {
try? fileHandle.close()
return try await FileSystem.shared.withFileHandle(forReadingAt: .init(dovEnvPath)) { fileHandle in
let buffer = try await fileHandle.readToEnd(maximumSizeAllowed: .unlimited)
return String(buffer: buffer)
}
let fileRegion = try FileRegion(fileHandle: fileHandle)
let contents = try fileHandle.withUnsafeFileDescriptor { descriptor in
[UInt8](unsafeUninitializedCapacity: fileRegion.readableBytes) { bytes, size in
size = fileRegion.readableBytes
read(descriptor, .init(bytes.baseAddress), size)
}
}
return String(bytes: contents, encoding: .utf8)
} catch {
return nil
}
Expand Down
80 changes: 36 additions & 44 deletions Sources/Hummingbird/Files/FileIO.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,24 @@ import HummingbirdCore
import Logging
import NIOCore
import NIOPosix
import _NIOFileSystem

/// Manages File reading and writing.
public struct FileIO: Sendable {
let fileIO: NonBlockingFileIO
struct FileError: Error {
internal enum Value {
case fileDoesNotExist
}
internal let value: Value

static var fileDoesNotExist: Self { .init(value: .fileDoesNotExist) }
}
let fileSystem: FileSystem

/// Initialize FileIO
/// - Parameter threadPool: ThreadPool to use for file operations
public init(threadPool: NIOThreadPool = .singleton) {
self.fileIO = .init(threadPool: threadPool)
self.fileSystem = .init(threadPool: threadPool)
}

/// Load file and return response body
Expand All @@ -39,12 +48,12 @@ public struct FileIO: Sendable {
public func loadFile(
path: String,
context: some RequestContext,
chunkLength: Int = NonBlockingFileIO.defaultChunkSize
chunkLength: Int = 128 * 1024
) async throws -> ResponseBody {
do {
let stat = try await fileIO.stat(path: path)
guard stat.st_size > 0 else { return .init() }
return self.readFile(path: path, range: 0...numericCast(stat.st_size - 1), context: context, chunkLength: chunkLength)
guard let info = try await self.fileSystem.info(forFileAt: .init(path)) else { throw FileError.fileDoesNotExist }
guard info.size > 0 else { return .init() }
return self.readFile(path: path, range: 0...numericCast(info.size - 1), context: context, chunkLength: chunkLength)
} catch {
throw HTTPError(.notFound)
}
Expand All @@ -64,12 +73,12 @@ public struct FileIO: Sendable {
path: String,
range: ClosedRange<Int>,
context: some RequestContext,
chunkLength: Int = NonBlockingFileIO.defaultChunkSize
chunkLength: Int = 128 * 1024
) async throws -> ResponseBody {
do {
let stat = try await fileIO.stat(path: path)
guard stat.st_size > 0 else { return .init() }
let fileRange: ClosedRange<Int> = 0...numericCast(stat.st_size - 1)
guard let info = try await self.fileSystem.info(forFileAt: .init(path)) else { throw FileError.fileDoesNotExist }
guard info.size > 0 else { return .init() }
let fileRange: ClosedRange<Int> = 0...numericCast(info.size - 1)
let range = range.clamped(to: fileRange)
return self.readFile(path: path, range: range, context: context, chunkLength: chunkLength)
} catch {
Expand All @@ -89,9 +98,12 @@ public struct FileIO: Sendable {
context: some RequestContext
) async throws where AS.Element == ByteBuffer {
context.logger.debug("[FileIO] PUT", metadata: ["hb.file.path": .string(path)])
try await self.fileIO.withFileHandle(path: path, mode: .write, flags: .allowFileCreation()) { handle in
for try await buffer in contents {
try await self.fileIO.write(fileHandle: handle, buffer: buffer)
try await self.fileSystem.withFileHandle(
forWritingAt: .init(path),
options: .newFile(replaceExisting: true)
) { fileHandle in
try await fileHandle.withBufferedWriter { writer in
_ = try await writer.write(contentsOf: contents)
}
}
}
Expand All @@ -108,8 +120,11 @@ public struct FileIO: Sendable {
context: some RequestContext
) async throws {
context.logger.debug("[FileIO] PUT", metadata: ["hb.file.path": .string(path)])
try await self.fileIO.withFileHandle(path: path, mode: .write, flags: .allowFileCreation()) { handle in
try await self.fileIO.write(fileHandle: handle, buffer: buffer)
try await self.fileSystem.withFileHandle(
forWritingAt: .init(path),
options: .newFile(replaceExisting: true)
) { fileHandle in
_ = try await fileHandle.write(contentsOf: buffer, toAbsoluteOffset: 0)
}
}

Expand All @@ -118,41 +133,18 @@ public struct FileIO: Sendable {
path: String,
range: ClosedRange<Int>,
context: some RequestContext,
chunkLength: Int = NonBlockingFileIO.defaultChunkSize
chunkLength: Int
) -> ResponseBody {
ResponseBody(contentLength: range.count) { writer in
try await self.fileIO.withFileHandle(path: path, mode: .read) { handle in
let endOffset = range.endIndex
let chunkLength = chunkLength
var fileOffset = range.startIndex
let allocator = ByteBufferAllocator()
try await self.fileSystem.withFileHandle(forReadingAt: .init(path)) { fileHandle in
let startOffset: Int64 = numericCast(range.lowerBound)
let endOffset: Int64 = numericCast(range.upperBound)

while case .inRange(let offset) = fileOffset {
let bytesLeft = range.distance(from: fileOffset, to: endOffset)
let bytesToRead = Swift.min(chunkLength, bytesLeft)
let buffer = try await self.fileIO.read(
fileHandle: handle,
fromOffset: numericCast(offset),
byteCount: bytesToRead,
allocator: allocator
)
fileOffset = range.index(fileOffset, offsetBy: bytesToRead)
try await writer.write(buffer)
for try await chunk in fileHandle.readChunks(in: startOffset...endOffset, chunkLength: .bytes(numericCast(chunkLength))) {
try await writer.write(chunk)
}
try await writer.finish(nil)
}
}
}
}

extension NonBlockingFileIO {
func stat(path: String) async throws -> stat {
let stat = try await self.lstat(path: path)
if stat.st_mode & S_IFMT == S_IFLNK {
let realPath = try await self.readlink(path: path)
return try await self.lstat(path: realPath)
} else {
return stat
}
}
}
12 changes: 4 additions & 8 deletions Sources/Hummingbird/Files/LocalFileSystem.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,12 @@ public struct LocalFileSystem: FileProvider {
/// - Returns: File attributes
public func getAttributes(id path: FileIdentifier) async throws -> FileAttributes? {
do {
let stat = try await self.fileIO.fileIO.stat(path: path)
let isFolder = (stat.st_mode & S_IFMT) == S_IFDIR
#if os(Linux)
let modificationDate = Double(stat.st_mtim.tv_sec) + (Double(stat.st_mtim.tv_nsec) / 1_000_000_000.0)
#else
let modificationDate = Double(stat.st_mtimespec.tv_sec) + (Double(stat.st_mtimespec.tv_nsec) / 1_000_000_000.0)
#endif
guard let info = try await self.fileIO.fileSystem.info(forFileAt: .init(path)) else { throw FileIO.FileError.fileDoesNotExist }
let isFolder = info.type == .directory
let modificationDate = Double(info.lastDataModificationTime.seconds)
return .init(
isFolder: isFolder,
size: numericCast(stat.st_size),
size: numericCast(info.size),
modificationDate: Date(timeIntervalSince1970: modificationDate)
)
} catch {
Expand Down
101 changes: 59 additions & 42 deletions Tests/HummingbirdTests/FileIOTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import Hummingbird
import HummingbirdTesting
import XCTest
import _NIOFileSystem

final class FileIOTests: XCTestCase {
static func randomBuffer(size: Int) -> ByteBuffer {
Expand All @@ -23,24 +24,41 @@ final class FileIOTests: XCTestCase {
return ByteBufferAllocator().buffer(bytes: data)
}

static func withFile<Buffer: Sequence & Sendable, ReturnValue>(
_ path: String,
contents: Buffer,
process: () async throws -> ReturnValue
) async throws -> ReturnValue where Buffer.Element == UInt8 {
let fileSystem = FileSystem(threadPool: .singleton)
try await fileSystem.withFileHandle(forWritingAt: .init(path)) { write in
_ = try await write.write(contentsOf: contents, toAbsoluteOffset: 0)
}
do {
let value = try await process()
_ = try? await fileSystem.removeItem(at: .init(path))
return value
} catch {
_ = try? await fileSystem.removeItem(at: .init(path))
throw error
}
}

func testReadFileIO() async throws {
let router = Router()
router.get("test.jpg") { _, context -> Response in
let fileIO = FileIO(threadPool: .singleton)
let body = try await fileIO.loadFile(path: "testReadFileIO.jpg", context: context)
return .init(status: .ok, headers: [:], body: body)
}
let buffer = Self.randomBuffer(size: 320_003)
let data = Data(buffer: buffer)
let fileURL = URL(fileURLWithPath: "testReadFileIO.jpg")
XCTAssertNoThrow(try data.write(to: fileURL))
defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) }

let app = Application(responder: router.buildResponder())

try await app.test(.router) { client in
try await client.execute(uri: "/test.jpg", method: .get) { response in
XCTAssertEqual(response.body, buffer)
let buffer = Self.randomBuffer(size: 320_003)

try await FileIOTests.withFile("testReadFileIO.jpg", contents: buffer.readableBytesView) {
try await app.test(.router) { client in
try await client.execute(uri: "/test.jpg", method: .get) { response in
XCTAssertEqual(response.body, buffer)
}
}
}
}
Expand All @@ -53,19 +71,17 @@ final class FileIOTests: XCTestCase {
return .init(status: .ok, headers: [:], body: body)
}
let buffer = Self.randomBuffer(size: 54003)
let data = Data(buffer: buffer)
let fileURL = URL(fileURLWithPath: "testReadMultipleFilesOnSameConnection.jpg")
XCTAssertNoThrow(try data.write(to: fileURL))
defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) }

let app = Application(responder: router.buildResponder())

try await app.test(.live) { client in
try await client.execute(uri: "/test.jpg", method: .get) { response in
XCTAssertEqual(response.body, buffer)
}
try await client.execute(uri: "/test.jpg", method: .get) { response in
XCTAssertEqual(response.body, buffer)
try await FileIOTests.withFile("testReadMultipleFilesOnSameConnection.jpg", contents: buffer.readableBytesView) {
try await app.test(.live) { client in
try await client.execute(uri: "/test.jpg", method: .get) { response in
XCTAssertEqual(response.body, buffer)
}
try await client.execute(uri: "/test.jpg", method: .get) { response in
XCTAssertEqual(response.body, buffer)
}
}
}
}
Expand All @@ -87,10 +103,11 @@ final class FileIOTests: XCTestCase {
}
}

let fileURL = URL(fileURLWithPath: filename)
let data = try Data(contentsOf: fileURL)
defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) }
XCTAssertEqual(String(decoding: data, as: Unicode.UTF8.self), "This is a test")
let contents = try await FileSystem.shared.withFileHandle(forReadingAt: .init(filename)) { read in
try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .unlimited)
}
try await FileSystem.shared.removeItem(at: .init(filename))
XCTAssertEqual(String(buffer: contents), "This is a test")
}

func testWriteLargeFile() async throws {
Expand All @@ -109,10 +126,11 @@ final class FileIOTests: XCTestCase {
XCTAssertEqual(response.status, .ok)
}

let fileURL = URL(fileURLWithPath: filename)
let data = try Data(contentsOf: fileURL)
defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) }
XCTAssertEqual(Data(buffer: buffer), data)
let contents = try await FileSystem.shared.withFileHandle(forReadingAt: .init(filename)) { read in
try await read.readToEnd(fromAbsoluteOffset: 0, maximumSizeAllowed: .unlimited)
}
try await FileSystem.shared.removeItem(at: .init(filename))
XCTAssertEqual(contents, buffer)
}
}

Expand All @@ -123,16 +141,15 @@ final class FileIOTests: XCTestCase {
let body = try await fileIO.loadFile(path: "empty.txt", context: context)
return .init(status: .ok, headers: [:], body: body)
}
let data = Data()
let fileURL = URL(fileURLWithPath: "empty.txt")
XCTAssertNoThrow(try data.write(to: fileURL))
defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) }

let app = Application(responder: router.buildResponder())

try await app.test(.router) { client in
try await client.execute(uri: "/empty.txt", method: .get) { response in
XCTAssertEqual(response.status, .ok)
let buffer = ByteBuffer()

try await FileIOTests.withFile("empty.txt", contents: buffer.readableBytesView) {
try await app.test(.router) { client in
try await client.execute(uri: "/empty.txt", method: .get) { response in
XCTAssertEqual(response.status, .ok)
}
}
}
}
Expand All @@ -144,16 +161,16 @@ final class FileIOTests: XCTestCase {
let body = try await fileIO.loadFile(path: "empty.txt", range: 0...10, context: context)
return .init(status: .ok, headers: [:], body: body)
}
let data = Data()
let fileURL = URL(fileURLWithPath: "empty.txt")
XCTAssertNoThrow(try data.write(to: fileURL))
defer { XCTAssertNoThrow(try FileManager.default.removeItem(at: fileURL)) }

let app = Application(responder: router.buildResponder())

try await app.test(.router) { client in
try await client.execute(uri: "/empty.txt", method: .get) { response in
XCTAssertEqual(response.status, .ok)
let buffer = ByteBuffer()

try await FileIOTests.withFile("empty.txt", contents: buffer.readableBytesView) {
try await app.test(.router) { client in
try await client.execute(uri: "/empty.txt", method: .get) { response in
XCTAssertEqual(response.status, .ok)
}
}
}
}
Expand Down
Loading
Loading