fix(sidebar): read a linked SQL file's header in the encoding the whole file uses (#3083)

Author: datlechin

CHANGELOG.md

@@ -100,6 +100,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Global saved query inside a folder missing from every other connection. (#3045)
 - Saved query and folder drawn nowhere when the folder holding it was gone.
 - Keyword accepted for a global saved query while another connection already held it.
+- Garbled name and ISO-8859-1 label on a UTF-8, UTF-16 or UTF-32 linked SQL file, and garbled big-endian UTF-32 files.
 - Cleared keyword, folder or **Global** on a saved query or its folder never reaching another device.
 - Renaming a folder putting back the scope another window had just set.
 - Saved queries and their folders deleted at launch when their connection had not arrived from iCloud.

TablePro/Core/Utilities/File/FileTextLoader.swift

@@ -22,6 +22,9 @@ internal enum FileTextLoader {
 
     static func load(_ url: URL) -> LoadedText? {
         let modifiedAt = modificationDate(of: url)
+        if startsWithByteOrderMark(url) {
+            return loadByteOrderMarked(url, modifiedAt: modifiedAt)
+        }
         var detected: String.Encoding = .utf8
         if let content = try? String(contentsOf: url, usedEncoding: &detected) {
             return LoadedText(content: content, encoding: detected, modifiedAt: modifiedAt)
@@ -36,20 +39,8 @@ internal enum FileTextLoader {
     }
 
     static func decode(_ data: Data) -> String? {
-        let utf8ByteOrderMark: [UInt8] = [0xEF, 0xBB, 0xBF]
-        if data.starts(with: utf8ByteOrderMark) {
-            return String(data: data.dropFirst(utf8ByteOrderMark.count), encoding: .utf8)
-        }
-        if data.starts(with: [0xFF, 0xFE, 0x00, 0x00]) || data.starts(with: [0x00, 0x00, 0xFE, 0xFF]) {
-            return String(data: data, encoding: .utf32)
-        }
-        if data.starts(with: [0xFF, 0xFE]) || data.starts(with: [0xFE, 0xFF]) {
-            return String(data: data, encoding: .utf16)
-        }
-        if let content = String(data: data, encoding: .utf8) {
-            return content
-        }
-        return String(data: data, encoding: .isoLatin1)
+        guard !data.isEmpty else { return "" }
+        return TextPrefixDecoder.decode(data, prefixLength: data.count)?.content
     }
 
     static func modificationDate(of url: URL) -> Date? {
@@ -59,16 +50,23 @@ internal enum FileTextLoader {
     static func loadHeader(_ url: URL, maxBytes: Int = 4_096) -> LoadedText? {
         guard let handle = try? FileHandle(forReadingFrom: url) else { return nil }
         defer { try? handle.close() }
-        guard let data = try? handle.read(upToCount: maxBytes), !data.isEmpty else { return nil }
+        guard let bytes = try? handle.read(upToCount: maxBytes + TextPrefixDecoder.lookaheadLength),
+              let decoded = TextPrefixDecoder.decode(bytes, prefixLength: maxBytes) else { return nil }
         let modifiedAt = modificationDate(of: url)
+        return LoadedText(content: decoded.content, encoding: decoded.encoding, modifiedAt: modifiedAt)
+    }
 
-        if let content = String(data: data, encoding: .utf8) {
-            return LoadedText(content: content, encoding: .utf8, modifiedAt: modifiedAt)
-        }
-        if let content = String(data: data, encoding: .isoLatin1) {
-            return LoadedText(content: content, encoding: .isoLatin1, modifiedAt: modifiedAt)
-        }
-        return nil
+    private static func startsWithByteOrderMark(_ url: URL) -> Bool {
+        guard let handle = try? FileHandle(forReadingFrom: url) else { return false }
+        defer { try? handle.close() }
+        guard let bytes = try? handle.read(upToCount: ByteOrderMark.longestLength) else { return false }
+        return ByteOrderMark.leading(bytes) != nil
+    }
+
+    private static func loadByteOrderMarked(_ url: URL, modifiedAt: Date?) -> LoadedText? {
+        guard let bytes = try? Data(contentsOf: url),
+              let decoded = TextPrefixDecoder.decode(bytes, prefixLength: bytes.count) else { return nil }
+        return LoadedText(content: decoded.content, encoding: decoded.encoding, modifiedAt: modifiedAt)
     }
 }
 

TablePro/Core/Utilities/File/TextPrefixDecoder.swift

@@ -0,0 +1,46 @@
+//
+//  TextPrefixDecoder.swift
+//  TablePro
+//
+
+import Foundation
+
+internal enum TextPrefixDecoder {
+    struct Decoded {
+        let content: String
+        let encoding: String.Encoding
+    }
+
+    static let lookaheadLength = 3
+
+    static func decode(_ bytes: Data, prefixLength: Int) -> Decoded? {
+        guard !bytes.isEmpty, prefixLength > 0 else { return nil }
+        let mark = ByteOrderMark.leading(bytes)
+        let encoding = mark?.encoding ?? .utf8
+        let codeUnitLength = mark?.codeUnitLength ?? 1
+        if let content = decode(bytes, prefixLength: prefixLength, as: encoding, codeUnitLength: codeUnitLength) {
+            return Decoded(content: content, encoding: encoding)
+        }
+        guard let content = String(data: bytes.prefix(prefixLength), encoding: .isoLatin1) else { return nil }
+        return Decoded(content: content, encoding: .isoLatin1)
+    }
+
+    private static func decode(
+        _ bytes: Data,
+        prefixLength: Int,
+        as encoding: String.Encoding,
+        codeUnitLength: Int
+    ) -> String? {
+        for end in candidateEnds(of: bytes, prefixLength: prefixLength) where end.isMultiple(of: codeUnitLength) {
+            if let content = String(data: bytes.prefix(end), encoding: encoding) {
+                return content
+            }
+        }
+        return nil
+    }
+
+    private static func candidateEnds(of bytes: Data, prefixLength: Int) -> ClosedRange<Int> {
+        guard bytes.count > prefixLength else { return bytes.count...bytes.count }
+        return prefixLength...min(bytes.count, prefixLength + lookaheadLength)
+    }
+}

TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift

@@ -92,40 +92,22 @@ struct SQLChunkDecoder {
         let byteOrderMarkLength: Int
     }
 
-    private static let utf16LittleEndianMark: [UInt8] = [0xFF, 0xFE]
-    private static let utf16BigEndianMark: [UInt8] = [0xFE, 0xFF]
-    private static let utf32LittleEndianMark: [UInt8] = [0xFF, 0xFE, 0x00, 0x00]
-    private static let utf32BigEndianMark: [UInt8] = [0x00, 0x00, 0xFE, 0xFF]
-
     /// `.utf8` is absent on purpose: Foundation consumes a UTF-8 mark itself, measured.
     private static func resolve(_ encoding: String.Encoding, startingWith data: Data) -> Resolution {
+        if let mark = ByteOrderMark.leading(data, allowedBy: encoding) {
+            return Resolution(encoding: mark.byteOrderedEncoding, byteOrderMarkLength: mark.length)
+        }
+        return Resolution(encoding: unmarkedByteOrder(of: encoding), byteOrderMarkLength: 0)
+    }
+
+    private static func unmarkedByteOrder(of encoding: String.Encoding) -> String.Encoding {
         switch encoding {
         case .utf16:
-            if data.starts(with: utf16LittleEndianMark) {
-                return Resolution(encoding: .utf16LittleEndian, byteOrderMarkLength: 2)
-            }
-            if data.starts(with: utf16BigEndianMark) {
-                return Resolution(encoding: .utf16BigEndian, byteOrderMarkLength: 2)
-            }
-            return Resolution(encoding: .utf16BigEndian, byteOrderMarkLength: 0)
-        case .utf16LittleEndian:
-            return Resolution(encoding: encoding, byteOrderMarkLength: data.starts(with: utf16LittleEndianMark) ? 2 : 0)
-        case .utf16BigEndian:
-            return Resolution(encoding: encoding, byteOrderMarkLength: data.starts(with: utf16BigEndianMark) ? 2 : 0)
+            return .utf16BigEndian
         case .utf32:
-            if data.starts(with: utf32LittleEndianMark) {
-                return Resolution(encoding: .utf32LittleEndian, byteOrderMarkLength: 4)
-            }
-            if data.starts(with: utf32BigEndianMark) {
-                return Resolution(encoding: .utf32BigEndian, byteOrderMarkLength: 4)
-            }
-            return Resolution(encoding: .utf32BigEndian, byteOrderMarkLength: 0)
-        case .utf32LittleEndian:
-            return Resolution(encoding: encoding, byteOrderMarkLength: data.starts(with: utf32LittleEndianMark) ? 4 : 0)
-        case .utf32BigEndian:
-            return Resolution(encoding: encoding, byteOrderMarkLength: data.starts(with: utf32BigEndianMark) ? 4 : 0)
+            return .utf32BigEndian
         default:
-            return Resolution(encoding: encoding, byteOrderMarkLength: 0)
+            return encoding
         }
     }
 

TablePro/Core/Utilities/Text/ByteOrderMark.swift

@@ -0,0 +1,65 @@
+//
+//  ByteOrderMark.swift
+//  TablePro
+//
+
+import Foundation
+
+internal enum ByteOrderMark {
+    case utf32LittleEndian
+    case utf32BigEndian
+    case utf16LittleEndian
+    case utf16BigEndian
+
+    private static let longestFirst: [ByteOrderMark] = [
+        .utf32LittleEndian, .utf32BigEndian, .utf16LittleEndian, .utf16BigEndian
+    ]
+
+    static var longestLength: Int { longestFirst.map(\.length).max() ?? 0 }
+
+    static func leading(_ bytes: Data) -> ByteOrderMark? {
+        longestFirst.first { bytes.starts(with: $0.bytes) }
+    }
+
+    static func leading(_ bytes: Data, allowedBy declaredEncoding: String.Encoding) -> ByteOrderMark? {
+        longestFirst.first { $0.isAllowed(by: declaredEncoding) && bytes.starts(with: $0.bytes) }
+    }
+
+    var bytes: [UInt8] {
+        switch self {
+        case .utf32LittleEndian: return [0xFF, 0xFE, 0x00, 0x00]
+        case .utf32BigEndian: return [0x00, 0x00, 0xFE, 0xFF]
+        case .utf16LittleEndian: return [0xFF, 0xFE]
+        case .utf16BigEndian: return [0xFE, 0xFF]
+        }
+    }
+
+    var length: Int { bytes.count }
+
+    var encoding: String.Encoding {
+        switch self {
+        case .utf32LittleEndian, .utf32BigEndian: return .utf32
+        case .utf16LittleEndian, .utf16BigEndian: return .utf16
+        }
+    }
+
+    var byteOrderedEncoding: String.Encoding {
+        switch self {
+        case .utf32LittleEndian: return .utf32LittleEndian
+        case .utf32BigEndian: return .utf32BigEndian
+        case .utf16LittleEndian: return .utf16LittleEndian
+        case .utf16BigEndian: return .utf16BigEndian
+        }
+    }
+
+    var codeUnitLength: Int {
+        switch self {
+        case .utf32LittleEndian, .utf32BigEndian: return 4
+        case .utf16LittleEndian, .utf16BigEndian: return 2
+        }
+    }
+
+    private func isAllowed(by declaredEncoding: String.Encoding) -> Bool {
+        declaredEncoding == encoding || declaredEncoding == byteOrderedEncoding
+    }
+}

TableProTests/Core/Git/GitOutputParserTests.swift

@@ -299,4 +299,12 @@ struct FileTextLoaderDecodeTests {
         #expect(FileTextLoader.decode(Data(text.utf8)) == text)
         #expect(FileTextLoader.decode(Data([0x53, 0xE9])) == "Sé")
     }
+
+    @Test("A big-endian UTF-32 blob and an empty blob decode")
+    func bigEndianAndEmpty() throws {
+        let text = "SELECT 'é';"
+        let utf32 = try #require(text.data(using: .utf32BigEndian))
+        #expect(FileTextLoader.decode(Data([0x00, 0x00, 0xFE, 0xFF]) + utf32) == text)
+        #expect(FileTextLoader.decode(Data()) == "")
+    }
 }