From 95142a1a6204da54ce5c075326ce990a006fe132 Mon Sep 17 00:00:00 2001 From: Ryc O'Chet Date: Mon, 5 Feb 2024 22:45:33 +0000 Subject: [PATCH 1/2] Add named custom dictionary support --- .../__snapshots__/index.test.ts.snap | 10 +++ src/cli.ts | 2 +- .../__snapshots__/index.test.ts.snap | 10 +++ src/custom/__tests__/custom.test.ts | 5 +- src/custom/compressToCustom.ts | 9 +++ src/custom/decompressFromCustom.ts | 12 ++- src/custom/dictionaries.ts | 77 +++++++++++++++++++ src/custom/index.ts | 12 +++ src/index.ts | 25 +++++- tsconfig.json | 4 +- 10 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 src/custom/dictionaries.ts diff --git a/src/__tests__/__snapshots__/index.test.ts.snap b/src/__tests__/__snapshots__/index.test.ts.snap index 28e0a36..030a32d 100644 --- a/src/__tests__/__snapshots__/index.test.ts.snap +++ b/src/__tests__/__snapshots__/index.test.ts.snap @@ -12,12 +12,22 @@ exports[`index.ts > was the change deliberate? 1`] = ` "compressToUint8Array": [Function], "convertFromUint8Array": [Function], "convertToUint8Array": [Function], + "customBase16Dict": "0123456789ABCDEF", + "customBase32Dict": "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", + "customBase32HexDict": "0123456789ABCDEEFGHIJKLMNOPQRSTU", + "customBase36Dict": "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "customBase58Dict": "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", + "customBase62Dict": "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", + "customBase64Dict": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=", + "customBase95Dict": " !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_\`abcdefghijklmnopqrstuvwxyz{|}~", + "customSafeDict": " !#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_abcdefghijklmnopqrstuvwxyz{|}~", "decompress": [Function], "decompressFromBase64": [Function], "decompressFromCustom": [Function], "decompressFromEncodedURIComponent": [Function], "decompressFromUTF16": [Function], "decompressFromUint8Array": [Function], + "getCustomDictionary": [Function], "loadBinaryFile": [Function], "saveBinaryFile": [Function], } diff --git a/src/cli.ts b/src/cli.ts index 90c7d31..3382912 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -19,7 +19,7 @@ program .description("Use lz-string to compress or decompress a file") .addOption(new Option("-d, --decompress", "if unset then this will compress")) .addOption(new Option("-e, --encoder ", "character encoding to use").choices(encoders).default("raw")) - .addOption(new Option("-c, --custom", "dictionary for custom encoder")) + .addOption(new Option("-c, --custom ", "dictionary for custom encoder")) .addOption(new Option("-v, --verify", "verify before returning").default(true)) .addOption(new Option("-o, --output ", "output file, otherwise write to stdout")) .addOption(new Option("-q, --quiet", "don't print any error messages")) diff --git a/src/custom/__tests__/__snapshots__/index.test.ts.snap b/src/custom/__tests__/__snapshots__/index.test.ts.snap index b2ea26a..e446139 100644 --- a/src/custom/__tests__/__snapshots__/index.test.ts.snap +++ b/src/custom/__tests__/__snapshots__/index.test.ts.snap @@ -3,6 +3,16 @@ exports[`custom/index.ts > was the change deliberate? 1`] = ` { "compressToCustom": [Function], + "customBase16Dict": "0123456789ABCDEF", + "customBase32Dict": "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", + "customBase32HexDict": "0123456789ABCDEEFGHIJKLMNOPQRSTU", + "customBase36Dict": "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "customBase58Dict": "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", + "customBase62Dict": "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", + "customBase64Dict": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=", + "customBase95Dict": " !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_\`abcdefghijklmnopqrstuvwxyz{|}~", + "customSafeDict": " !#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_abcdefghijklmnopqrstuvwxyz{|}~", "decompressFromCustom": [Function], + "getCustomDictionary": [Function], } `; diff --git a/src/custom/__tests__/custom.test.ts b/src/custom/__tests__/custom.test.ts index 698fdd8..e68a673 100644 --- a/src/custom/__tests__/custom.test.ts +++ b/src/custom/__tests__/custom.test.ts @@ -27,9 +27,8 @@ describe("custom", () => { }); describe("base62", () => { - const base62 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - const compress = (input: string | null) => compressToCustom(input, base62); - const decompress = (input: string | null) => decompressFromCustom(input, base62); + const compress = (input: string | null) => compressToCustom(input, "base62"); + const decompress = (input: string | null) => decompressFromCustom(input, "base62"); runTestSet("", compress, decompress); }); diff --git a/src/custom/compressToCustom.ts b/src/custom/compressToCustom.ts index 00112da..d8be4b3 100644 --- a/src/custom/compressToCustom.ts +++ b/src/custom/compressToCustom.ts @@ -5,10 +5,19 @@ */ import { compress } from "../raw/compress"; +import { getCustomDictionary } from "./dictionaries"; +/** + * Custom dictionary support, this is not ready for production use yet, so lock + * the versions if using. + * + * @alpha + */ export function compressToCustom(uncompressed: string | null, dict: string): string { if (uncompressed == null) return ""; + dict = getCustomDictionary(dict); + const compressed: string = compress(uncompressed); const charsPerUnicodeChar: number = Math.ceil(Math.log(65536) / Math.log(dict.length)); let res: string = ""; diff --git a/src/custom/decompressFromCustom.ts b/src/custom/decompressFromCustom.ts index eb431c4..39e3ac5 100644 --- a/src/custom/decompressFromCustom.ts +++ b/src/custom/decompressFromCustom.ts @@ -5,10 +5,20 @@ */ import { decompress } from "../raw/decompress"; +import { getCustomDictionary } from "./dictionaries"; -export function decompressFromCustom(compressed: string | null, dict: string): string | null { +/** + * Custom dictionary support, this is not ready for production use yet, so lock + * the versions if using. + * + * @alpha + */ +export function decompressFromCustom(compressed: string | null, dict: string) { if (compressed == null) return ""; if (compressed == "") return null; + + dict = getCustomDictionary(dict); + if (dict.length < 2) return null; const charsPerUnicodeChar: number = Math.ceil(Math.log(65536) / Math.log(dict.length)); diff --git a/src/custom/dictionaries.ts b/src/custom/dictionaries.ts new file mode 100644 index 0000000..fcfb612 --- /dev/null +++ b/src/custom/dictionaries.ts @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2013 Pieroxy + * + * SPDX-License-Identifier: MIT + */ + +import keyStrBase64 from "../base64/keyStrBase64"; + +/** + * Otherwise known as hexadecimal. + */ +export const customBase16Dict = "0123456789ABCDEF"; + +/** + * Base32 dictionary as defined in rfc4648: + * https://datatracker.ietf.org/doc/html/rfc4648#section-6 + */ +export const customBase32Dict = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +/** + * Base64 dictionary using the more well-known extended hex format: + * https://datatracker.ietf.org/doc/html/rfc4648#section-7 + */ +export const customBase32HexDict = "0123456789ABCDEEFGHIJKLMNOPQRSTU"; + +/** + * Base36 dictionary. + */ +export const customBase36Dict = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + +/** + * Base58 dictionary. + */ +export const customBase58Dict = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + +/** + * Base62 dictionary. + */ +export const customBase62Dict = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +/** + * Base64 dictionary: + * https://datatracker.ietf.org/doc/html/rfc4648#section-4 + */ +export const customBase64Dict = keyStrBase64; + +/** + * A simple dictionary for safe source-code inclusion. Effectively base91, as + * it contains base95 with all three quote types and the escape character + * removed. + */ +export const customSafeDict = + " !#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_abcdefghijklmnopqrstuvwxyz{|}~"; + +/** + * Base95 dictionary. + * (Two escaped characters in source). + */ +export const customBase95Dict = + " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"; + +/** + * If the dictionary passed is a known named dictionary then return that, + * otherwise return the source and assume it's correct. + */ +export const getCustomDictionary = (dict: string) => + ({ + safe: customSafeDict, + base16: customBase16Dict, + base32: customBase32Dict, + base32hex: customBase32HexDict, + base36: customBase36Dict, + base58: customBase58Dict, + base62: customBase62Dict, + base64: customBase64Dict, + base95: customBase95Dict, + }[dict]) || dict; diff --git a/src/custom/index.ts b/src/custom/index.ts index 376d38d..57ace6e 100644 --- a/src/custom/index.ts +++ b/src/custom/index.ts @@ -6,3 +6,15 @@ export { compressToCustom } from "./compressToCustom"; export { decompressFromCustom } from "./decompressFromCustom"; +export { + customSafeDict, + customBase16Dict, + customBase32Dict, + customBase32HexDict, + customBase36Dict, + customBase58Dict, + customBase62Dict, + customBase64Dict, + customBase95Dict, + getCustomDictionary, +} from "./dictionaries"; diff --git a/src/index.ts b/src/index.ts index 21e7876..a882303 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,20 @@ import { _compress } from "./_compress"; import { _decompress } from "./_decompress"; import { compressToBase64, decompressFromBase64 } from "./base64"; -import { compressToCustom, decompressFromCustom } from "./custom"; +import { + compressToCustom, + customBase16Dict, + customBase32Dict, + customBase32HexDict, + customBase36Dict, + customBase58Dict, + customBase62Dict, + customBase64Dict, + customBase95Dict, + customSafeDict, + decompressFromCustom, + getCustomDictionary, +} from "./custom"; import { compressToEncodedURIComponent, decompressFromEncodedURIComponent } from "./encodedURIComponent"; import { loadBinaryFile, saveBinaryFile } from "./node"; import { compress, decompress } from "./raw"; @@ -30,12 +43,22 @@ export default { compressToUTF16, convertFromUint8Array, convertToUint8Array, + customSafeDict, + customBase16Dict, + customBase32Dict, + customBase32HexDict, + customBase36Dict, + customBase58Dict, + customBase62Dict, + customBase64Dict, + customBase95Dict, decompress, decompressFromBase64, decompressFromCustom, decompressFromEncodedURIComponent, decompressFromUint8Array, decompressFromUTF16, + getCustomDictionary, loadBinaryFile, saveBinaryFile, }; diff --git a/tsconfig.json b/tsconfig.json index 9c58578..6f419c0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,7 +11,7 @@ "skipLibCheck": true, "sourceMap": true, "strict": true, - "target": "ESNext", + "target": "ESNext" }, - "include": ["src"], + "include": ["src"] } From f3735c736ae471b1332bf1ad777d7f1cca7473bd Mon Sep 17 00:00:00 2001 From: Ryc O'Chet Date: Mon, 5 Feb 2024 23:32:37 +0000 Subject: [PATCH 2/2] Correct base64 dictionary --- src/custom/dictionaries.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/custom/dictionaries.ts b/src/custom/dictionaries.ts index fcfb612..8de3c14 100644 --- a/src/custom/dictionaries.ts +++ b/src/custom/dictionaries.ts @@ -4,8 +4,6 @@ * SPDX-License-Identifier: MIT */ -import keyStrBase64 from "../base64/keyStrBase64"; - /** * Otherwise known as hexadecimal. */ @@ -42,7 +40,7 @@ export const customBase62Dict = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijk * Base64 dictionary: * https://datatracker.ietf.org/doc/html/rfc4648#section-4 */ -export const customBase64Dict = keyStrBase64; +export const customBase64Dict = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; /** * A simple dictionary for safe source-code inclusion. Effectively base91, as