diff --git a/.github/workflows/bun-test.yml b/.github/workflows/bun-test.yml index f562cc5a..671eff86 100644 --- a/.github/workflows/bun-test.yml +++ b/.github/workflows/bun-test.yml @@ -7,7 +7,7 @@ on: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 75 steps: - name: Checkout code @@ -17,20 +17,19 @@ jobs: uses: oven-sh/setup-bun@v2 with: bun-version: latest - + - name: Install dependencies run: bun install - - name: Cache setup artifacts - id: cache-setup - uses: actions/cache@v4 - with: - path: ./db.sqlite3 - key: db-sqlite3-${{ hashFiles('scripts/setup-*.ts') }} + - name: Install cf-proxy dependencies + run: cd cf-proxy && bun install - - name: Run setup if cache miss - if: steps.cache-setup.outputs.cache-hit != 'true' - run: bun run setup + - name: Set up test database + env: + JLCSEARCH_DB_PATH: .tmp/test-db.sqlite3 + run: bun run scripts/setup-test-db.ts - name: Run tests + env: + JLCSEARCH_DB_PATH: .tmp/test-db.sqlite3 run: bun test diff --git a/cf-proxy/src/components.ts b/cf-proxy/src/components.ts index 6b25a12a..d169ade1 100644 --- a/cf-proxy/src/components.ts +++ b/cf-proxy/src/components.ts @@ -8,6 +8,7 @@ export interface ComponentCatalogQueryParams { search?: string is_basic?: string is_preferred?: string + is_extended_promotional?: string } export async function queryComponentCatalog( @@ -22,6 +23,7 @@ export async function queryComponentCatalog( package: string | null basic: number | null preferred: number | null + is_extended_promotional: number | null description: string | null stock: number | null price: string | null @@ -34,6 +36,7 @@ export async function queryComponentCatalog( subcategory_name: params.subcategory_name, is_basic: params.is_basic, is_preferred: params.is_preferred, + is_extended_promotional: params.is_extended_promotional, limit: "100", }) diff --git a/cf-proxy/src/index.ts b/cf-proxy/src/index.ts index 36b6160c..9268f64c 100644 --- a/cf-proxy/src/index.ts +++ b/cf-proxy/src/index.ts @@ -1,7 +1,7 @@ import { CacheService, addCorsHeaders, addVaryHeader } from "./cache-service" import { queryComponentCatalog } from "./components" -import { getD1Client } from "./db/get-d1-client" import { getD1Handler } from "./d1-routes" +import { getD1Client } from "./db/get-d1-client" import { renderD1TablePage, renderHomePage } from "./render" import { searchIndex } from "./search" @@ -367,6 +367,7 @@ async function handleD1Search( package: row.package ?? "", is_basic: Boolean(row.basic), is_preferred: Boolean(row.preferred), + is_extended_promotional: Boolean(row.is_extended_promotional), description: row.description ?? "", stock: row.stock ?? 0, price: row.price1 ?? extractSmallQuantityPrice(row.price), @@ -491,6 +492,7 @@ async function handleD1ComponentsList( subcategory: row.subcategory ?? "", is_basic: Boolean(row.basic), is_preferred: Boolean(row.preferred), + is_extended_promotional: Boolean(row.is_extended_promotional), })), } diff --git a/cf-proxy/src/render.ts b/cf-proxy/src/render.ts index 11ce224c..9d9e44d2 100644 --- a/cf-proxy/src/render.ts +++ b/cf-proxy/src/render.ts @@ -1,9 +1,9 @@ import { + type FilterOptions, + type QueryParams, ROUTE_TO_TABLE, TABLE_CONFIGS, TABLE_RESPONSE_KEY, - type QueryParams, - type FilterOptions, } from "./handlers" const escapeHtml = (value: unknown): string => @@ -147,6 +147,7 @@ const COLUMN_LABELS: Record = { in_stock: "In Stock", is_basic: "Basic", is_preferred: "Preferred", + is_extended_promotional: "Extended Promotional", capacitance_farads: "Capacitance", tolerance_fraction: "Tolerance", voltage_rating: "Voltage", @@ -546,6 +547,9 @@ const renderComponentsFilters = (
+
+ +
` diff --git a/cf-proxy/src/search.ts b/cf-proxy/src/search.ts index f1b60d3c..07bb2cfd 100644 --- a/cf-proxy/src/search.ts +++ b/cf-proxy/src/search.ts @@ -1,4 +1,4 @@ -import { sql, type Kysely, type RawBuilder } from "kysely" +import { type Kysely, type RawBuilder, sql } from "kysely" import type { DB } from "./db/types" import { buildSearchTokenGroups } from "./search-query" @@ -9,6 +9,7 @@ export interface SearchQueryParams { limit?: string is_basic?: string is_preferred?: string + is_extended_promotional?: string } interface SearchRow { @@ -21,6 +22,7 @@ interface SearchRow { price1: number | null basic: number | null preferred: number | null + is_extended_promotional: number | null category: string | null subcategory: string | null } @@ -110,6 +112,14 @@ export async function searchIndex( conditions.push(sql`search_index.preferred = 1`) } + if ( + params.is_extended_promotional === "true" || + params.is_extended_promotional === "1" + ) { + conditions.push(sql`search_index.preferred = 1`) + conditions.push(sql`COALESCE(search_index.basic, 0) = 0`) + } + const raw = params.q?.trim() if (raw) { @@ -151,6 +161,12 @@ export async function searchIndex( search_index.price1, search_index.basic, search_index.preferred, + CASE + WHEN search_index.preferred = 1 + AND COALESCE(search_index.basic, 0) = 0 + THEN 1 + ELSE 0 + END AS is_extended_promotional, search_index.category, search_index.subcategory FROM search_index diff --git a/cf-proxy/test/render.test.ts b/cf-proxy/test/render.test.ts index 844b04a2..0dc83b98 100644 --- a/cf-proxy/test/render.test.ts +++ b/cf-proxy/test/render.test.ts @@ -59,4 +59,28 @@ describe("render helpers", () => { "Feature":"Overcurrent Protection(OCP)"", ) }) + + it("renders the extended promotional components filter", () => { + const html = renderD1TablePage( + "/components/list", + { + components: [ + { + lcsc: 123, + mfr: "PROMO-PART", + package: "SMD", + is_extended_promotional: true, + }, + ], + }, + { is_extended_promotional: "true" }, + "https://example.com/components/list?is_extended_promotional=true", + ) + + expect(html).toContain("Extended Promotional") + expect(html).toContain('name="is_extended_promotional"') + expect(html).toContain( + 'name="is_extended_promotional" value="true" checked', + ) + }) }) diff --git a/demos/extended-promotional-filter-demo.mp4 b/demos/extended-promotional-filter-demo.mp4 new file mode 100644 index 00000000..8e281392 Binary files /dev/null and b/demos/extended-promotional-filter-demo.mp4 differ diff --git a/lib/util/extended-promotional.ts b/lib/util/extended-promotional.ts new file mode 100644 index 00000000..9c610bd6 --- /dev/null +++ b/lib/util/extended-promotional.ts @@ -0,0 +1,16 @@ +type ComponentPromotionFlags = { + basic?: boolean | number | string | null + preferred?: boolean | number | string | null + is_basic?: boolean | number | string | null + is_preferred?: boolean | number | string | null +} + +const isTruthyFlag = (value: boolean | number | string | null | undefined) => + value === true || value === 1 || value === "1" || value === "true" + +export const isExtendedPromotional = (component: ComponentPromotionFlags) => { + const basic = component.basic ?? component.is_basic + const preferred = component.preferred ?? component.is_preferred + + return isTruthyFlag(preferred) && !isTruthyFlag(basic) +} diff --git a/routes/api/search.tsx b/routes/api/search.tsx index a95d0da2..bf92c181 100644 --- a/routes/api/search.tsx +++ b/routes/api/search.tsx @@ -1,7 +1,8 @@ import { sql } from "kysely" +import { isExtendedPromotional } from "lib/util/extended-promotional" import { - buildSearchTokenGroups, type SearchTokenGroup, + buildSearchTokenGroups, tokenizeSearchTerm, } from "lib/util/search-token-groups" import { withWinterSpec } from "lib/with-winter-spec" @@ -50,6 +51,7 @@ export default withWinterSpec({ limit: z.string().optional(), is_basic: z.boolean().optional(), is_preferred: z.boolean().optional(), + is_extended_promotional: z.boolean().optional(), }), jsonResponse: z.any(), } as const)(async (req, ctx) => { @@ -72,6 +74,9 @@ export default withWinterSpec({ if (req.query.is_preferred) { query = query.where("preferred", "=", 1) } + if (req.query.is_extended_promotional) { + query = query.where("preferred", "=", 1).where("basic", "=", 0) + } const baseQuery = query let fallbackLikeTokens: string[] = [] @@ -147,7 +152,10 @@ export default withWinterSpec({ } } - const fullComponents = await query.execute() + const fullComponents = (await query.execute()).map((c) => ({ + ...c, + is_extended_promotional: isExtendedPromotional(c), + })) if (fallbackLikeTokens.length > 0 && fullComponents.length === 0) { let fallbackQuery = baseQuery @@ -181,7 +189,10 @@ export default withWinterSpec({ for (const component of fallbackComponents) { if (seenLcsc.has(component.lcsc)) continue - fullComponents.push(component) + fullComponents.push({ + ...component, + is_extended_promotional: isExtendedPromotional(component), + }) seenLcsc.add(component.lcsc) if (fullComponents.length >= limit) break } @@ -193,6 +204,7 @@ export default withWinterSpec({ package: c.package, is_basic: Boolean(c.basic), is_preferred: Boolean(c.preferred), + is_extended_promotional: c.is_extended_promotional, description: c.description, stock: c.stock, price: extractSmallQuantityPrice(c.price), diff --git a/routes/components/list.tsx b/routes/components/list.tsx index 640785d1..5d45336c 100644 --- a/routes/components/list.tsx +++ b/routes/components/list.tsx @@ -1,6 +1,7 @@ import { sql } from "kysely" -import { Table } from "lib/ui/Table" import { ExpressionBuilder } from "kysely" +import { Table } from "lib/ui/Table" +import { isExtendedPromotional } from "lib/util/extended-promotional" import { buildSearchTokenGroups } from "lib/util/search-token-groups" import { withWinterSpec } from "lib/with-winter-spec" import { z } from "zod" @@ -35,6 +36,7 @@ export default withWinterSpec({ search: z.string().optional(), is_basic: z.boolean().optional(), is_preferred: z.boolean().optional(), + is_extended_promotional: z.boolean().optional(), }), jsonResponse: z.any(), } as const)(async (req, ctx) => { @@ -51,6 +53,7 @@ export default withWinterSpec({ "price", "extra", "basic", + "preferred", ]) .limit(limit) .orderBy("stock", "desc") @@ -70,6 +73,9 @@ export default withWinterSpec({ if (req.query.is_preferred) { query = query.where("preferred", "=", 1) } + if (req.query.is_extended_promotional) { + query = query.where("preferred", "=", 1).where("basic", "=", 0) + } if (req.query.search) { const search = req.query.search @@ -103,7 +109,10 @@ export default withWinterSpec({ } } - const fullComponents = await query.execute() + const fullComponents = (await query.execute()).map((c: any) => ({ + ...c, + is_extended_promotional: isExtendedPromotional(c), + })) const components = fullComponents.map((c: any) => ({ lcsc: c.lcsc, @@ -111,6 +120,7 @@ export default withWinterSpec({ package: c.package, is_basic: Boolean(c.basic), is_preferred: Boolean(c.preferred), + is_extended_promotional: c.is_extended_promotional, description: c.description, stock: c.stock, price: extractSmallQuantityPrice(c.price), @@ -155,6 +165,17 @@ export default withWinterSpec({ /> +
+ +
diff --git a/scripts/setup-7z.ts b/scripts/setup-7z.ts index 75f1b7c0..e2884862 100644 --- a/scripts/setup-7z.ts +++ b/scripts/setup-7z.ts @@ -1,16 +1,16 @@ -import { mkdir, chmod } from "node:fs/promises" import { existsSync } from "node:fs" -import { platform, arch } from "node:os" +import { chmod, mkdir } from "node:fs/promises" +import { arch, platform } from "node:os" const BINARY_DIR = ".bin" const BINARY_NAME = "7zz" // Map of platform-arch combinations to download URLs const BINARY_URLS: Record = { - "linux-x64": "https://7-zip.org/a/7z2408-linux-x64.tar.xz", - "linux-arm64": "https://7-zip.org/a/7z2408-linux-arm64.tar.xz", - "darwin-x64": "https://7-zip.org/a/7z2408-mac.tar.xz", - "darwin-arm64": "https://7-zip.org/a/7z2408-mac.tar.xz", + "linux-x64": "https://7-zip.org/a/7z2501-linux-x64.tar.xz", + "linux-arm64": "https://7-zip.org/a/7z2501-linux-arm64.tar.xz", + "darwin-x64": "https://7-zip.org/a/7z2501-mac.tar.xz", + "darwin-arm64": "https://7-zip.org/a/7z2501-mac.tar.xz", } async function downloadAndExtract7z() { diff --git a/scripts/setup-test-db.ts b/scripts/setup-test-db.ts new file mode 100644 index 00000000..bb56bab7 --- /dev/null +++ b/scripts/setup-test-db.ts @@ -0,0 +1,170 @@ +import { Database } from "bun:sqlite" +import { mkdirSync, rmSync } from "node:fs" +import Path from "node:path" + +const dbPath = process.env.JLCSEARCH_DB_PATH ?? ".tmp/test-db.sqlite3" +const resolvedDbPath = Path.resolve(process.cwd(), dbPath) + +mkdirSync(Path.dirname(resolvedDbPath), { recursive: true }) +rmSync(resolvedDbPath, { force: true }) + +const db = new Database(resolvedDbPath) + +db.exec(` + CREATE TABLE categories ( + id INTEGER PRIMARY KEY, + category TEXT NOT NULL, + subcategory TEXT NOT NULL + ); + + CREATE TABLE components ( + lcsc INTEGER PRIMARY KEY, + mfr TEXT NOT NULL, + package TEXT NOT NULL, + description TEXT NOT NULL, + datasheet TEXT NOT NULL DEFAULT '', + price TEXT NOT NULL, + stock INTEGER NOT NULL, + last_update INTEGER NOT NULL DEFAULT 0, + manufacturer_id INTEGER NOT NULL DEFAULT 0, + category_id INTEGER NOT NULL, + extra TEXT, + basic INTEGER NOT NULL DEFAULT 0, + preferred INTEGER NOT NULL DEFAULT 0, + joints INTEGER NOT NULL DEFAULT 0, + flag INTEGER NOT NULL DEFAULT 0, + last_on_stock INTEGER NOT NULL DEFAULT 0 + ); + + CREATE VIRTUAL TABLE components_fts USING fts5( + lcsc, + mfr, + mfr_chars, + description + ); + + CREATE VIEW v_components AS + SELECT + components.*, + categories.category, + categories.subcategory + FROM components + INNER JOIN categories ON categories.id = components.category_id; +`) + +const categories = [ + [1, "Integrated Circuits", "ST Microelectronics"], + [2, "Resistors", "Chip Resistor - Surface Mount"], + [3, "Connectors", "USB Connectors"], + [4, "Optoelectronics", "Light Emitting Diodes (LED)"], + [5, "Integrated Circuits", "Timers"], +] as const + +const insertCategory = db.prepare( + "INSERT INTO categories (id, category, subcategory) VALUES (?, ?, ?)", +) +for (const category of categories) { + insertCategory.run(...category) +} + +type ComponentSeed = { + lcsc: number + mfr: string + package: string + description: string + categoryId: number + basic?: number + preferred?: number +} + +const components: ComponentSeed[] = [ + { + lcsc: 1002, + mfr: "C1002 Test Component", + package: "SOT-23", + description: "Generic test component for direct LCSC lookup", + categoryId: 1, + }, + { + lcsc: 11702, + mfr: "RC0402FR-075K1L", + package: "0402", + description: "0402 5.1k resistor", + categoryId: 2, + preferred: 1, + }, + { + lcsc: 2765186, + mfr: "USB Type-C 16P Connector", + package: "SMD", + description: "USB Type-C 16P connector receptacle", + categoryId: 3, + }, + { + lcsc: 965793, + mfr: "Red LED 0402", + package: "0402", + description: "0402 red LED indicator", + categoryId: 4, + }, + { + lcsc: 40164, + mfr: "STM32F401RCT6 STMicroelectronics", + package: "LQFP-64", + description: "STM32F401RCT6 ARM Cortex-M4 microcontroller", + categoryId: 1, + preferred: 1, + }, + { + lcsc: 555001, + mfr: "NE555 Timer", + package: "SOIC-8", + description: "555 Timer integrated circuit", + categoryId: 5, + basic: 1, + }, +] + +const price = JSON.stringify([{ qty: 1, price: "0.01" }]) +const insertComponent = db.prepare(` + INSERT INTO components ( + lcsc, + mfr, + package, + description, + datasheet, + price, + stock, + category_id, + extra, + basic, + preferred + ) + VALUES (?, ?, ?, ?, '', ?, 100, ?, '{}', ?, ?) +`) +const insertFts = db.prepare(` + INSERT INTO components_fts (lcsc, mfr, mfr_chars, description) + VALUES (?, ?, ?, ?) +`) + +for (const component of components) { + insertComponent.run( + component.lcsc, + component.mfr, + component.package, + component.description, + price, + component.categoryId, + component.basic ?? 0, + component.preferred ?? 0, + ) + insertFts.run( + String(component.lcsc), + component.mfr, + component.mfr.replace(/\s+/g, ""), + component.description, + ) +} + +db.close() +console.log(`Created test database at ${resolvedDbPath}`) diff --git a/tests/lib/extended-promotional.test.ts b/tests/lib/extended-promotional.test.ts new file mode 100644 index 00000000..aed5f330 --- /dev/null +++ b/tests/lib/extended-promotional.test.ts @@ -0,0 +1,16 @@ +import { expect, test } from "bun:test" +import { isExtendedPromotional } from "lib/util/extended-promotional" + +test("extended promotional parts are preferred but not basic", () => { + expect(isExtendedPromotional({ preferred: 1, basic: 0 })).toBe(true) + expect(isExtendedPromotional({ preferred: true, basic: false })).toBe(true) + expect(isExtendedPromotional({ is_preferred: "true", is_basic: "0" })).toBe( + true, + ) +}) + +test("basic or non-preferred parts are not extended promotional", () => { + expect(isExtendedPromotional({ preferred: 1, basic: 1 })).toBe(false) + expect(isExtendedPromotional({ preferred: 0, basic: 0 })).toBe(false) + expect(isExtendedPromotional({ preferred: "0", basic: "0" })).toBe(false) +}) diff --git a/tests/preload.ts b/tests/preload.ts index 44a3aa43..12ffadd3 100644 --- a/tests/preload.ts +++ b/tests/preload.ts @@ -1,4 +1,5 @@ import { afterEach } from "bun:test" +import { getDbClient } from "lib/db/get-db-client" import { setupDerivedTables } from "lib/db/derivedtables/setup-derived-tables" declare global { @@ -7,7 +8,10 @@ declare global { } globalThis.deferredCleanupFns ??= [] -globalThis.derivedTablesSetupPromise ??= setupDerivedTables({ populate: false }) +globalThis.derivedTablesSetupPromise ??= setupDerivedTables({ + db: getDbClient(), + populate: false, +}) await globalThis.derivedTablesSetupPromise