Merge pull request #131 from dorylab/sqlite-demo

dory-finn · web-flow · commit e92d2257c51f · 2026-03-31T22:34:28.000+08:00
refactor: ingest demo data on org created
diff --git a/apps/web/app/api/connection/route.ts b/apps/web/app/api/connection/route.ts
@@ -5,7 +5,6 @@ import { handleApiError } from '../utils/handle-error';
 import { parseJsonBody } from '../utils/parse-json';
 import { withManagedOrganizationHandler, withOrganizationHandler } from '../utils/with-organization-handler';
 import { getApiLocale, translateApi } from '@/app/api/utils/i18n';
-import { ensureDemoConnection } from '@/lib/demo/ensure-demo-connection';
 
 // GET /api/connections?id=xxx
 export const GET = withOrganizationHandler(async ({ req, db, organizationId }) => {
@@ -28,17 +27,7 @@ export const GET = withOrganizationHandler(async ({ req, db, organizationId }) =
             return NextResponse.json(ResponseUtil.success(record));
         }
 
-        let data = await db.connections.list(organizationId);
-
-        if (data.length === 0 && userId) {
-            try {
-                await ensureDemoConnection(db, userId, organizationId);
-                data = await db.connections.list(organizationId);
-            } catch (err) {
-                console.warn('[api/connection] failed to create demo connection:', err);
-            }
-        }
-
+        const data = await db.connections.list(organizationId);
         return NextResponse.json(ResponseUtil.success(data));
     } catch (err: any) {
         return handleApiError(err);
diff --git a/apps/web/lib/auth.ts b/apps/web/lib/auth.ts
@@ -7,6 +7,7 @@ import Stripe from 'stripe';
 import { and, eq, isNull, or } from 'drizzle-orm';
 import { createCachedAsyncFactory } from '@dory/auth-core';
 import type { PostgresDBClient } from '../types';
+import { getDBService } from './database';
 import { getClient } from './database/postgres/client';
 import { getDatabaseProvider } from './database/provider';
 import { schema } from './database/schema';
@@ -262,6 +263,10 @@ function createAuth() {
                                 },
                             };
                         },
+                        afterCreateOrganization: async ({ organization, user }) => {
+                            const dbService = await getDBService();
+                            await dbService.organizations.ensureOrganizationDefaults(user.id, organization.id, dbService.connections);
+                        },
                     },
                     sendInvitationEmail: async ({ id, email, role, organization, inviter }) => {
                         if (!publicAuthBaseUrl) {
diff --git a/apps/web/lib/connection/config-builder.ts b/apps/web/lib/connection/config-builder.ts
@@ -1,4 +1,5 @@
 import type { ConnectionListIdentity, ConnectionListItem, ConnectionSsh, TestConnectionPayload } from '@/types/connections';
+import { resolveStoredSqlitePath } from '@/lib/demo/paths';
 import { UnsupportedTypeError } from './base/errors';
 import type { BaseConfig } from './base/types';
 import { applyQueryRequestTimeout } from './defaults';
@@ -99,7 +100,7 @@ export function buildStoredConnectionConfig(
     const type = resolveConnectionType(connection.type ?? connection.engine ?? 'clickhouse');
 
     if (type === 'sqlite') {
-        const normalizedPath = connection.path?.trim();
+        const normalizedPath = resolveStoredSqlitePath(connection.path);
         if (!normalizedPath) {
             throw createError('missing_path');
         }
@@ -150,7 +151,7 @@ export function buildTestConnectionConfig(
     const type = resolveConnectionType(connection.type ?? connection.engine ?? 'clickhouse');
 
     if (type === 'sqlite') {
-        const normalizedPath = connection.path?.trim();
+        const normalizedPath = resolveStoredSqlitePath(connection.path);
         if (!normalizedPath) {
             throw createError('missing_path');
         }
diff --git a/apps/web/lib/connection/display.ts b/apps/web/lib/connection/display.ts
@@ -1,10 +1,14 @@
 import type { ConnectionListItem } from '@/types/connections';
+import { isDemoSqliteConnectionPath } from '@/lib/demo/paths';
 
 export function getConnectionLocationLabel(connection?: ConnectionListItem['connection'] | null) {
     if (!connection) return null;
 
     if (connection.type === 'sqlite') {
         const normalizedPath = connection.path?.trim();
+        if (isDemoSqliteConnectionPath(normalizedPath)) {
+            return 'Built-in demo.sqlite';
+        }
         return normalizedPath || null;
     }
 
diff --git a/apps/web/lib/connection/drivers/sqlite/sqlite-driver.ts b/apps/web/lib/connection/drivers/sqlite/sqlite-driver.ts
@@ -1,6 +1,7 @@
 import path from 'node:path';
 import Database from 'better-sqlite3';
 import { MAX_RESULT_ROWS } from '@/app/config/sql-console';
+import { resolveStoredSqlitePath } from '@/lib/demo/paths';
 import { enforceSelectLimit } from '@/lib/connection/base/limit';
 import { compileParams } from '@/lib/connection/base/params/compile';
 import type { DriverQueryParams } from '@/lib/connection/base/params/types';
@@ -70,7 +71,7 @@ function getSqliteVersion(db: SqliteDatabase): string | undefined {
 }
 
 export function resolveSqlitePath(config: BaseConfig): string {
-    return assertAbsolutePath(config.path);
+    return assertAbsolutePath(resolveStoredSqlitePath(config.path));
 }
 
 export function openSqliteDatabase(config: BaseConfig): SqliteDatabase {
diff --git a/apps/web/lib/database/postgres/impl/organization/index.ts b/apps/web/lib/database/postgres/impl/organization/index.ts
@@ -3,8 +3,10 @@ import { DatabaseError } from '@/lib/errors/DatabaseError';
 import { PostgresDBClient } from '@/types';
 import { and, eq, isNull, or } from 'drizzle-orm';
 import { organizations } from '@/lib/database/schema';
+import { ensureDemoConnection } from '@/lib/demo/ensure-demo-connection';
 import { organizationMembers } from '../../schemas';
 import { translateDatabase } from '@/lib/database/i18n';
+import type { PostgresConnectionsRepository } from '../connections';
 
 export class PostgresOrganizationsRepository {
     private db!: PostgresDBClient;
@@ -24,6 +26,20 @@ export class PostgresOrganizationsRepository {
         return this.db.select().from(organizationMembers).where(eq(organizationMembers.userId, userId));
     }
 
+    async ensureOrganizationDefaults(
+        userId: string,
+        organizationId: string,
+        connectionsRepository: PostgresConnectionsRepository,
+    ) {
+        return ensureDemoConnection(
+            {
+                connections: connectionsRepository,
+            },
+            userId,
+            organizationId,
+        );
+    }
+
     async getOrganizationBySlugOrId(value: string) {
         const rows = await this.db
             .select()
@@ -46,7 +62,7 @@ export class PostgresOrganizationsRepository {
                 ),
             )
             .limit(1);
-    
+
         return rows.length > 0;
     }
 }
diff --git a/apps/web/lib/demo/ensure-demo-connection.ts b/apps/web/lib/demo/ensure-demo-connection.ts
@@ -1,28 +1,52 @@
 import fs from 'node:fs';
 import type { DBService } from '@/lib/database';
-import { getDemoSqlitePath } from './paths';
+import { DEMO_SQLITE_CONNECTION_PATH, getDemoSqlitePath } from './paths';
 
 const DEMO_CONNECTION_NAME = 'Demo Database';
+type DemoConnectionService = Pick<DBService, 'connections'>;
+export type EnsureDemoConnectionResult = 'created' | 'updated' | 'exists' | 'skipped';
 
 /**
  * Ensure a "Demo Database" SQLite connection exists for the given organization.
  * Idempotent: skips if the connection already exists or if the demo.sqlite file is not available.
  */
 export async function ensureDemoConnection(
-    db: DBService,
+    db: DemoConnectionService,
     userId: string,
     organizationId: string,
-): Promise<void> {
+): Promise<EnsureDemoConnectionResult> {
     const demoPath = getDemoSqlitePath();
     if (!demoPath || !fs.existsSync(demoPath)) {
         console.log('[demo] demo.sqlite not found, skipping demo connection creation');
-        return;
+        return 'skipped';
     }
 
     const existing = await db.connections.list(organizationId);
-    const hasDemoConnection = existing.some((c) => c.name === DEMO_CONNECTION_NAME);
-    if (hasDemoConnection) {
-        return;
+    const existingDemoConnection = existing.find(item => item.connection.name === DEMO_CONNECTION_NAME);
+    if (existingDemoConnection) {
+        if (existingDemoConnection.connection.path !== DEMO_SQLITE_CONNECTION_PATH) {
+            await db.connections.update(organizationId, existingDemoConnection.connection.id, {
+                connection: {
+                    organizationId,
+                    type: 'sqlite',
+                    engine: 'sqlite',
+                    name: existingDemoConnection.connection.name,
+                    description: existingDemoConnection.connection.description ?? undefined,
+                    host: existingDemoConnection.connection.host,
+                    port: existingDemoConnection.connection.port,
+                    httpPort: existingDemoConnection.connection.httpPort ?? undefined,
+                    database: existingDemoConnection.connection.database ?? undefined,
+                    options: existingDemoConnection.connection.options,
+                    status: existingDemoConnection.connection.status,
+                    environment: existingDemoConnection.connection.environment,
+                    tags: existingDemoConnection.connection.tags,
+                    path: DEMO_SQLITE_CONNECTION_PATH,
+                },
+                identities: [],
+            });
+            return 'updated';
+        }
+        return 'exists';
     }
 
     console.log(`[demo] creating "${DEMO_CONNECTION_NAME}" connection for org ${organizationId}`);
@@ -37,21 +61,29 @@ export async function ensureDemoConnection(
             host: null,
             port: null,
             database: 'main',
-            path: demoPath,
-            status: 'ready',
+            path: DEMO_SQLITE_CONNECTION_PATH,
         },
         identities: [
             {
+                id: '',
+                connectionId: '',
+                organizationId,
                 name: 'Default',
                 username: 'sqlite',
+                role: undefined,
+                options: '{}',
                 isDefault: true,
                 database: 'main',
                 enabled: true,
-                role: null,
-                options: '{}',
-            } as any,
+                status: 'active',
+                createdAt: new Date(),
+                updatedAt: new Date(),
+                deletedAt: null,
+                password: undefined,
+            },
         ],
     });
 
     console.log(`[demo] "${DEMO_CONNECTION_NAME}" connection created`);
+    return 'created';
 }
diff --git a/apps/web/lib/demo/paths.ts b/apps/web/lib/demo/paths.ts
@@ -1,28 +1,34 @@
 import path from 'node:path';
-import { fileURLToPath } from 'node:url';
 
 const DEMO_SQLITE_FILENAME = 'demo.sqlite';
+const DEMO_SQLITE_DIR = path.join('public', 'resources');
+const DEMO_SQLITE_RELATIVE_PATH = path.join(DEMO_SQLITE_DIR, DEMO_SQLITE_FILENAME);
+export const DEMO_SQLITE_CONNECTION_PATH = 'dory://demo-sqlite';
 
 /**
- * Resolve the absolute path for demo.sqlite based on PGLITE_DB_PATH.
- * The demo file lives as a sibling of the PGlite data directory.
- *
- * Example: PGLITE_DB_PATH = "file:///app/data/dory" → "/app/data/demo.sqlite"
+ * Resolve the absolute path for the bundled demo SQLite file.
+ * The file is treated as a fixed app resource rather than a generated runtime artifact.
  */
 export function resolveDemoSqlitePath(): string {
-    const raw = process.env.PGLITE_DB_PATH;
-    if (!raw) {
-        throw new Error('[demo] PGLITE_DB_PATH is not set, cannot resolve demo.sqlite path');
-    }
+    return path.resolve(process.cwd(), DEMO_SQLITE_RELATIVE_PATH);
+}
 
-    const fsPath = raw.startsWith('file:') ? fileURLToPath(raw) : decodeURIComponent(raw);
-    const parentDir = path.dirname(path.resolve(fsPath));
-    return path.join(parentDir, DEMO_SQLITE_FILENAME);
+export function isDemoSqliteConnectionPath(value: string | null | undefined): boolean {
+    return value?.trim() === DEMO_SQLITE_CONNECTION_PATH;
+}
+
+export function resolveStoredSqlitePath(value: string | null | undefined): string | undefined {
+    const normalized = value?.trim();
+    if (!normalized) return undefined;
+    if (isDemoSqliteConnectionPath(normalized)) {
+        return resolveDemoSqlitePath();
+    }
+    return normalized;
 }
 
 /**
- * Get the demo.sqlite path from the environment variable set during bootstrap.
+ * Get the fixed absolute path for demo.sqlite.
  */
 export function getDemoSqlitePath(): string | undefined {
-    return process.env.DEMO_SQLITE_PATH || undefined;
+    return resolveDemoSqlitePath();
 }
diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts
@@ -9,7 +9,7 @@ const nextConfig: NextConfig = {
     output: 'standalone',
     serverExternalPackages: ['@electric-sql/pglite', 'pino', 'better-sqlite3', 'electron'],
     outputFileTracingIncludes: {
-        '/*': ['./registry/**/*'],
+        '/*': ['./registry/**/*', './public/resources/demo.sqlite'],
     },
     logging: {
         fetches: {
diff --git a/apps/web/package.json b/apps/web/package.json
@@ -16,6 +16,7 @@
     "pglite:studio": "drizzle-kit studio --config ./drizzle.pglite.config.ts",
     "drizzle:push": "drizzle-kit push",
     "drizzle:migrate": "tsx ./scripts/migrate.ts",
+    "demo:backfill": "tsx ./scripts/backfill-demo-connections.ts",
     "postgres:check": "tsx ./scripts/check-postgres-connection.ts",
     "postgres:repair-migrations": "tsx ./scripts/repair-drizzle-migrations.ts",
     "postgres:migrate:generate": "drizzle-kit generate",
diff --git a/apps/web/public/resources/demo.sqlite b/apps/web/public/resources/demo.sqlite
diff --git a/apps/web/public/resources/demo.sqlite-shm b/apps/web/public/resources/demo.sqlite-shm
diff --git a/apps/web/public/resources/demo.sqlite-wal b/apps/web/public/resources/demo.sqlite-wal
diff --git a/apps/web/scripts/backfill-demo-connections.ts b/apps/web/scripts/backfill-demo-connections.ts
@@ -0,0 +1,57 @@
+import 'dotenv/config';
+import { getDBService } from '@/lib/database';
+import { getClient } from '@/lib/database/postgres/client';
+import { organizations } from '@/lib/database/postgres/schemas/organizations/organizations';
+
+async function main() {
+    const client = await getClient();
+    const db = await getDBService();
+
+    const rows = await client.select({
+        id: organizations.id,
+        ownerUserId: organizations.ownerUserId,
+        name: organizations.name,
+    }).from(organizations);
+
+    let created = 0;
+    let updated = 0;
+    let exists = 0;
+    let skipped = 0;
+    let failed = 0;
+
+    for (const organization of rows) {
+        try {
+            const result = await db.organizations.ensureOrganizationDefaults(
+                organization.ownerUserId,
+                organization.id,
+                db.connections,
+            );
+            if (result === 'created') created += 1;
+            if (result === 'updated') updated += 1;
+            if (result === 'exists') exists += 1;
+            if (result === 'skipped') skipped += 1;
+            console.log(`[backfill-demo] ${result} demo connection for org ${organization.id} (${organization.name})`);
+        } catch (error) {
+            failed += 1;
+            console.error(`[backfill-demo] failed for org ${organization.id} (${organization.name})`, error);
+        }
+    }
+
+    console.log('[backfill-demo] complete', {
+        organizations: rows.length,
+        created,
+        updated,
+        exists,
+        skipped,
+        failed,
+    });
+
+    if (failed > 0) {
+        process.exitCode = 1;
+    }
+}
+
+main().catch(error => {
+    console.error('[backfill-demo] fatal error', error);
+    process.exit(1);
+});
diff --git a/apps/web/scripts/bootstrap.ts b/apps/web/scripts/bootstrap.ts
@@ -9,7 +9,6 @@ import { DEFAULT_PGLITE_DB_PATH, DESKTOP_PGLITE_DB_PATH } from '@/shared/data/ap
 import { ensureFileUrl } from '@/lib/database/pglite/url';
 import { isDesktopRuntime } from '@/lib/runtime/runtime';
 import { resolveDemoSqlitePath } from '@/lib/demo/paths';
-import { generateDemoSqlite } from '@/lib/demo/generate-demo-sqlite';
 
 
 async function ensureDirForFile(filePath: string) {
@@ -39,14 +38,13 @@ async function bootstrapPglite() {
   await migratePgliteDB();
 }
 
-function bootstrapDemoSqlite() {
+async function verifyDemoSqlite() {
+    const demoPath = resolveDemoSqlitePath();
     try {
-        const demoPath = resolveDemoSqlitePath();
-        generateDemoSqlite(demoPath);
-        process.env.DEMO_SQLITE_PATH = demoPath;
-        console.log('[bootstrap] DEMO_SQLITE_PATH =', demoPath);
+        await fs.access(demoPath);
+        console.log('[bootstrap] fixed demo sqlite =', demoPath);
     } catch (error) {
-        console.warn('[bootstrap] skipping demo sqlite:', error);
+        console.warn('[bootstrap] fixed demo sqlite missing:', demoPath, error);
     }
 }
 
@@ -60,7 +58,7 @@ export async function bootstrap() {
         console.log('[bootstrap] skip bootstrap');
     }
 
-    bootstrapDemoSqlite();
+    await verifyDemoSqlite();
 }
 
 bootstrap().catch(err => {
diff --git a/apps/web/scripts/dev-bootstrap.ts b/apps/web/scripts/dev-bootstrap.ts
diff --git a/apps/web/types/connections.ts b/apps/web/types/connections.ts

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`import path from 'node:path';`
`2`	`2`	`import Database from 'better-sqlite3';`
`3`	`3`	`import { MAX_RESULT_ROWS } from '@/app/config/sql-console';`
	`4`	`+import { resolveStoredSqlitePath } from '@/lib/demo/paths';`
`4`	`5`	`import { enforceSelectLimit } from '@/lib/connection/base/limit';`
`5`	`6`	`import { compileParams } from '@/lib/connection/base/params/compile';`
`6`	`7`	`import type { DriverQueryParams } from '@/lib/connection/base/params/types';`
`@@ -70,7 +71,7 @@ function getSqliteVersion(db: SqliteDatabase): string \| undefined {`
`70`	`71`	`}`
`71`	`72`
`72`	`73`	`export function resolveSqlitePath(config: BaseConfig): string {`
`73`		`- return assertAbsolutePath(config.path);`
	`74`	`+ return assertAbsolutePath(resolveStoredSqlitePath(config.path));`
`74`	`75`	`}`
`75`	`76`
`76`	`77`	`export function openSqliteDatabase(config: BaseConfig): SqliteDatabase {`