Skip to content

Fix delivery funnel session and PR counts #245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Aymen (aymenfurter) merged 1 commit into from
Aug 26, 2026
Merged

Fix delivery funnel session and PR counts #245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
336 changes: 251 additions & 85 deletions src/core/github-app-analytics.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,132 +3,298 @@
* Licensed under the MIT License. See LICENSE in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { describe, expect, it } from 'vitest';
import { DatabaseSync } from 'node:sqlite';
import { afterEach, describe, expect, it } from 'vitest';
import {
loadGitHubAppMetrics,
parseGitHubAppIssueSessionReferences,
parseGitHubAppMetricsRows,
parseGitHubAppWorkspacePrCohorts,
} from './github-app-analytics';
import type { GitHubAppMetrics } from './types';

function row(date: string, overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
date,
cohortPullRequestsRaised: 4,
cohortPullRequestsMerged: 3,
totalProjectSessions: 12,
sessionsWithIssue: 7,
sessionsWithPullRequest: 8,
sessionsWithMergedPullRequest: 6,
lastActivityAt: '2026-08-24T10:00:00.000Z',
...overrides,
};
const APP_SCHEMA_SQL = `
CREATE TABLE sessions (
id TEXT PRIMARY KEY,
session_type TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE workspaces (
id TEXT PRIMARY KEY,
session_id TEXT,
created_at TEXT NOT NULL,
source_issue_number INTEGER,
source_pr_number INTEGER,
source_pr_merged_at TEXT,
source_pr_state TEXT,
created_pr_number INTEGER,
created_pr_merged_at TEXT,
created_pr_state TEXT
);
CREATE TABLE workspace_session_aliases (
session_id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE TABLE workspace_repo_contexts (
workspace_id TEXT NOT NULL,
source_issue_number INTEGER,
source_pr_number INTEGER,
source_pr_merged_at TEXT,
source_pr_state TEXT,
created_pr_number INTEGER,
created_pr_merged_at TEXT,
created_pr_state TEXT
);
`;

const SESSION_STORE_SCHEMA_SQL = `
CREATE TABLE sessions (
id TEXT PRIMARY KEY,
repository TEXT
);
CREATE TABLE session_refs (
session_id TEXT NOT NULL,
ref_type TEXT NOT NULL,
ref_value TEXT NOT NULL
);
CREATE TABLE turns (
session_id TEXT NOT NULL,
turn_index INTEGER NOT NULL,
user_message TEXT,
assistant_response TEXT
);
`;

const tempDirectories: string[] = [];

function createDatabase(databasePath: string, schema: string, fixtures: string): void {
const database = new DatabaseSync(databasePath);
database.exec(schema);
database.exec(fixtures);
database.close();
}

function createFixtureDatabases(
appFixtures: string,
sessionStoreFixtures = '',
): { databasePath: string; sessionStorePath: string } {
const directory = fs.mkdtempSync(path.join(os.homedir(), '.ai-engineer-coach-test-'));
tempDirectories.push(directory);
const databasePath = path.join(directory, 'data.db');
const sessionStorePath = path.join(directory, 'session-store.db');
createDatabase(databasePath, APP_SCHEMA_SQL, appFixtures);
createDatabase(sessionStorePath, SESSION_STORE_SCHEMA_SQL, sessionStoreFixtures);
return { databasePath, sessionStorePath };
}

async function loadFixture(
appFixtures: string,
sessionStoreFixtures = '',
): Promise<GitHubAppMetrics> {
const snapshot = await loadGitHubAppMetrics(
createFixtureDatabases(appFixtures, sessionStoreFixtures),
);
expect(snapshot.status).toBe('ready');
if (snapshot.status !== 'ready') throw new Error('Expected ready metrics.');
return snapshot.metrics;
}

function queryResult(
sql: string,
references: Record<string, unknown>[] = [],
workspaceLinks: Record<string, unknown>[] = [],
): string {
if (sql.includes('WITH RECURSIVE')) return JSON.stringify([row('2026-08-24')]);
if (sql.includes('FROM session_refs')) return JSON.stringify(references);
if (sql.includes('workspace_session_aliases')) return JSON.stringify(workspaceLinks);
throw new Error('Unexpected query');
function cohortTotals(metrics: GitHubAppMetrics): { raised: number; merged: number } {
return metrics.workspacePrCohorts.reduce(
(totals, day) => ({
raised: totals.raised + day.pullRequestsRaised,
merged: totals.merged + day.pullRequestsMerged,
}),
{ raised: 0, merged: 0 },
);
}

afterEach(() => {
for (const directory of tempDirectories.splice(0)) {
fs.rmSync(directory, { recursive: true, force: true });
}
});

describe('GitHub App analytics parsing', () => {
it('parses summary and daily PR merge cohorts', () => {
const metrics = parseGitHubAppMetricsRows(JSON.stringify([
row('2026-08-23'),
row('2026-08-24', { cohortPullRequestsRaised: 0, cohortPullRequestsMerged: 0 }),
]));

expect(metrics).toMatchObject({
totalProjectSessions: 12,
sessionsWithIssue: 7,
sessionsWithPullRequest: 8,
sessionsWithMergedPullRequest: 6,
});
expect(metrics.mergeHistory).toEqual([
it('parses workspace PR cohorts', () => {
expect(parseGitHubAppWorkspacePrCohorts(JSON.stringify([
{ date: '2026-08-23', cohortPullRequestsRaised: 4, cohortPullRequestsMerged: 3 },
{ date: '2026-08-24', cohortPullRequestsRaised: 0, cohortPullRequestsMerged: 0 },
]))).toEqual([
{ date: '2026-08-23', pullRequestsRaised: 4, pullRequestsMerged: 3 },
{ date: '2026-08-24', pullRequestsRaised: 0, pullRequestsMerged: 0 },
]);
});

it('parses issue-linked session identifiers', () => {
expect(parseGitHubAppIssueSessionReferences(JSON.stringify([
{ sessionId: 'session-1' },
{ sessionId: 'session-2' },
]))).toEqual(['session-1', 'session-2']);
});

});

describe('GitHub App analytics availability', () => {
it('hides the feature when the App database is absent', async () => {
const metrics = await loadGitHubAppMetrics({
databasePath: path.join(os.tmpdir(), 'missing-copilot-data.db'),
sessionStorePath: path.join(os.tmpdir(), 'missing-session-store.db'),
databasePath: path.join(os.homedir(), 'missing-copilot-data.db'),
sessionStorePath: path.join(os.homedir(), 'missing-session-store.db'),
exists: () => false,
query: () => Promise.reject(new Error('query should not run')),
});

expect(metrics).toEqual({ status: 'absent' });
});

it('keeps the feature visible when an installed database cannot be queried', async () => {
it('reports unavailable when the session store is absent', async () => {
const databasePath = path.join(os.homedir(), 'copilot-data.db');
const metrics = await loadGitHubAppMetrics({
databasePath: path.join(os.tmpdir(), 'copilot-data.db'),
sessionStorePath: path.join(os.tmpdir(), 'missing-session-store.db'),
databasePath,
sessionStorePath: path.join(os.homedir(), 'missing-session-store.db'),
exists: filePath => filePath === databasePath,
query: () => Promise.reject(new Error('query should not run')),
});

expect(metrics).toEqual({ status: 'unavailable' });
});

it('does not return fallback metrics after a query failure', async () => {
const metrics = await loadGitHubAppMetrics({
databasePath: path.join(os.homedir(), 'copilot-data.db'),
sessionStorePath: path.join(os.homedir(), 'session-store.db'),
exists: () => true,
query: () => Promise.reject(new Error('sqlite unavailable')),
});

expect(metrics).toEqual({ status: 'unavailable' });
});
});

it('loads seven completed days without querying the optional session store', async () => {
const databasePath = path.join(os.tmpdir(), 'copilot-data.db');
const queries: string[] = [];
const metrics = await loadGitHubAppMetrics({
databasePath,
sessionStorePath: path.join(os.tmpdir(), 'missing-session-store.db'),
exists: filePath => filePath === databasePath,
query: (_requestedPath, sql) => {
queries.push(sql);
return Promise.resolve(queryResult(sql));
},
describe('GitHub App delivery references', () => {
it('counts structured issues and early pasted links only', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES
('structured', 'project', '2026-08-20T10:00:00.000Z'),
('pasted', 'project', '2026-08-21T10:00:00.000Z'),
('late', 'project', '2026-08-22T10:00:00.000Z'),
('prose', 'project', '2026-08-23T10:00:00.000Z');
`, `
INSERT INTO sessions VALUES
('structured', 'org/repo'),
('pasted', 'org/repo'),
('late', 'org/repo'),
('prose', 'org/repo');
INSERT INTO session_refs VALUES ('structured', 'issue', '42');
INSERT INTO turns VALUES
('pasted', 0, 'Fix https://github.com/org/repo/issues/123', NULL),
('late', 2, 'Later: https://github.com/org/repo/issues/456', NULL),
('prose', 0, 'Issue 789 is not an explicit link', NULL);
`);

expect(metrics.delivery).toMatchObject({
totalProjectSessions: 4,
sessionsWithIssue: 2,
sessionsWithPullRequest: 0,
sessionsWithMergedPullRequest: 0,
});
});

it('counts a structured PR reference without assuming it merged', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES ('referenced-pr', 'project', '2026-08-20T10:00:00.000Z');
`, `
INSERT INTO sessions VALUES ('referenced-pr', 'org/repo');
INSERT INTO session_refs VALUES ('referenced-pr', 'pr', '99');
`);

expect(metrics.status).toBe('ready');
expect(queries).toHaveLength(1);
expect(queries[0]).toContain("VALUES(date('now', '-7 days'))");
expect(queries[0]).toContain("date('now', '-1 day')");
expect(metrics.delivery.sessionsWithPullRequest).toBe(1);
expect(metrics.delivery.sessionsWithMergedPullRequest).toBe(0);
});
});

describe('GitHub App issue references', () => {
it('counts only issue references that map to project workspaces', async () => {
const databasePath = path.join(os.tmpdir(), 'copilot-data.db');
const sessionStorePath = path.join(os.tmpdir(), 'session-store.db');
const metrics = await loadGitHubAppMetrics({
databasePath,
sessionStorePath,
exists: () => true,
query: (_requestedPath, sql) => Promise.resolve(queryResult(sql, [
{ sessionId: 'issue-session-1' },
{ sessionId: 'unrelated-issue-session' },
], [
{ workspaceId: 'workspace-direct', hasDirectIssue: 1, sessionId: null },
{ workspaceId: 'workspace-linked', hasDirectIssue: 0, sessionId: 'issue-session-1' },
{ workspaceId: 'workspace-linked', hasDirectIssue: 0, sessionId: 'alias-session' },
{ workspaceId: 'workspace-unlinked', hasDirectIssue: 0, sessionId: 'other-session' },
])),
describe('GitHub App merge evidence', () => {
it('counts only explicit assistant merge evidence', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES
('merged', 'project', '2026-08-20T10:00:00.000Z'),
('not-merged', 'project', '2026-08-21T10:00:00.000Z'),
('merge-request', 'project', '2026-08-22T10:00:00.000Z');
`, `
INSERT INTO sessions VALUES
('merged', 'org/repo'),
('not-merged', 'org/repo'),
('merge-request', 'org/repo');
INSERT INTO session_refs VALUES
('merged', 'pr', '1'),
('not-merged', 'pr', '2'),
('merge-request', 'pr', '3');
INSERT INTO turns VALUES
('merged', 2, NULL, 'The pull request has been merged.'),
('not-merged', 2, NULL, 'The pull request is not merged.'),
('merge-request', 2, NULL, 'I can merge the pull request after confirmation.');
`);

expect(metrics.delivery.sessionsWithPullRequest).toBe(3);
expect(metrics.delivery.sessionsWithMergedPullRequest).toBe(1);
});
});

describe('GitHub App delivery ownership', () => {
it('attributes a workspace outcome only to its primary session', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES
('primary', 'project', '2026-08-20T10:00:00.000Z'),
('stale-alias', 'project', '2026-08-21T10:00:00.000Z');
INSERT INTO workspaces VALUES
('workspace', 'primary', datetime('now', '-1 day'), 7, NULL, NULL, NULL, 101, datetime('now'), 'merged');
INSERT INTO workspace_session_aliases VALUES
('stale-alias', 'workspace', datetime('now', '-2 days'));
`);

expect(metrics.delivery).toMatchObject({
totalProjectSessions: 2,
sessionsWithIssue: 1,
sessionsWithPullRequest: 1,
sessionsWithMergedPullRequest: 1,
});
});

it('uses the latest alias when a workspace has no primary session', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES
('latest-alias', 'project', '2026-08-20T10:00:00.000Z'),
('older-alias', 'general_chat', '2026-08-21T10:00:00.000Z');
INSERT INTO workspaces VALUES
('workspace', NULL, datetime('now', '-1 day'), NULL, NULL, NULL, NULL, 101, NULL, 'open');
INSERT INTO workspace_session_aliases VALUES
('older-alias', 'workspace', datetime('now', '-2 days')),
('latest-alias', 'workspace', datetime('now', '-1 hour'));
`);

expect(metrics.delivery.totalProjectSessions).toBe(1);
expect(metrics.delivery.sessionsWithPullRequest).toBe(1);
});
});

describe('GitHub App PR outcome integrity', () => {
it('requires a created PR number before counting merge metadata', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES ('merge-state-only', 'project', '2026-08-20T10:00:00.000Z');
INSERT INTO workspaces VALUES
('workspace', 'merge-state-only', datetime('now', '-1 day'), NULL, NULL, NULL, NULL, NULL, datetime('now'), 'merged');
`);

expect(metrics.delivery.sessionsWithPullRequest).toBe(0);
expect(metrics.delivery.sessionsWithMergedPullRequest).toBe(0);
expect(cohortTotals(metrics)).toEqual({ raised: 0, merged: 0 });
});

it('uses context PRs without also counting legacy workspace fields', async () => {
const metrics = await loadFixture(`
INSERT INTO sessions VALUES ('owner', 'project', '2026-08-20T10:00:00.000Z');
INSERT INTO workspaces VALUES
('workspace', 'owner', datetime('now', '-1 day'), NULL, NULL, NULL, NULL, 100, NULL, 'open');
INSERT INTO workspace_repo_contexts VALUES
('workspace', NULL, NULL, NULL, NULL, 200, datetime('now'), 'merged');
`);

expect(metrics.status).toBe('ready');
if (metrics.status !== 'ready') throw new Error('Expected ready metrics.');
expect(metrics.metrics.sessionsWithIssue).toBe(2);
expect(metrics.delivery.sessionsWithPullRequest).toBe(1);
expect(metrics.delivery.sessionsWithMergedPullRequest).toBe(1);
expect(cohortTotals(metrics)).toEqual({ raised: 1, merged: 1 });
});
});
Loading
Loading