Skip to content

Commit 6af031c

Browse files
brendan-kellambrendan-kellam
committed
wip
1 parent cf7f85b commit 6af031c

File tree

13 files changed

+186
-120
lines changed

13 files changed

+186
-120
lines changed

‎packages/backend/src/permissionSyncer.ts‎

Lines changed: 105 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
import { PrismaClient } from "@sourcebot/db";
1+
import * as Sentry from "@sentry/node";
2+
import { PrismaClient, Repo, RepoPermissionSyncStatus } from "@sourcebot/db";
23
import { createLogger } from "@sourcebot/logger";
34
import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type";
45
import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
56
import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
67
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
78
import { Job, Queue, Worker } from 'bullmq';
89
import { Redis } from 'ioredis';
10+
import { env } from "./env.js";
911
import { createOctokitFromConfig, getUserIdsWithReadAccessToRepo } from "./github.js";
1012
import { RepoWithConnections } from "./types.js";
1113

@@ -17,6 +19,8 @@ const QUEUE_NAME = 'repoPermissionSyncQueue';
1719

1820
const logger = createLogger('permission-syncer');
1921

22+
const SUPPORTED_CODE_HOST_TYPES = ['github'];
23+
2024
export class RepoPermissionSyncer {
2125
private queue: Queue<RepoPermissionSyncJob>;
2226
private worker: Worker<RepoPermissionSyncJob>;
@@ -30,48 +34,94 @@ export class RepoPermissionSyncer {
3034
});
3135
this.worker = new Worker<RepoPermissionSyncJob>(QUEUE_NAME, this.runJob.bind(this), {
3236
connection: redis,
37+
concurrency: 1,
3338
});
3439
this.worker.on('completed', this.onJobCompleted.bind(this));
3540
this.worker.on('failed', this.onJobFailed.bind(this));
3641
}
3742

38-
public async scheduleJob(repoId: number) {
39-
await this.queue.add(QUEUE_NAME, {
40-
repoId,
41-
});
42-
}
43-
4443
public startScheduler() {
4544
logger.debug('Starting scheduler');
4645

47-
// @todo: we should only sync permissions for a repository if it has been at least ~24 hours since the last sync.
4846
return setInterval(async () => {
47+
// @todo: make this configurable
48+
const thresholdDate = new Date(Date.now() - 1000 * 60 * 60 * 24);
4949
const repos = await this.db.repo.findMany({
50+
// Repos need their permissions to be synced against the code host when...
5051
where: {
51-
external_codeHostType: {
52-
in: ['github'],
53-
}
52+
// They belong to a code host that supports permissions syncing
53+
AND: [
54+
{
55+
external_codeHostType: {
56+
in: SUPPORTED_CODE_HOST_TYPES,
57+
}
58+
},
59+
// and, they either require a sync (SYNC_NEEDED) or have been in a completed state (SYNCED or FAILED)
60+
// for > some duration (default 24 hours)
61+
{
62+
OR: [
63+
{
64+
permissionSyncStatus: RepoPermissionSyncStatus.SYNC_NEEDED
65+
},
66+
{
67+
AND: [
68+
{
69+
OR: [
70+
{ permissionSyncStatus: RepoPermissionSyncStatus.SYNCED },
71+
{ permissionSyncStatus: RepoPermissionSyncStatus.FAILED },
72+
]
73+
},
74+
{
75+
OR: [
76+
{ permissionSyncJobLastCompletedAt: null },
77+
{ permissionSyncJobLastCompletedAt: { lt: thresholdDate } }
78+
]
79+
}
80+
]
81+
}
82+
]
83+
},
84+
]
5485
}
5586
});
5687

57-
for (const repo of repos) {
58-
await this.scheduleJob(repo.id);
59-
}
60-
61-
// @todo: make this configurable
62-
}, 1000 * 60);
88+
await this.schedulePermissionSync(repos);
89+
}, 1000 * 30);
6390
}
6491

6592
public dispose() {
6693
this.worker.close();
6794
this.queue.close();
6895
}
6996

97+
private async schedulePermissionSync(repos: Repo[]) {
98+
await this.db.$transaction(async (tx) => {
99+
await tx.repo.updateMany({
100+
where: { id: { in: repos.map(repo => repo.id) } },
101+
data: { permissionSyncStatus: RepoPermissionSyncStatus.IN_SYNC_QUEUE },
102+
});
103+
104+
await this.queue.addBulk(repos.map(repo => ({
105+
name: 'repoPermissionSyncJob',
106+
data: {
107+
repoId: repo.id,
108+
},
109+
opts: {
110+
removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE,
111+
removeOnFail: env.REDIS_REMOVE_ON_FAIL,
112+
}
113+
})))
114+
});
115+
}
116+
70117
private async runJob(job: Job<RepoPermissionSyncJob>) {
71118
const id = job.data.repoId;
72-
const repo = await this.db.repo.findUnique({
119+
const repo = await this.db.repo.update({
73120
where: {
74-
id,
121+
id
122+
},
123+
data: {
124+
permissionSyncStatus: RepoPermissionSyncStatus.SYNCING,
75125
},
76126
include: {
77127
connections: {
@@ -86,6 +136,8 @@ export class RepoPermissionSyncer {
86136
throw new Error(`Repo ${id} not found`);
87137
}
88138

139+
logger.info(`Syncing permissions for repo ${repo.displayName}...`);
140+
89141
const connection = getFirstConnectionWithToken(repo);
90142
if (!connection) {
91143
throw new Error(`No connection with token found for repo ${id}`);
@@ -119,8 +171,6 @@ export class RepoPermissionSyncer {
119171
return [];
120172
})();
121173

122-
logger.info(`User IDs with read access to repo ${id}: ${userIds}`);
123-
124174
await this.db.repo.update({
125175
where: {
126176
id: repo.id,
@@ -141,11 +191,43 @@ export class RepoPermissionSyncer {
141191
}
142192

143193
private async onJobCompleted(job: Job<RepoPermissionSyncJob>) {
144-
logger.info(`Repo permission sync job completed for repo ${job.data.repoId}`);
194+
const repo = await this.db.repo.update({
195+
where: {
196+
id: job.data.repoId,
197+
},
198+
data: {
199+
permissionSyncStatus: RepoPermissionSyncStatus.SYNCED,
200+
permissionSyncJobLastCompletedAt: new Date(),
201+
},
202+
});
203+
204+
logger.info(`Permissions synced for repo ${repo.displayName ?? repo.name}`);
145205
}
146206

147207
private async onJobFailed(job: Job<RepoPermissionSyncJob> | undefined, err: Error) {
148-
logger.error(`Repo permission sync job failed for repo ${job?.data.repoId}: ${err}`);
208+
Sentry.captureException(err, {
209+
tags: {
210+
repoId: job?.data.repoId,
211+
queue: QUEUE_NAME,
212+
}
213+
});
214+
215+
const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err}`;
216+
217+
if (job) {
218+
const repo = await this.db.repo.update({
219+
where: {
220+
id: job?.data.repoId,
221+
},
222+
data: {
223+
permissionSyncStatus: RepoPermissionSyncStatus.FAILED,
224+
permissionSyncJobLastCompletedAt: new Date(),
225+
},
226+
});
227+
logger.error(errorMessage(repo.displayName ?? repo.name));
228+
} else {
229+
logger.error(errorMessage('unknown repo (id not found)'));
230+
}
149231
}
150232
}
151233

‎packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
-- CreateEnum
2+
CREATE TYPE "RepoPermissionSyncStatus" AS ENUM ('SYNC_NEEDED', 'IN_SYNC_QUEUE', 'SYNCING', 'SYNCED', 'FAILED');
3+
4+
-- AlterTable
5+
ALTER TABLE "Repo" ADD COLUMN "permissionSyncJobLastCompletedAt" TIMESTAMP(3),
6+
ADD COLUMN "permissionSyncStatus" "RepoPermissionSyncStatus" NOT NULL DEFAULT 'SYNC_NEEDED';

‎packages/db/prisma/schema.prisma‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ enum ConnectionSyncStatus {
3030
FAILED
3131
}
3232

33+
enum RepoPermissionSyncStatus {
34+
SYNC_NEEDED
35+
IN_SYNC_QUEUE
36+
SYNCING
37+
SYNCED
38+
FAILED
39+
}
40+
3341
enum StripeSubscriptionStatus {
3442
ACTIVE
3543
INACTIVE
@@ -58,6 +66,10 @@ model Repo {
5866
repoIndexingStatus RepoIndexingStatus @default(NEW)
5967
permittedUsers UserToRepoPermission[]
6068
69+
permissionSyncStatus RepoPermissionSyncStatus @default(SYNC_NEEDED)
70+
/// When the repo permissions were last synced, either successfully or unsuccessfully.
71+
permissionSyncJobLastCompletedAt DateTime?
72+
6173
// The id of the repo in the external service
6274
external_id String
6375
// The type of the external service (e.g., github, gitlab, etc.)

‎packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,15 @@ interface CodePreviewPanelProps {
1010
path: string;
1111
repoName: string;
1212
revisionName?: string;
13-
domain: string;
1413
}
1514

16-
export const CodePreviewPanel = async ({ path, repoName, revisionName, domain }: CodePreviewPanelProps) => {
15+
export const CodePreviewPanel = async ({ path, repoName, revisionName }: CodePreviewPanelProps) => {
1716
const [fileSourceResponse, repoInfoResponse] = await Promise.all([
1817
getFileSource({
1918
fileName: path,
2019
repository: repoName,
2120
branch: revisionName,
22-
}, domain),
21+
}),
2322
getRepoInfoByName(repoName),
2423
]);
2524

‎packages/web/src/app/[domain]/browse/[...path]/page.tsx‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import { TreePreviewPanel } from "./components/treePreviewPanel";
77
interface BrowsePageProps {
88
params: Promise<{
99
path: string[];
10-
domain: string;
1110
}>;
1211
}
1312

@@ -16,7 +15,6 @@ export default async function BrowsePage(props: BrowsePageProps) {
1615

1716
const {
1817
path: _rawPath,
19-
domain
2018
} = params;
2119

2220
const rawPath = _rawPath.join('/');
@@ -35,7 +33,6 @@ export default async function BrowsePage(props: BrowsePageProps) {
3533
path={path}
3634
repoName={repoName}
3735
revisionName={revisionName}
38-
domain={domain}
3936
/>
4037
) : (
4138
<TreePreviewPanel

‎packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import { useQuery } from "@tanstack/react-query";
44
import { CodePreview } from "./codePreview";
55
import { SearchResultFile } from "@/features/search/types";
6-
import { useDomain } from "@/hooks/useDomain";
76
import { SymbolIcon } from "@radix-ui/react-icons";
87
import { SetStateAction, Dispatch, useMemo } from "react";
98
import { getFileSource } from "@/features/search/fileSourceApi";
@@ -22,7 +21,6 @@ export const CodePreviewPanel = ({
2221
onClose,
2322
onSelectedMatchIndexChange,
2423
}: CodePreviewPanelProps) => {
25-
const domain = useDomain();
2624

2725
// If there are multiple branches pointing to the same revision of this file, it doesn't
2826
// matter which branch we use here, so use the first one.
@@ -31,13 +29,13 @@ export const CodePreviewPanel = ({
3129
}, [previewedFile]);
3230

3331
const { data: file, isLoading, isPending, isError } = useQuery({
34-
queryKey: ["source", previewedFile, branch, domain],
32+
queryKey: ["source", previewedFile, branch],
3533
queryFn: () => unwrapServiceError(
3634
getFileSource({
3735
fileName: previewedFile.fileName.text,
3836
repository: previewedFile.repository,
3937
branch,
40-
}, domain)
38+
})
4139
),
4240
select: (data) => {
4341
return {

‎packages/web/src/app/api/(server)/search/route.ts‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,8 @@ import { isServiceError } from "@/lib/utils";
55
import { NextRequest } from "next/server";
66
import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
77
import { searchRequestSchema } from "@/features/search/schemas";
8-
import { ErrorCode } from "@/lib/errorCodes";
9-
import { StatusCodes } from "http-status-codes";
108

119
export const POST = async (request: NextRequest) => {
12-
const domain = request.headers.get("X-Org-Domain");
13-
const apiKey = request.headers.get("X-Sourcebot-Api-Key") ?? undefined;
14-
if (!domain) {
15-
return serviceErrorResponse({
16-
statusCode: StatusCodes.BAD_REQUEST,
17-
errorCode: ErrorCode.MISSING_ORG_DOMAIN_HEADER,
18-
message: "Missing X-Org-Domain header",
19-
});
20-
}
21-
2210
const body = await request.json();
2311
const parsed = await searchRequestSchema.safeParseAsync(body);
2412
if (!parsed.success) {
@@ -27,7 +15,7 @@ export const POST = async (request: NextRequest) => {
2715
);
2816
}
2917

30-
const response = await search(parsed.data, domain, apiKey);
18+
const response = await search(parsed.data);
3119
if (isServiceError(response)) {
3220
return serviceErrorResponse(response);
3321
}

‎packages/web/src/app/api/(server)/source/route.ts‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,8 @@ import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"
55
import { isServiceError } from "@/lib/utils";
66
import { NextRequest } from "next/server";
77
import { fileSourceRequestSchema } from "@/features/search/schemas";
8-
import { ErrorCode } from "@/lib/errorCodes";
9-
import { StatusCodes } from "http-status-codes";
108

119
export const POST = async (request: NextRequest) => {
12-
const domain = request.headers.get("X-Org-Domain");
13-
const apiKey = request.headers.get("X-Sourcebot-Api-Key") ?? undefined;
14-
if (!domain) {
15-
return serviceErrorResponse({
16-
statusCode: StatusCodes.BAD_REQUEST,
17-
errorCode: ErrorCode.MISSING_ORG_DOMAIN_HEADER,
18-
message: "Missing X-Org-Domain header",
19-
});
20-
}
21-
2210
const body = await request.json();
2311
const parsed = await fileSourceRequestSchema.safeParseAsync(body);
2412
if (!parsed.success) {
@@ -27,7 +15,7 @@ export const POST = async (request: NextRequest) => {
2715
);
2816
}
2917

30-
const response = await getFileSource(parsed.data, domain, apiKey);
18+
const response = await getFileSource(parsed.data);
3119
if (isServiceError(response)) {
3220
return serviceErrorResponse(response);
3321
}

‎packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/revie
22
import { getFileSource } from "@/features/search/fileSourceApi";
33
import { fileSourceResponseSchema } from "@/features/search/schemas";
44
import { isServiceError } from "@/lib/utils";
5-
import { env } from "@/env.mjs";
65
import { createLogger } from "@sourcebot/logger";
76

87
const logger = createLogger('fetch-file-content');
@@ -17,7 +16,7 @@ export const fetchFileContent = async (pr_payload: sourcebot_pr_payload, filenam
1716
}
1817
logger.debug(JSON.stringify(fileSourceRequest, null, 2));
1918

20-
const response = await getFileSource(fileSourceRequest, "~", env.REVIEW_AGENT_API_KEY);
19+
const response = await getFileSource(fileSourceRequest);
2120
if (isServiceError(response)) {
2221
throw new Error(`Failed to fetch file content for ${filename} from ${repoPath}: ${response.message}`);
2322
}

0 commit comments

Comments
 (0)