diff --git a/bin/tcfeed.ts b/bin/tcfeed.ts index 7453e1a..01f53ec 100755 --- a/bin/tcfeed.ts +++ b/bin/tcfeed.ts @@ -1,7 +1,13 @@ #!/usr/bin/env -S npx tsx /** - * tcfeed — read the newest posts on a subreddit, find the repositories they - * link, scan each one, and print a shortlist worth reading. + * tcfeed — find repositories worth a look, scan each one, and print a + * shortlist worth reading. + * + * Two sources, merged and deduplicated: the newest posts on a subreddit, and + * GitHub's own repository search sorted by most recently updated. Either is + * allowed to fail on its own — reddit refuses this address often enough that + * making it fatal meant a whole run produced nothing on an afternoon when the + * search was answering perfectly. * * The scan reports, and that is all the scan does. It does not fork anything * and it does not open pull requests off its own findings. Four repositories @@ -33,6 +39,10 @@ * TCFEED_MAX repos cloned per run, default 20 * TCFEED_PAUSE seconds between clones, default 1 * TCFEED_SUB subreddit, default coolgithubprojects + * TCFEED_GH results taken from the search, default 25, 0 turns it off + * TCFEED_GH_QUERY the search, default `stars:>1000`. The qualifier is + * `stars`; `starts` is a free-text search that returns + * repositories with no stars at all and looks like it worked * TC_BIN the scanner, default whatever `threatcrush` resolves to * TCFEED_CACHE where seen repos and reports live, default ~/.cache/tcfeed * @@ -218,6 +228,59 @@ function reposIn(body: string): string[] { return [...found].sort(); } +/** + * The other source: GitHub's own repository search, newest activity first. + * + * This is the API behind + * https://github.com/search?q=stars:>1000&type=repositories&s=updated&o=desc, + * asked through gh so it uses the token already on this machine — the HTML + * page is rate-limited hard for anyone not signed in, and parsing it would be + * a scraper of a page that changes shape without warning. + * + * `stars:>1000` — more than a thousand, sorted by most recently pushed. The + * qualifier is `stars`, and it is worth being careful about: `starts:>1000` is + * not an error, it is a *free-text search* for the word, and it quietly + * returns repositories with no stars at all. A query that is wrong in that + * direction looks like it worked. + * + * Override with TCFEED_GH_QUERY, which takes any GitHub search qualifier. + * + * Archived and forked repositories are dropped here rather than left for + * metadata() to reject one HTTP call later, because the search already knows. + * Size is not filtered here — the search has no qualifier for it — so the + * TOO_BIG_KB check downstream does more work with this query than the feed + * ever gave it: a repository with this many stars is often a monorepo. + */ +async function searchRepos(query: string, limit: number): Promise { + const { stdout } = await run( + 'gh', + [ + 'search', + 'repos', + query, + '--sort', + 'updated', + '--order', + 'desc', + '--limit', + String(limit), + '--json', + 'fullName,isArchived,isFork', + ], + { maxBuffer: 32 * 1024 * 1024 } + ); + + const found = JSON.parse(stdout) as { + fullName: string; + isArchived: boolean; + isFork: boolean; + }[]; + + return found + .filter((entry) => !entry.isArchived && !entry.isFork && entry.fullName) + .map((entry) => entry.fullName); +} + async function metadata( repo: string ): Promise<{ archived: boolean; sizeKb: number; stars: number } | null> { @@ -1010,20 +1073,61 @@ async function main(): Promise { fs.appendFileSync(seenFile, `${repo}\n`); }; - let body: string; + // Two sources, and either one is allowed to fail. + // + // Reddit refuses this address often enough that making it fatal meant a run + // produced nothing on an afternoon when the other source was answering + // perfectly. Whichever source came back is scanned; only losing both is an + // error, and a source that failed says so rather than looking empty. + const fromReddit: string[] = []; + const fromSearch: string[] = []; + const broke: string[] = []; + try { - body = await readFeed(sub, limit, cache); + fromReddit.push(...reposIn(await readFeed(sub, limit, cache))); } catch (error) { - console.error(`tcfeed: ${(error as Error).message}`); - return 1; + broke.push(`reddit: ${(error as Error).message}`); } - const repos = reposIn(body); - if (repos.length === 0) { - console.log('tcfeed: the feed mentioned no repositories'); + const query = process.env.TCFEED_GH_QUERY ?? 'stars:>1000'; + const searchWanted = num('TCFEED_GH', 25); + if (searchWanted > 0 && query) { + try { + fromSearch.push(...(await searchRepos(query, searchWanted))); + } catch (error) { + broke.push(`github search: ${(error as Error).message}`); + } + } + + for (const why of broke) console.log(`tcfeed: ${why}`); + + if (fromReddit.length === 0 && fromSearch.length === 0) { + // Both empty is only an error if both were *asked*. A run with the search + // turned off and a quiet feed has simply found nothing today. + if (broke.length > 0) { + console.error('tcfeed: no source answered'); + return 1; + } + console.log('tcfeed: neither source mentioned a repository'); return 0; } + // Deduplicated across sources, because a repository trending on reddit is + // exactly the kind that also turns up in a search sorted by recent activity, + // and cloning it twice in one run is the one thing worth avoiding here. + const seenInThisRun = new Set(); + const repos = [...fromReddit, ...fromSearch].filter((repo) => { + if (seenInThisRun.has(repo)) return false; + seenInThisRun.add(repo); + return true; + }); + + const overlap = fromReddit.length + fromSearch.length - repos.length; + console.log( + `tcfeed: ${repos.length} to consider — ${fromReddit.length} from r/${sub}, ` + + `${fromSearch.length} from search (${query})${overlap > 0 ? `, ${overlap} in both` : ''}` + ); + // A run is capped and paced. One invocation that clones ninety repositories // back to back is a scraper, and the point of this is a shortlist to read // rather than a mirror of the feed.