Add feed discovery plugin

Author: ralyodio

apps/commandboard-api/package.json

@@ -15,6 +15,7 @@
     "@logicsrc/plugin-core": "file:../../packages/plugin-core",
     "@logicsrc/plugin-c0mpute": "file:../../plugins/c0mpute",
     "@logicsrc/plugin-coinpay": "file:../../plugins/coinpay",
+    "@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery",
     "@logicsrc/plugin-sh1pt": "file:../../plugins/sh1pt",
     "@logicsrc/plugin-ugig": "file:../../plugins/ugig",
     "@logicsrc/validators": "file:../../packages/validators"

apps/commandboard-api/src/contract.test.ts

@@ -47,7 +47,7 @@ describe("CommandBoard API contracts", () => {
     };
 
     expect(response.status).toBe(200);
-    expect(body.plugins.map((plugin) => plugin.id)).toEqual(["coinpay", "ugig", "sh1pt", "c0mpute"]);
+    expect(body.plugins.map((plugin) => plugin.id)).toEqual(["coinpay", "ugig", "sh1pt", "c0mpute", "feed-discovery"]);
     expect(body.plugins.find((plugin) => plugin.id === "sh1pt")).toMatchObject({
       enabled: true,
       capabilities: expect.arrayContaining(["projects.sync", "actions.publish", "deployments.status"])
@@ -58,6 +58,28 @@ describe("CommandBoard API contracts", () => {
     });
     expect(body.capabilities["actions.publish"]).toEqual(["sh1pt"]);
     expect(body.capabilities["compute.jobs.dispatch"]).toEqual(["c0mpute"]);
+    expect(body.capabilities["feeds.discover"]).toEqual(["feed-discovery"]);
+  });
+
+  it("exposes feed discovery plugin endpoints", async () => {
+    const providersResponse = await fetch(`${baseUrl}/api/feeds/providers`);
+    const providersBody = await providersResponse.json() as { providers: Array<{ id: string; enabledByDefault: boolean }> };
+    const discoverResponse = await fetch(`${baseUrl}/api/feeds/discover?q=microsaas&providers=manual-curated&includeUnvalidated=true`);
+    const discoverBody = await discoverResponse.json() as { query: string; results: Array<{ feedUrl: string; provider: string }> };
+    const rssResponse = await fetch(`${baseUrl}/rss/discover/microsaas.xml?providers=manual-curated&includeUnvalidated=true`);
+    const rssBody = await rssResponse.text();
+    const opmlResponse = await fetch(`${baseUrl}/opml/discover/microsaas.xml?providers=manual-curated&includeUnvalidated=true`);
+    const opmlBody = await opmlResponse.text();
+
+    expect(providersResponse.status).toBe(200);
+    expect(providersBody.providers.map((provider) => provider.id)).toContain("manual-curated");
+    expect(discoverResponse.status).toBe(200);
+    expect(discoverBody.query).toBe("microsaas");
+    expect(discoverBody.results[0]).toMatchObject({ provider: "manual-curated" });
+    expect(rssResponse.status).toBe(200);
+    expect(rssBody).toContain("<rss");
+    expect(opmlResponse.status).toBe(200);
+    expect(opmlBody).toContain("<opml");
   });
 
   it("exposes sh1pt project and action contracts", async () => {

apps/commandboard-api/src/index.ts

@@ -3,11 +3,12 @@ import { pathToFileURL } from "node:url";
 import { createPluginRegistry } from "@logicsrc/plugin-core";
 import { c0mputePlugin } from "@logicsrc/plugin-c0mpute";
 import { coinPayPlugin } from "@logicsrc/plugin-coinpay";
+import { discoverFeeds, feedDiscoveryPlugin, listFeedProviders, renderAtom, renderJsonFeed, renderOpml, renderRss, type FeedKind } from "@logicsrc/plugin-feed-discovery";
 import { sh1ptPlugin } from "@logicsrc/plugin-sh1pt";
 import { uGigPlugin } from "@logicsrc/plugin-ugig";
 import { schemas, validate } from "@logicsrc/validators";
 
-const registry = createPluginRegistry([coinPayPlugin, uGigPlugin, sh1ptPlugin, c0mputePlugin]);
+const registry = createPluginRegistry([coinPayPlugin, uGigPlugin, sh1ptPlugin, c0mputePlugin, feedDiscoveryPlugin]);
 
 const boards = [
   { path: "/general", title: "General", description: "CommandBoard.run general discussion." },
@@ -69,7 +70,7 @@ async function route(request: IncomingMessage, response: ServerResponse) {
     json(response, 200, {
       ok: true,
       service: "commandboard-api",
-      endpoints: ["/health", "/api/boards", "/api/tasks", "/api/plugins", "/api/schemas"]
+      endpoints: ["/health", "/api/boards", "/api/tasks", "/api/plugins", "/api/schemas", "/api/feeds/discover", "/api/feeds/providers", "/rss/discover/:keyword.xml", "/opml/discover/:keyword.xml", "/atom/discover/:keyword.xml", "/json-feed/discover/:keyword.json"]
     });
     return;
   }
@@ -113,6 +114,53 @@ async function route(request: IncomingMessage, response: ServerResponse) {
     return;
   }
 
+  if (request.method === "GET" && (url.pathname === "/api/feeds/providers" || url.pathname === "/api/rss/providers")) {
+    json(response, 200, { providers: listFeedProviders() });
+    return;
+  }
+
+  if (request.method === "GET" && (url.pathname === "/api/feeds/discover" || url.pathname === "/api/rss/discover")) {
+    const query = url.searchParams.get("q");
+    if (!query) {
+      json(response, 422, { error: "Expected q query parameter" });
+      return;
+    }
+    const result = await discoverFeeds({
+      q: query,
+      type: (url.searchParams.get("type") ?? "all") as FeedKind | "all",
+      limit: numberParam(url.searchParams.get("limit")),
+      providers: listParam(url.searchParams.get("providers")),
+      includeUnvalidated: url.searchParams.get("includeUnvalidated") === "true"
+    });
+    json(response, 200, result);
+    return;
+  }
+
+  const formattedDiscover = matchFormattedDiscoverPath(url.pathname);
+  if (request.method === "GET" && formattedDiscover) {
+    const result = await discoverFeeds({
+      q: formattedDiscover.keyword,
+      type: "all",
+      limit: numberParam(url.searchParams.get("limit")),
+      providers: listParam(url.searchParams.get("providers")),
+      includeUnvalidated: url.searchParams.get("includeUnvalidated") === "true"
+    });
+    if (formattedDiscover.format === "rss") {
+      text(response, 200, "application/rss+xml; charset=utf-8", renderRss(result));
+      return;
+    }
+    if (formattedDiscover.format === "opml") {
+      text(response, 200, "text/x-opml; charset=utf-8", renderOpml(result));
+      return;
+    }
+    if (formattedDiscover.format === "atom") {
+      text(response, 200, "application/atom+xml; charset=utf-8", renderAtom(result));
+      return;
+    }
+    text(response, 200, "application/feed+json; charset=utf-8", renderJsonFeed(result));
+    return;
+  }
+
   if (request.method === "GET" && url.pathname === "/api/plugins/sh1pt/projects") {
     json(response, 200, { projects: sh1ptProjects });
     return;
@@ -194,6 +242,11 @@ function json(response: ServerResponse, status: number, data: unknown) {
   response.end(JSON.stringify(data, null, 2));
 }
 
+function text(response: ServerResponse, status: number, contentType: string, body: string) {
+  response.writeHead(status, { "content-type": contentType });
+  response.end(body);
+}
+
 async function readJson(request: IncomingMessage) {
   const chunks: Buffer[] = [];
   for await (const chunk of request) {
@@ -207,6 +260,41 @@ function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+function numberParam(value: string | null) {
+  if (!value) {
+    return undefined;
+  }
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+function listParam(value: string | null) {
+  return value
+    ?.split(",")
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+}
+
+function matchFormattedDiscoverPath(pathname: string) {
+  const rss = /^\/rss\/discover\/(.+)\.xml$/.exec(pathname);
+  if (rss) {
+    return { format: "rss" as const, keyword: decodeURIComponent(rss[1]) };
+  }
+  const opml = /^\/opml\/discover\/(.+)\.xml$/.exec(pathname) ?? /^\/rss\/discover\/(.+)\.opml$/.exec(pathname);
+  if (opml) {
+    return { format: "opml" as const, keyword: decodeURIComponent(opml[1]) };
+  }
+  const atom = /^\/atom\/discover\/(.+)\.xml$/.exec(pathname);
+  if (atom) {
+    return { format: "atom" as const, keyword: decodeURIComponent(atom[1]) };
+  }
+  const jsonFeed = /^\/json-feed\/discover\/(.+)\.json$/.exec(pathname);
+  if (jsonFeed) {
+    return { format: "json-feed" as const, keyword: decodeURIComponent(jsonFeed[1]) };
+  }
+  return undefined;
+}
+
 export function startCommandBoardServer(port = Number(process.env.PORT ?? 4010)) {
   const server = createCommandBoardServer();
   server.listen(port, () => {