Refactor seeding logic to be testable and reusable

The logic used to fetch and store information about repositories and
dependencies will need to be reused to allow webhooks to make changes to
the database. Using the logic in the application rather than in one-off
seeding scripts makes it more important that it is tested to avoid
unexpected changes to behaviour. This moves the actual seed scripts to
wrapper scripts in the `seed` folder, which call the methods in the
original files that have been refactored to be more testable.

db/index.js

@@ -31,6 +31,7 @@ export class TowtruckDatabase {
   #getStatement;
   #getAllForNameStatement;
   #getAllForScopeStatement;
+  #deleteAllForScopeStatement;
 
   constructor(filename = "./data/towtruck.db", options) {
     this.#db = initialiseDatabase(filename, options);
@@ -39,6 +40,7 @@ export class TowtruckDatabase {
     this.#getStatement = this.#db.prepare("SELECT value FROM towtruck_data WHERE scope = ? AND name = ? AND key = ?;");
     this.#getAllForNameStatement = this.#db.prepare("SELECT key, value FROM towtruck_data WHERE scope = ? AND name = ?;");
     this.#getAllForScopeStatement = this.#db.prepare("SELECT name, key, value FROM towtruck_data WHERE scope = ?;");
+    this.#deleteAllForScopeStatement = this.#db.prepare("DELETE FROM towtruck_data WHERE scope = ?;");
   }
 
   #save(scope, name, key, data) {
@@ -75,6 +77,10 @@ export class TowtruckDatabase {
     return result;
   }
 
+  #deleteAllForScope(scope) {
+    return this.#deleteAllForScopeStatement.run(scope);
+  }
+
   saveToRepository(name, key, data) {
     return this.#save("repository", name, key, data);
   }
@@ -91,6 +97,10 @@ export class TowtruckDatabase {
     return this.#getAllForScope("repository");
   }
 
+  deleteAllRepositories() {
+    return this.#deleteAllForScope("repository");
+  }
+
   saveToDependency(name, key, data) {
     return this.#save("dependency", name, key, data);
   }
@@ -107,6 +117,10 @@ export class TowtruckDatabase {
     return this.#getAllForScope("dependency");
   }
 
+  deleteAllDependencies() {
+    return this.#deleteAllForScope("dependency");
+  }
+
   transaction(fn) {
     return this.#db.transaction(fn);
   }

db/index.test.js

@@ -228,6 +228,43 @@ describe("TowtruckDatabase", () => {
     });
   });
 
+  describe("deleteAllRepositories", () => {
+    it("removes the expected data from the table", () => {
+      const db = new TowtruckDatabase(testDbPath);
+
+      const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);");
+      const selectStatement = new Database(testDbPath).prepare("SELECT COUNT(*) FROM towtruck_data WHERE scope = 'repository';");
+
+      const testRepoSomeData = {
+        array: [1, 2, 3],
+        text: "Text",
+        object: {
+          boolean: true,
+          missing: null,
+        },
+      };
+
+      const testRepoSomeOtherData = {
+        foo: "bar",
+        baz: false,
+        quux: 0.123456789,
+      };
+
+      const anotherRepoSomeData = [1, "foo", true, null];
+
+      insertStatement.run("repository", "test-repo", "some-data", JSON.stringify(testRepoSomeData));
+      insertStatement.run("repository", "test-repo", "some-other-data", JSON.stringify(testRepoSomeOtherData));
+      insertStatement.run("repository", "another-repo", "some-data", JSON.stringify(anotherRepoSomeData));
+      insertStatement.run("dependency", "test-dependency", "some-data", JSON.stringify({}));
+
+      db.deleteAllRepositories();
+
+      const actual = selectStatement.get();
+
+      expect.deepStrictEqual(actual, { "COUNT(*)": 0 });
+    });
+  });
+
   describe("saveToDependency", () => {
     it("inserts the expected data into the table", () => {
       const db = new TowtruckDatabase(testDbPath);
@@ -415,4 +452,41 @@ describe("TowtruckDatabase", () => {
       expect.deepStrictEqual(actual, {});
     });
   });
+
+  describe("deleteAllDependencies", () => {
+    it("removes the expected data from the table", () => {
+      const db = new TowtruckDatabase(testDbPath);
+
+      const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);");
+      const selectStatement = new Database(testDbPath).prepare("SELECT COUNT(*) FROM towtruck_data WHERE scope = 'dependency';");
+
+      const testDepSomeData = {
+        array: [1, 2, 3],
+        text: "Text",
+        object: {
+          boolean: true,
+          missing: null,
+        },
+      };
+
+      const testDepSomeOtherData = {
+        foo: "bar",
+        baz: false,
+        quux: 0.123456789,
+      };
+
+      const anotherDepSomeData = [1, "foo", true, null];
+
+      insertStatement.run("dependency", "test-dep", "some-data", JSON.stringify(testDepSomeData));
+      insertStatement.run("dependency", "test-dep", "some-other-data", JSON.stringify(testDepSomeOtherData));
+      insertStatement.run("dependency", "another-dep", "some-data", JSON.stringify(anotherDepSomeData));
+      insertStatement.run("repository", "test-repository", "some-data", JSON.stringify({}));
+
+      db.deleteAllDependencies();
+
+      const actual = selectStatement.get();
+
+      expect.deepStrictEqual(actual, { "COUNT(*)": 0 });
+    });
+  });
 });

package.json

@@ -12,8 +12,8 @@
     "lint:fix": "eslint . --fix",
     "format": "prettier --write .",
     "seed": "npm run seed:repos && npm run seed:lifetimes",
-    "seed:repos": "node --env-file=.env ./utils/githubApi/fetchAllRepos.js",
-    "seed:lifetimes": "node --env-file=.env ./utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js"
+    "seed:repos": "node --env-file=.env ./seed/repos.js",
+    "seed:lifetimes": "node --env-file=.env ./seed/lifetimes.js"
   },
   "author": "dxw",
   "license": "MIT",

seed/lifetimes.js

@@ -0,0 +1,20 @@
+import { TowtruckDatabase } from "../db/index.js";
+import {
+  fetchAllDependencyLifetimes,
+  saveAllDependencyLifetimes,
+} from "../utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js";
+import { EndOfLifeDateApiClient } from "../utils/endOfLifeDateApi/index.js";
+
+const seed = async () => {
+  const db = new TowtruckDatabase();
+
+  db.deleteAllDependencies();
+
+  const allLifetimes = await fetchAllDependencyLifetimes(
+    db,
+    new EndOfLifeDateApiClient(),
+  );
+  await saveAllDependencyLifetimes(allLifetimes, db);
+};
+
+await seed();

seed/repos.js

@@ -0,0 +1,34 @@
+import { TowtruckDatabase } from "../db/index.js";
+import {
+  fetchForRepo,
+  saveAllRepos,
+} from "../utils/githubApi/fetchAllRepos.js";
+import { OctokitApp } from "../octokitApp.js";
+
+/**
+ * Fetches all repos from the GitHub API.
+ * Each repo is enriched with data fetched through further API calls
+ * @returns {Promise<StoredRepo[]>}
+ */
+const fetchAllRepos = async () => {
+  const repos = [];
+
+  await OctokitApp.app.eachRepository(async ({ repository, octokit }) => {
+    const repo = await fetchForRepo(repository, octokit);
+
+    if (repo) repos.push(repo);
+  });
+
+  return repos;
+};
+
+const seed = async () => {
+  const db = new TowtruckDatabase();
+
+  db.deleteAllRepositories();
+
+  const allRepos = await fetchAllRepos();
+  saveAllRepos(allRepos, db);
+};
+
+await seed();

utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js

@@ -1,6 +1,3 @@
-import { EndOfLifeDateApiClient } from "./index.js";
-import { TowtruckDatabase } from "../../db/index.js";
-
 /**
  * @typedef {Object} DependencyLifetimes
  * @property {string} dependency - The name of the dependency
@@ -11,19 +8,18 @@ import { TowtruckDatabase } from "../../db/index.js";
  * Fetches all lifetime information for all repository dependencies from the endoflife.date
  * API.
  * Dependencies for which no information could be found are ignored.
+ * @param {TowtruckDatabase} db
+ * @param {EndOfLifeDateApiClient} apiClient
  * @returns {Promise<DependencyLifetimes[]>}
  */
-const fetchAllDependencyLifetimes = async () => {
-  const db = new TowtruckDatabase();
+export const fetchAllDependencyLifetimes = async (db, apiClient) => {
   const persistedRepoData = db.getAllRepositories();
 
   const dependencySet = new Set();
   Object.entries(persistedRepoData)
     .flatMap(([, repo]) => repo.dependencies)
     .forEach((dependency) => dependencySet.add(dependency.name));
 
-  const apiClient = new EndOfLifeDateApiClient();
-
   const lifetimes = await Promise.all(
     dependencySet.values().map(async (dependency) => {
       const response = await apiClient.getAllCycles(dependency);
@@ -43,14 +39,11 @@ const fetchAllDependencyLifetimes = async () => {
 
 /**
  * Saves all lifetime information for all repository dependencies to a JSON file.
+ * @param {DependencyLifetimes[]} allLifetimes
+ * @param {TowtruckDatabase} db
  */
-export const saveAllDependencyLifetimes = async () => {
-  console.info("Fetching all dependency EOL info...");
-  const allLifetimes = await fetchAllDependencyLifetimes();
-
+export const saveAllDependencyLifetimes = async (allLifetimes, db) => {
   try {
-    const db = new TowtruckDatabase();
-
     console.info("Saving all dependency EOL info...");
     const saveAllLifetimes = db.transaction((lifetimes) => {
       lifetimes.forEach((item) => db.saveToDependency(item.dependency, "lifetimes", item.lifetimes))
@@ -61,5 +54,3 @@ export const saveAllDependencyLifetimes = async () => {
     console.error("Error saving all dependency EOL info", error);
   }
 };
-
-await saveAllDependencyLifetimes();