From 255680835dff73096d11b80f4de9b60153bc3453 Mon Sep 17 00:00:00 2001
From: lcflight <lcarthur747@gmail.com>
Date: Mon, 11 Mar 2024 11:49:09 -0400
Subject: [PATCH 1/8] markdown file write

---
 .gitignore          |  1 +
 package.json        |  3 ++-
 scripts/pull.ts     | 28 ++++++++++++++++++++++++++++
 src/lib/env.ts      |  2 +-
 src/lib/getPosts.ts |  4 +++-
 5 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 scripts/pull.ts

diff --git a/.gitignore b/.gitignore
index b882a72..cb5dd13 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ package-lock.json
 # build output
 dist/
 .cache/
+src/content/posts/
 
 # generated types
 .astro/
diff --git a/package.json b/package.json
index c14176d..915c0a5 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,8 @@
     "lint": "eslint .",
     "format": "prettier --write .",
     "preinstall": "npx only-allow pnpm",
-    "puppeteer": "tsx ./scripts/puppeteer.ts"
+    "puppeteer": "tsx ./scripts/puppeteer.ts",
+    "md:pull": "tsx ./scripts/pull.ts"
   },
   "dependencies": {
     "@astrojs/prefetch": "^0.4.1",
diff --git a/scripts/pull.ts b/scripts/pull.ts
new file mode 100644
index 0000000..8321f9e
--- /dev/null
+++ b/scripts/pull.ts
@@ -0,0 +1,28 @@
+import fs from "fs";
+import path from "path";
+import "dotenv/config";
+import fetchPosts from "../src/lib/fetchPosts";
+
+const promises = fetchPosts();
+const dir = "./src/content/posts";
+
+console.log("Delete the directory if it does exist");
+if (fs.existsSync(dir)) {
+  fs.rmdirSync(dir, { recursive: true });
+}
+
+console.log("Create directory");
+fs.mkdirSync(dir, { recursive: true });
+
+promises.forEach((p) => {
+  p.then((post) => {
+    console.log(post.slug, post.source);
+    if (!post.slug) {
+      return;
+    }
+    fs.writeFileSync(
+      path.join(dir, `${String(post.slug)}.md`),
+      String(post.md),
+    );
+  });
+});
diff --git a/src/lib/env.ts b/src/lib/env.ts
index dc363b3..3aecf00 100644
--- a/src/lib/env.ts
+++ b/src/lib/env.ts
@@ -1,5 +1,5 @@
 // WORKAROUND: `import.meta.env` is not available during Astro config evaluation.
 // https://docs.astro.build/en/guides/configuring-astro/#environment-variables
 export default function env(key: string): string | undefined {
-  return import.meta.env[key] || process.env[key];
+  return import.meta.env?.[key] || process.env[key];
 }
diff --git a/src/lib/getPosts.ts b/src/lib/getPosts.ts
index d62bb23..d022a92 100644
--- a/src/lib/getPosts.ts
+++ b/src/lib/getPosts.ts
@@ -14,7 +14,9 @@ const makePosts = memoize((): Promise<Post>[] =>
   fetchPosts().map((p) =>
     p.then((d) => {
       const result = post.safeParse(d);
-      if (result.success) return result.data;
+      if (result.success) {
+        return result.data;
+      }
       throw new Error(
         `Failed to parse post ${d.source}: ${result.error.message}`,
         result.error,

From f6c73f83e130d771cbff7b06d2c95fe38dcc3c74 Mon Sep 17 00:00:00 2001
From: lcflight <lcarthur747@gmail.com>
Date: Mon, 11 Mar 2024 13:06:08 -0400
Subject: [PATCH 2/8] changing getPosts to use collection

---
 astro.config.ts          |  10 ++--
 package.json             |   1 +
 pnpm-lock.yaml           |  19 +++++++
 scripts/pull.ts          |   8 ++-
 src/env.d.ts             |   1 +
 src/lib/getPosts.spec.ts | 105 +++++++++++++++++++++------------------
 src/lib/getPosts.ts      |  31 ++++++------
 vitest.setup.ts          |   3 ++
 8 files changed, 104 insertions(+), 74 deletions(-)

diff --git a/astro.config.ts b/astro.config.ts
index 4ab3a62..31cff16 100644
--- a/astro.config.ts
+++ b/astro.config.ts
@@ -1,6 +1,6 @@
 import { defineConfig } from "astro/config";
 import prefetch from "@astrojs/prefetch";
-import getRedirects from "./src/lib/getRedirects";
+// import getRedirects from "./src/lib/getRedirects";
 
 import sitemap from "@astrojs/sitemap";
 import "dotenv/config";
@@ -11,10 +11,10 @@ export default defineConfig({
   image: {
     domains: ["blog.beeminder.com", "user-images.githubusercontent.com"],
   },
-  redirects: await getRedirects().catch((e) => {
-    console.error(e);
-    throw e;
-  }),
+  // redirects: await getRedirects().catch((e) => {
+  //   console.error(e);
+  //   throw e;
+  // }),
   integrations: [
     prefetch({
       selector: "a",
diff --git a/package.json b/package.json
index 915c0a5..acdc612 100644
--- a/package.json
+++ b/package.json
@@ -43,6 +43,7 @@
     "memize": "^2.1.0",
     "node-fetch-cache": "^3.1.3",
     "p-limit": "^4.0.0",
+    "sanitize-filename": "^1.6.3",
     "sanitize-html": "^2.11.0",
     "sharp": "^0.32.5",
     "striptags": "^3.2.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e49c1dc..9f22375 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -56,6 +56,9 @@ dependencies:
   p-limit:
     specifier: ^4.0.0
     version: 4.0.0
+  sanitize-filename:
+    specifier: ^1.6.3
+    version: 1.6.3
   sanitize-html:
     specifier: ^2.11.0
     version: 2.11.0
@@ -4824,6 +4827,12 @@ packages:
   /safer-buffer@2.1.2:
     resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
 
+  /sanitize-filename@1.6.3:
+    resolution: {integrity: sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==}
+    dependencies:
+      truncate-utf8-bytes: 1.0.2
+    dev: false
+
   /sanitize-html@2.11.0:
     resolution: {integrity: sha512-BG68EDHRaGKqlsNjJ2xUB7gpInPA8gVx/mvjO743hZaeMCZ2DwzW7xvsqZ+KNU4QKwj86HJ3uu2liISf2qBBUA==}
     dependencies:
@@ -5287,6 +5296,12 @@ packages:
     resolution: {integrity: sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==}
     dev: false
 
+  /truncate-utf8-bytes@1.0.2:
+    resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==}
+    dependencies:
+      utf8-byte-length: 1.0.4
+    dev: false
+
   /ts-api-utils@1.0.3(typescript@5.2.2):
     resolution: {integrity: sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==}
     engines: {node: '>=16.13.0'}
@@ -5507,6 +5522,10 @@ packages:
     resolution: {integrity: sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g==}
     dev: true
 
+  /utf8-byte-length@1.0.4:
+    resolution: {integrity: sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA==}
+    dev: false
+
   /util-deprecate@1.0.2:
     resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
 
diff --git a/scripts/pull.ts b/scripts/pull.ts
index 8321f9e..492e8de 100644
--- a/scripts/pull.ts
+++ b/scripts/pull.ts
@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import "dotenv/config";
 import fetchPosts from "../src/lib/fetchPosts";
+import sanitize from "sanitize-filename";
 
 const promises = fetchPosts();
 const dir = "./src/content/posts";
@@ -17,12 +18,9 @@ fs.mkdirSync(dir, { recursive: true });
 promises.forEach((p) => {
   p.then((post) => {
     console.log(post.slug, post.source);
-    if (!post.slug) {
-      return;
-    }
     fs.writeFileSync(
-      path.join(dir, `${String(post.slug)}.md`),
-      String(post.md),
+      path.join(dir, `${sanitize(String(post.source))}.json`),
+      JSON.stringify(post, null, 2),
     );
   });
 });
diff --git a/src/env.d.ts b/src/env.d.ts
index f964fe0..acef35f 100644
--- a/src/env.d.ts
+++ b/src/env.d.ts
@@ -1 +1,2 @@
+/// <reference path="../.astro/types.d.ts" />
 /// <reference types="astro/client" />
diff --git a/src/lib/getPosts.spec.ts b/src/lib/getPosts.spec.ts
index cb043b2..bad6ca1 100644
--- a/src/lib/getPosts.spec.ts
+++ b/src/lib/getPosts.spec.ts
@@ -4,47 +4,36 @@ import fetchPost from "./fetchPost";
 import readSources from "./readSources";
 import ether from "./test/ether";
 import meta from "./test/meta";
+import { getCollection } from "astro:content";
 
 describe("getPosts", () => {
   beforeEach(() => {
-    vi.mocked(readSources).mockReturnValue([meta()]);
-  });
-
-  it("only loads sources once", async () => {
-    await getPosts();
-    await getPosts();
-
-    expect(readSources).toHaveBeenCalledTimes(1);
-  });
-
-  it("fetches post content", async () => {
-    await getPosts();
-
-    expect(fetchPost).toBeCalledWith(expect.stringContaining("the_source"));
-  });
-
-  it("sorts post by date descending", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      meta({ source: "new", date: "2013-02-22" }),
-      meta({ source: "old", date: "2013-02-21" }),
-    ]);
-
-    await getPosts();
-
-    expect(fetchPost).toBeCalledWith(expect.stringContaining("new"));
+    vi.mocked(getCollection).mockResolvedValue([
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+    ] as any);
   });
 
   it("includes excerpts", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ excerpt: undefined })]);
-
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: "word",
-        frontmatter: {
-          excerpt: "MAGIC_AUTO_EXTRACT",
+    vi.mocked(getCollection).mockResolvedValue([
+      {
+        data: {
+          ...meta({
+            excerpt: undefined,
+          }),
+          md: ether({
+            content: "word",
+            frontmatter: {
+              excerpt: "MAGIC_AUTO_EXTRACT",
+            },
+          }),
         },
-      }),
-    );
+      },
+    ] as any);
 
     const posts = await getPosts();
 
@@ -52,7 +41,16 @@ describe("getPosts", () => {
   });
 
   it("excludes unpublished posts by default", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ status: "draft" })]);
+    vi.mocked(getCollection).mockResolvedValue([
+      {
+        data: {
+          ...meta({
+            status: "draft",
+          }),
+          md: ether(),
+        },
+      },
+    ] as any);
 
     const posts = await getPosts();
 
@@ -60,20 +58,22 @@ describe("getPosts", () => {
   });
 
   it("includes unpublished posts when requested", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ status: "draft" })]);
+    vi.mocked(getCollection).mockResolvedValue([
+      {
+        data: {
+          ...meta({
+            status: "draft",
+          }),
+          md: ether(),
+        },
+      },
+    ] as any);
 
     const posts = await getPosts({ includeUnpublished: true });
 
     expect(posts).toHaveLength(1);
   });
 
-  it('caches posts without reference to "includeUnpublished"', async () => {
-    await getPosts();
-    await getPosts({ includeUnpublished: true });
-
-    expect(fetchPost).toHaveBeenCalledTimes(1);
-  });
-
   it("sets disqus id", async () => {
     const posts = await getPosts();
     const result = posts[0];
@@ -89,11 +89,16 @@ describe("getPosts", () => {
   });
 
   it("extracts image url", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: '<img src="https://example.com/image.png" />',
-      }),
-    );
+    vi.mocked(getCollection).mockResolvedValue([
+      {
+        data: {
+          ...meta(),
+          md: ether({
+            content: '<img src="https://example.com/image.png" />',
+          }),
+        },
+      },
+    ] as any);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -471,4 +476,8 @@ https://blog.beeminder.com/depunish
 
     await expect(getPosts()).rejects.toThrow(/Duplicate disqus/);
   });
+  it("gets Collection", async () => {
+    await getPosts();
+    expect(getCollection).toBeCalled();
+  });
 });
diff --git a/src/lib/getPosts.ts b/src/lib/getPosts.ts
index d022a92..c0e80fb 100644
--- a/src/lib/getPosts.ts
+++ b/src/lib/getPosts.ts
@@ -1,6 +1,6 @@
 import memoize from "./memoize";
 import { type Post, post } from "../schemas/post";
-import fetchPosts from "./fetchPosts";
+import { getCollection } from "astro:content";
 
 const getDuplicates = (
   arr: Array<Record<string, unknown>>,
@@ -10,20 +10,19 @@ const getDuplicates = (
   return keys.filter((key) => keys.indexOf(key) !== keys.lastIndexOf(key));
 };
 
-const makePosts = memoize((): Promise<Post>[] =>
-  fetchPosts().map((p) =>
-    p.then((d) => {
-      const result = post.safeParse(d);
-      if (result.success) {
-        return result.data;
-      }
-      throw new Error(
-        `Failed to parse post ${d.source}: ${result.error.message}`,
-        result.error,
-      );
-    }),
-  ),
-);
+const makePosts = memoize(async (): Promise<Post[]> => {
+  const posts = await getCollection("posts");
+  return posts.map((p) => {
+    const result = post.safeParse(p.data);
+    if (result.success) {
+      return result.data;
+    }
+    throw new Error(
+      `Failed to parse post ${p.data.source}: ${result.error.message}`,
+      result.error,
+    );
+  });
+});
 
 const getPosts = memoize(
   async ({
@@ -33,7 +32,7 @@ const getPosts = memoize(
     includeUnpublished?: boolean;
     sort?: boolean;
   } = {}): Promise<Post[]> => {
-    const posts = await Promise.all(makePosts());
+    const posts = await makePosts();
 
     const duplicateSlugs = getDuplicates(posts, "slug");
     if (duplicateSlugs.length > 0) {
diff --git a/vitest.setup.ts b/vitest.setup.ts
index 00dac51..2f2642a 100644
--- a/vitest.setup.ts
+++ b/vitest.setup.ts
@@ -33,6 +33,9 @@ vi.mock("astro", () => ({
     stop: vi.fn(),
   })),
 }));
+vi.mock("astro:content", () => ({
+  getCollection: vi.fn(async () => []),
+}));
 vi.mock("pixelmatch");
 vi.mock("pngjs");
 vi.mock("sharp");

From 555ccdcbda11b955fc2883c727b10eed915f9d47 Mon Sep 17 00:00:00 2001
From: lcflight <lcarthur747@gmail.com>
Date: Mon, 11 Mar 2024 13:07:38 -0400
Subject: [PATCH 3/8] collection config

---
 src/content/config.ts | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 src/content/config.ts

diff --git a/src/content/config.ts b/src/content/config.ts
new file mode 100644
index 0000000..6328844
--- /dev/null
+++ b/src/content/config.ts
@@ -0,0 +1,11 @@
+// 1. Import utilities from `astro:content`
+import { defineCollection } from "astro:content";
+// 2. Define your collection(s)
+const postsCollection = defineCollection({
+  type: "data",
+});
+// 3. Export a single `collections` object to register your collection(s)
+//    This key should match your collection directory name in "src/content"
+export const collections = {
+  posts: postsCollection,
+};

From dfc6a72e38d410ba506ff4ff5089461aa589d992 Mon Sep 17 00:00:00 2001
From: Nathan Arthur <nathan@nathanarthur.com>
Date: Mon, 11 Mar 2024 16:18:35 -0400
Subject: [PATCH 4/8] fix tests

---
 src/content/config.ts       |  12 +-
 src/lib/getArchives.spec.ts |  48 ++++-
 src/lib/getPosts.spec.ts    | 414 +++++++++++++++++++-----------------
 src/lib/getPosts.ts         |   8 +-
 src/lib/getTags.spec.ts     |  29 ++-
 src/lib/test/ether.ts       |  18 +-
 6 files changed, 301 insertions(+), 228 deletions(-)

diff --git a/src/content/config.ts b/src/content/config.ts
index 6328844..b131e5f 100644
--- a/src/content/config.ts
+++ b/src/content/config.ts
@@ -1,11 +1,7 @@
-// 1. Import utilities from `astro:content`
 import { defineCollection } from "astro:content";
-// 2. Define your collection(s)
-const postsCollection = defineCollection({
-  type: "data",
-});
-// 3. Export a single `collections` object to register your collection(s)
-//    This key should match your collection directory name in "src/content"
+
 export const collections = {
-  posts: postsCollection,
+  posts: defineCollection({
+    type: "data",
+  }),
 };
diff --git a/src/lib/getArchives.spec.ts b/src/lib/getArchives.spec.ts
index 287ec51..df650b9 100644
--- a/src/lib/getArchives.spec.ts
+++ b/src/lib/getArchives.spec.ts
@@ -2,10 +2,19 @@ import { describe, it, expect, beforeEach, vi } from "vitest";
 import getArchives from "./getArchives";
 import readSources from "./readSources";
 import meta from "./test/meta";
+import ether from "./test/ether";
+import { getCollection } from "astro:content";
 
 describe("getArchives", () => {
   beforeEach(() => {
-    vi.mocked(readSources).mockReturnValue([meta()]);
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+    ]);
   });
 
   it("batches by month", async () => {
@@ -32,7 +41,26 @@ describe("getArchives", () => {
   });
 
   it("handles three posts in one month", async () => {
-    vi.mocked(readSources).mockReturnValue([meta(), meta(), meta()]);
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+    ]);
 
     const result = await getArchives();
 
@@ -40,9 +68,19 @@ describe("getArchives", () => {
   });
 
   it("sorts posts by date", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      meta({ title: "A", date: "2013-02-22" }),
-      meta({ title: "B", date: "2013-02-21" }),
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta({ title: "A", date: "2013-02-22" }),
+          md: ether(),
+        },
+      },
+      {
+        data: {
+          ...meta({ title: "B", date: "2013-02-21" }),
+          md: ether(),
+        },
+      },
     ]);
 
     const result = await getArchives();
diff --git a/src/lib/getPosts.spec.ts b/src/lib/getPosts.spec.ts
index bad6ca1..3bd18c7 100644
--- a/src/lib/getPosts.spec.ts
+++ b/src/lib/getPosts.spec.ts
@@ -1,39 +1,45 @@
 import getPosts from "./getPosts";
 import { describe, it, expect, vi, beforeEach } from "vitest";
-import fetchPost from "./fetchPost";
-import readSources from "./readSources";
-import ether from "./test/ether";
+import ether, { type Ether } from "./test/ether";
 import meta from "./test/meta";
 import { getCollection } from "astro:content";
 
+const entry = ({
+  meta: m = {},
+  ether: e = {},
+}: {
+  meta?: Record<string, unknown>;
+  ether?: Ether;
+} = {}) => ({
+  data: {
+    ...meta(m),
+    md: ether(e),
+  },
+});
+
+function loadEntries(entries: Record<string, unknown>[]): void {
+  vi.mocked(getCollection).mockResolvedValue(entries as any);
+}
+
 describe("getPosts", () => {
   beforeEach(() => {
-    vi.mocked(getCollection).mockResolvedValue([
-      {
-        data: {
-          ...meta(),
-          md: ether(),
-        },
-      },
-    ] as any);
+    loadEntries([entry()]);
   });
 
   it("includes excerpts", async () => {
-    vi.mocked(getCollection).mockResolvedValue([
-      {
-        data: {
-          ...meta({
-            excerpt: undefined,
-          }),
-          md: ether({
-            content: "word",
-            frontmatter: {
-              excerpt: "MAGIC_AUTO_EXTRACT",
-            },
-          }),
+    loadEntries([
+      entry({
+        meta: {
+          excerpt: undefined,
+        },
+        ether: {
+          content: "word",
+          frontmatter: {
+            excerpt: "MAGIC_AUTO_EXTRACT",
+          },
         },
-      },
-    ] as any);
+      }),
+    ]);
 
     const posts = await getPosts();
 
@@ -41,16 +47,13 @@ describe("getPosts", () => {
   });
 
   it("excludes unpublished posts by default", async () => {
-    vi.mocked(getCollection).mockResolvedValue([
-      {
-        data: {
-          ...meta({
-            status: "draft",
-          }),
-          md: ether(),
+    loadEntries([
+      entry({
+        meta: {
+          status: "draft",
         },
-      },
-    ] as any);
+      }),
+    ]);
 
     const posts = await getPosts();
 
@@ -58,16 +61,13 @@ describe("getPosts", () => {
   });
 
   it("includes unpublished posts when requested", async () => {
-    vi.mocked(getCollection).mockResolvedValue([
-      {
-        data: {
-          ...meta({
-            status: "draft",
-          }),
-          md: ether(),
+    loadEntries([
+      entry({
+        meta: {
+          status: "draft",
         },
-      },
-    ] as any);
+      }),
+    ]);
 
     const posts = await getPosts({ includeUnpublished: true });
 
@@ -89,16 +89,13 @@ describe("getPosts", () => {
   });
 
   it("extracts image url", async () => {
-    vi.mocked(getCollection).mockResolvedValue([
-      {
-        data: {
-          ...meta(),
-          md: ether({
-            content: '<img src="https://example.com/image.png" />',
-          }),
+    loadEntries([
+      entry({
+        ether: {
+          content: '<img src="https://example.com/image.png" />',
         },
-      },
-    ] as any);
+      }),
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -107,15 +104,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter title", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ title: undefined })]);
-
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          title: "Hello",
-        },
+    loadEntries([
+      entry({
+        meta: { title: undefined },
+        ether: { frontmatter: { title: "Hello" } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -124,15 +118,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter author", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ author: undefined })]);
-
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          author: "Alice",
-        },
+    loadEntries([
+      entry({
+        meta: { author: undefined },
+        ether: { frontmatter: { author: "Alice" } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -141,14 +132,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter excerpt", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ excerpt: undefined })]);
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          excerpt: "Hello",
-        },
+    loadEntries([
+      entry({
+        meta: { excerpt: undefined },
+        ether: { frontmatter: { excerpt: "Hello" } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -157,14 +146,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter tags", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ tags: undefined })]);
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          tags: ["a", "b", "c"],
-        },
+    loadEntries([
+      entry({
+        meta: { tags: undefined },
+        ether: { frontmatter: { tags: ["a", "b", "c"] } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -173,14 +160,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter date", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ date: undefined })]);
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          date: new Date("2021-09-02"),
-        },
+    loadEntries([
+      entry({
+        meta: { date: undefined },
+        ether: { frontmatter: { date: new Date("2021-09-02") } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -189,14 +174,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter slug", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ slug: undefined })]);
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          slug: "hello",
-        },
+    loadEntries([
+      entry({
+        meta: { slug: undefined },
+        ether: { frontmatter: { slug: "hello" } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts.find((p) => p.slug === "hello");
@@ -212,14 +195,12 @@ describe("getPosts", () => {
   });
 
   it("uses frontmatter status", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ status: undefined })]);
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          status: "publish",
-        },
+    loadEntries([
+      entry({
+        meta: { status: undefined },
+        ether: { frontmatter: { status: "publish" } },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const result = posts[0];
@@ -228,11 +209,13 @@ describe("getPosts", () => {
   });
 
   it("returns html", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: "hello world",
+    loadEntries([
+      entry({
+        ether: {
+          content: "hello world",
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -241,11 +224,13 @@ describe("getPosts", () => {
   });
 
   it("uses smartypants", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: '"hello world"',
+    loadEntries([
+      entry({
+        ether: {
+          content: '"hello world"',
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -254,14 +239,16 @@ describe("getPosts", () => {
   });
 
   it("does not require new line after html element", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: `
+    loadEntries([
+      entry({
+        ether: {
+          content: `
 <h1>heading</h1>
 paragraph
 `,
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -270,11 +257,13 @@ paragraph
   });
 
   it("allows for PHP Markdown Extra-style IDs", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: "# heading {#id}",
+    loadEntries([
+      entry({
+        ether: {
+          content: "# heading {#id}",
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -283,11 +272,13 @@ paragraph
   });
 
   it("handles id properly", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: "## More Real-World Commitment Devices  {#AUG}",
+    loadEntries([
+      entry({
+        ether: {
+          content: "## More Real-World Commitment Devices  {#AUG}",
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -296,15 +287,17 @@ paragraph
   });
 
   it("handles multiple IDs", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: `
+    loadEntries([
+      entry({
+        ether: {
+          content: `
 ## What Commitment Devices Have You Used on Yourself? {#POL}
 
 ## More Real-World Commitment Devices  {#AUG}
 `,
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -314,16 +307,18 @@ paragraph
   });
 
   it("supports link nonsense", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: `
+    loadEntries([
+      entry({
+        ether: {
+          content: `
 [paying is not punishment](
 https://blog.beeminder.com/depunish
 "Our paying-is-not-punishment post is also a prequel to our announcement of No-Excuses Mode"
 ) because
       `,
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -332,11 +327,13 @@ https://blog.beeminder.com/depunish
   });
 
   it("links footnotes", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: "$FN[foo] $FN[foo]",
+    loadEntries([
+      entry({
+        ether: {
+          content: "$FN[foo] $FN[foo]",
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -347,15 +344,17 @@ https://blog.beeminder.com/depunish
   });
 
   it("links footnotes with trailing number", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: `$FN[foo]
-        some text $DC2: some more text,
-        
-        $FN[DC2]
-        some more text`,
+    loadEntries([
+      entry({
+        ether: {
+          content: `$FN[foo]
+          some text $DC2: some more text,
+          
+          $FN[DC2]
+          some more text`,
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -364,16 +363,18 @@ https://blog.beeminder.com/depunish
   });
 
   it("separately links subscring footnote ids", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: `$FN[DC]
-        We computer scientists call this the principle of delayed commitment $DC2: commitment is bad, all else equal.
-        Why do today what only might need to be done tomorrow?
-        
-        $FN[DC2]
-        This is a footnote about commitment.`,
+    loadEntries([
+      entry({
+        ether: {
+          content: `$FN[DC]
+          We computer scientists call this the principle of delayed commitment $DC2: commitment is bad, all else equal.
+          Why do today what only might need to be done tomorrow?
+          
+          $FN[DC2]
+          This is a footnote about commitment.`,
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -382,11 +383,13 @@ https://blog.beeminder.com/depunish
   });
 
   it("expands refs", async () => {
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        content: "$REF[foo] $REF[bar]",
+    loadEntries([
+      entry({
+        ether: {
+          content: "$REF[foo] $REF[bar]",
+        },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { content } = posts[0] || {};
@@ -395,14 +398,18 @@ https://blog.beeminder.com/depunish
   });
 
   it("parses frontmatter", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ slug: undefined })]);
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: {
-          slug: "val",
+    loadEntries([
+      entry({
+        meta: {
+          slug: undefined,
+        },
+        ether: {
+          frontmatter: {
+            slug: "val",
+          },
         },
       }),
-    );
+    ]);
 
     const posts = await getPosts();
     const { slug } = posts.find((p) => p.slug === "val") || {};
@@ -411,7 +418,13 @@ https://blog.beeminder.com/depunish
   });
 
   it("uses wordpress excerpt", async () => {
-    vi.mocked(readSources).mockReturnValue([meta({ excerpt: "wp excerpt" })]);
+    loadEntries([
+      entry({
+        meta: {
+          excerpt: "wp excerpt",
+        },
+      }),
+    ]);
 
     const posts = await getPosts();
 
@@ -421,8 +434,12 @@ https://blog.beeminder.com/depunish
   });
 
   it("strips html from wp excerpts", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      meta({ excerpt: "<strong>wp excerpt</strong>" }),
+    loadEntries([
+      entry({
+        meta: {
+          excerpt: "<strong>wp excerpt</strong>",
+        },
+      }),
     ]);
 
     const posts = await getPosts();
@@ -433,51 +450,54 @@ https://blog.beeminder.com/depunish
   });
 
   it("throws on duplicate slugs", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      { source: "doc.bmndr.co/a" },
-      { source: "doc.bmndr.co/b" },
-    ]);
-
-    vi.mocked(fetchPost).mockResolvedValue(
-      ether({
-        frontmatter: meta({
-          slug: "the_slug",
-          date: new Date(),
-        }),
+    loadEntries([
+      entry({
+        meta: {
+          slug: undefined,
+        },
+        ether: {
+          frontmatter: {
+            slug: "the_slug",
+          },
+        },
       }),
-    );
+      entry({
+        meta: {
+          slug: undefined,
+        },
+        ether: {
+          frontmatter: {
+            slug: "the_slug",
+          },
+        },
+      }),
+    ]);
 
     await expect(getPosts()).rejects.toThrow(/Duplicate slug/);
   });
 
   it("throws on duplicate disqus IDs", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      { source: "doc.bmndr.co/a" },
-      { source: "doc.bmndr.co/b" },
-    ]);
-
-    vi.mocked(fetchPost).mockResolvedValueOnce(
-      ether({
-        frontmatter: meta({
-          disqus_id: "the_disqus_id",
-          date: new Date(),
-        }),
+    loadEntries([
+      entry({
+        meta: {
+          disqus_id: undefined,
+        },
+        ether: { frontmatter: { disqus_id: "the_disqus_id" } },
       }),
-    );
-
-    vi.mocked(fetchPost).mockResolvedValueOnce(
-      ether({
-        frontmatter: meta({
-          disqus_id: "the_disqus_id",
-          date: new Date(),
-        }),
+      entry({
+        meta: {
+          disqus_id: undefined,
+        },
+        ether: { frontmatter: { disqus_id: "the_disqus_id" } },
       }),
-    );
+    ]);
 
     await expect(getPosts()).rejects.toThrow(/Duplicate disqus/);
   });
+
   it("gets Collection", async () => {
     await getPosts();
+
     expect(getCollection).toBeCalled();
   });
 });
diff --git a/src/lib/getPosts.ts b/src/lib/getPosts.ts
index c0e80fb..96c9f75 100644
--- a/src/lib/getPosts.ts
+++ b/src/lib/getPosts.ts
@@ -10,9 +10,15 @@ const getDuplicates = (
   return keys.filter((key) => keys.indexOf(key) !== keys.lastIndexOf(key));
 };
 
+type CollectionItem = {
+  data: {
+    source: string;
+  };
+};
+
 const makePosts = memoize(async (): Promise<Post[]> => {
   const posts = await getCollection("posts");
-  return posts.map((p) => {
+  return posts.map((p: CollectionItem) => {
     const result = post.safeParse(p.data);
     if (result.success) {
       return result.data;
diff --git a/src/lib/getTags.spec.ts b/src/lib/getTags.spec.ts
index 3c4dae3..342c9b3 100644
--- a/src/lib/getTags.spec.ts
+++ b/src/lib/getTags.spec.ts
@@ -1,14 +1,20 @@
 import { describe, it, expect, beforeEach, vi } from "vitest";
 import getTags from "./getTags";
-import readSources from "./readSources";
 import meta from "./test/meta";
+import { getCollection } from "astro:content";
+import ether from "./test/ether";
 
 describe("getTags", () => {
   beforeEach(() => {
-    vi.mocked(readSources).mockReturnValue([
-      meta({
-        tags: ["the_tag"],
-      }),
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta({
+            tags: ["the_tag"],
+          }),
+          md: ether(),
+        },
+      },
     ]);
   });
 
@@ -37,10 +43,15 @@ describe("getTags", () => {
   });
 
   it("does not include blank tags", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      meta({
-        tags: [""],
-      }),
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta({
+            tags: [""],
+          }),
+          md: ether(),
+        },
+      },
     ]);
 
     const result = await getTags();
diff --git a/src/lib/test/ether.ts b/src/lib/test/ether.ts
index be7796a..6b98a6a 100644
--- a/src/lib/test/ether.ts
+++ b/src/lib/test/ether.ts
@@ -1,19 +1,21 @@
 import matter from "gray-matter";
 
-export default function ether({
-  frontmatter = {},
-  before = "",
-  content = "",
-  after = "",
-  title,
-}: {
+export type Ether = {
   frontmatter?: Record<string, unknown>;
   before?: string;
   content?: string;
   after?: string;
   title?: string;
   redirects?: string[];
-} = {}): string {
+};
+
+export default function ether({
+  frontmatter = {},
+  before = "",
+  content = "",
+  after = "",
+  title,
+}: Ether = {}): string {
   return matter.stringify(
     `
 ${before}

From b7e005a8d0115cb420fc0ada0f5fc179a909660e Mon Sep 17 00:00:00 2001
From: Nathan Arthur <nathan@nathanarthur.com>
Date: Mon, 11 Mar 2024 16:20:37 -0400
Subject: [PATCH 5/8] create astro:content mock file

---
 __mocks__/astro:content.ts | 3 +++
 vitest.setup.ts            | 4 +---
 2 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 __mocks__/astro:content.ts

diff --git a/__mocks__/astro:content.ts b/__mocks__/astro:content.ts
new file mode 100644
index 0000000..b547e61
--- /dev/null
+++ b/__mocks__/astro:content.ts
@@ -0,0 +1,3 @@
+import { vi } from "vitest";
+
+export const getCollection = vi.fn();
diff --git a/vitest.setup.ts b/vitest.setup.ts
index 2f2642a..abfc81d 100644
--- a/vitest.setup.ts
+++ b/vitest.setup.ts
@@ -33,9 +33,7 @@ vi.mock("astro", () => ({
     stop: vi.fn(),
   })),
 }));
-vi.mock("astro:content", () => ({
-  getCollection: vi.fn(async () => []),
-}));
+vi.mock("astro:content");
 vi.mock("pixelmatch");
 vi.mock("pngjs");
 vi.mock("sharp");

From 4cbe3f0904a28853e760d3fc4aff3b408516f9ef Mon Sep 17 00:00:00 2001
From: Nathan Arthur <nathan@nathanarthur.com>
Date: Mon, 11 Mar 2024 16:28:08 -0400
Subject: [PATCH 6/8] type fixes

---
 src/lib/getArchives.spec.ts | 6 +++---
 src/lib/getPosts.ts         | 8 +-------
 src/lib/getTags.spec.ts     | 4 ++--
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/lib/getArchives.spec.ts b/src/lib/getArchives.spec.ts
index df650b9..062c220 100644
--- a/src/lib/getArchives.spec.ts
+++ b/src/lib/getArchives.spec.ts
@@ -14,7 +14,7 @@ describe("getArchives", () => {
           md: ether(),
         },
       },
-    ]);
+    ] as any);
   });
 
   it("batches by month", async () => {
@@ -60,7 +60,7 @@ describe("getArchives", () => {
           md: ether(),
         },
       },
-    ]);
+    ] as any);
 
     const result = await getArchives();
 
@@ -81,7 +81,7 @@ describe("getArchives", () => {
           md: ether(),
         },
       },
-    ]);
+    ] as any);
 
     const result = await getArchives();
 
diff --git a/src/lib/getPosts.ts b/src/lib/getPosts.ts
index 96c9f75..c0e80fb 100644
--- a/src/lib/getPosts.ts
+++ b/src/lib/getPosts.ts
@@ -10,15 +10,9 @@ const getDuplicates = (
   return keys.filter((key) => keys.indexOf(key) !== keys.lastIndexOf(key));
 };
 
-type CollectionItem = {
-  data: {
-    source: string;
-  };
-};
-
 const makePosts = memoize(async (): Promise<Post[]> => {
   const posts = await getCollection("posts");
-  return posts.map((p: CollectionItem) => {
+  return posts.map((p) => {
     const result = post.safeParse(p.data);
     if (result.success) {
       return result.data;
diff --git a/src/lib/getTags.spec.ts b/src/lib/getTags.spec.ts
index 342c9b3..01af721 100644
--- a/src/lib/getTags.spec.ts
+++ b/src/lib/getTags.spec.ts
@@ -15,7 +15,7 @@ describe("getTags", () => {
           md: ether(),
         },
       },
-    ]);
+    ] as any);
   });
 
   it("returns tags", async () => {
@@ -52,7 +52,7 @@ describe("getTags", () => {
           md: ether(),
         },
       },
-    ]);
+    ] as any);
 
     const result = await getTags();
 

From 98c5f0a27161e68b02402348de486eb3d6f155ff Mon Sep 17 00:00:00 2001
From: Nathan Arthur <nathan@nathanarthur.com>
Date: Mon, 11 Mar 2024 16:39:02 -0400
Subject: [PATCH 7/8] enable contentCollectionCache

---
 astro.config.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/astro.config.ts b/astro.config.ts
index 31cff16..3508175 100644
--- a/astro.config.ts
+++ b/astro.config.ts
@@ -21,4 +21,7 @@ export default defineConfig({
     }),
     sitemap(),
   ],
+  experimental: {
+    contentCollectionCache: true,
+  },
 });

From 6e7622d5365bfe1ccc9de5a80a74f03ee282950a Mon Sep 17 00:00:00 2001
From: lcflight <lcarthur747@gmail.com>
Date: Tue, 12 Mar 2024 10:29:15 -0400
Subject: [PATCH 8/8] md:pull added to build for testing

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index acdc612..ac2c45f 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
   "scripts": {
     "dev": "astro dev",
     "start": "astro dev",
-    "build": "tsx ./scripts/image-cache.ts && astro build",
+    "build": " pnpm md:pull && tsx ./scripts/image-cache.ts && astro build",
     "preview": "astro preview",
     "astro": "astro",
     "test": "vitest",