From 9361418baa0a8fc4e6ab98db9812ebd9d92028fc Mon Sep 17 00:00:00 2001 From: Essential Randomness Date: Tue, 7 Oct 2025 00:51:35 -0700 Subject: [PATCH 1/4] Prepare for tag search tests --- .../01.html | 449 ++++++++++++++++++ .../02.html | 449 ++++++++++++++++++ .../03.html | 401 ++++++++++++++++ .../04.html | 399 ++++++++++++++++ .../01.html | 399 ++++++++++++++++ tests/mocks/handlers.ts | 2 + tests/mocks/handlers/tags/search.ts | 22 + tests/mocks/scripts/utils.mts | 48 ++ 8 files changed, 2169 insertions(+) create mode 100644 tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/01.html create mode 100644 tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/02.html create mode 100644 tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/03.html create mode 100644 tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/04.html create mode 100644 tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical/01.html create mode 100644 tests/mocks/handlers/tags/search.ts diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/01.html b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/01.html new file mode 100644 index 0000000..e206ca9 --- /dev/null +++ b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/01.html @@ -0,0 +1,449 @@ + + + + + + + + + + + + + + + Search Tags | Archive of Our Own + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/02.html b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/02.html new file mode 100644 index 0000000..97e51f4 --- /dev/null +++ b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/02.html @@ -0,0 +1,449 @@ + + + + + + + + + + + + + + + Search Tags | Archive of Our Own + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/03.html b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/03.html new file mode 100644 index 0000000..a4e5438 --- /dev/null +++ b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/03.html @@ -0,0 +1,401 @@ + + + + + + + + + + + + + + + Search Tags | Archive of Our Own + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/04.html b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/04.html new file mode 100644 index 0000000..9940ab3 --- /dev/null +++ b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/04.html @@ -0,0 +1,399 @@ + + + + + + + + + + + + + + + Search Tags | Archive of Our Own + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical/01.html b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical/01.html new file mode 100644 index 0000000..ddca95b --- /dev/null +++ b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical/01.html @@ -0,0 +1,399 @@ + + + + + + + + + + + + + + + Tags Matching 'asdfasdfasdfasdfasdfasdf' | Archive of Our Own + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/mocks/handlers.ts b/tests/mocks/handlers.ts index 888382d..1798ff9 100644 --- a/tests/mocks/handlers.ts +++ b/tests/mocks/handlers.ts @@ -1,6 +1,7 @@ import allHandlers from "./handlers/all"; import feedHandlers from "./handlers/tags/feed"; import nameHandlers from "./handlers/tags/name"; +import searchHandlers from "./handlers/tags/search"; import profileHandlers from "./handlers/users/profile"; import seriesHandlers from "./handlers/series"; import tagWorksHandlers from "./handlers/tags/works"; @@ -15,6 +16,7 @@ export default [ profileHandlers, feedHandlers, tagWorksHandlers, + searchHandlers, worksHandlers, nameHandlers, workPageHandlers, diff --git a/tests/mocks/handlers/tags/search.ts b/tests/mocks/handlers/tags/search.ts new file mode 100644 index 0000000..e46887a --- /dev/null +++ b/tests/mocks/handlers/tags/search.ts @@ -0,0 +1,22 @@ +import fs from "fs"; +import path from "path"; +import { http, HttpHandler, HttpResponse } from "msw"; +import { + getArchiveDataDir, + getFilePathForSearchUrl, +} from "../../scripts/utils.mjs"; + +export default http.all( + "https://archiveofourown.org/tags/search", + ({ request }) => { + const url = new URL(request.url); + const relativePath = getFilePathForSearchUrl(url); + const html = fs.readFileSync( + path.resolve(getArchiveDataDir(), relativePath) + ); + + return new HttpResponse(html, { + headers: { "Content-Type": "text/html" }, + }); + } +) satisfies HttpHandler; diff --git a/tests/mocks/scripts/utils.mts b/tests/mocks/scripts/utils.mts index d64d75c..795dd0b 100644 --- a/tests/mocks/scripts/utils.mts +++ b/tests/mocks/scripts/utils.mts @@ -87,6 +87,47 @@ export function getArchiveDataDir(archive: "ao3" | "superlove" = "ao3") { return path.join(getRootDataDir(), archive); } +// These parameters are excluded from the tag search folder name. +// Page is used for the page number instead (i.e. 01.html, 02.html, etc.), +// while view_adult is excluded because we simply add it to the all pages +// to make sure we bypass the adult banner. +const TAG_SEARCH_EXCLUDED_PARAMS = new Set(["page", "view_adult"]); + +const getNormalizedTagSearchFolder = (searchParams: URLSearchParams) => { + const entries: string[] = []; + // Sort the parameters to make the folder name deterministic + const keys = Array.from(new Set([...searchParams.keys()])).sort(); + + for (const key of keys) { + if (TAG_SEARCH_EXCLUDED_PARAMS.has(key)) { + continue; + } + const values = searchParams.getAll(key); + // Also sort the values of the same keys + const sortedValues = values.length > 1 ? [...values].sort() : values; + for (const value of sortedValues) { + const segment = `${key}=${value}`; + entries.push(filenamify(segment, { replacement: "_", maxLength: 100 })); + } + } + + if (entries.length === 0) { + return "default"; + } + + return entries.join("__"); +}; + +export const getFilePathForSearchUrl = (parsedUrl: URL) => { + const folderName = getNormalizedTagSearchFolder(parsedUrl.searchParams); + const rawPage = parsedUrl.searchParams.get("page"); + const page = Number.parseInt(rawPage ?? "1", 10); + const fileName = `${String(page).padStart(2, "0")}.html`; + + // TODO: make this support other search types with time + return path.join("tag-search", folderName, fileName); +}; + export function getFilePathFromUrl(url: string | URL) { const archive = getArchiveFromUrl(url); const parsedUrl = new URL(url); @@ -97,6 +138,13 @@ export function getFilePathFromUrl(url: string | URL) { const lastSegment = segments[segments.length - 1]; const hasExtension = lastSegment?.includes("."); + if (segments[0] === "tags" && lastSegment === "search") { + return path.join( + getArchiveDataDir(archive), + getFilePathForSearchUrl(parsedUrl) + ); + } + // If the last segment is a file, use segments up to the last one for the directory // Otherwise use all segments, unless the last path is "works". const dirSegments = From aecee3fad1e8e9eb04ea7a31a3997f2eea600e4e Mon Sep 17 00:00:00 2001 From: Essential Randomness Date: Wed, 8 Oct 2025 01:31:34 -0700 Subject: [PATCH 2/4] tag- --- src/page-loaders.ts | 16 +- src/tags/index.ts | 42 +- src/tags/search-getters.ts | 67 +++ src/urls.ts | 45 +- .../02.html | 449 ++++++++++++++++++ tests/mocks/scripts/utils.mts | 11 +- tests/tag-search.test.ts | 265 +++++++++++ types/entities.ts | 40 ++ 8 files changed, 924 insertions(+), 11 deletions(-) create mode 100644 src/tags/search-getters.ts create mode 100644 tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html create mode 100644 tests/tag-search.test.ts diff --git a/src/page-loaders.ts b/src/page-loaders.ts index 27f47a7..9daa5ee 100644 --- a/src/page-loaders.ts +++ b/src/page-loaders.ts @@ -1,6 +1,7 @@ import { getSeriesUrl, getTagUrl, + getSearchUrlFromTagFilters, getTagWorksFeedAtomUrl, getTagWorksFeedUrl, getUserProfileUrl, @@ -11,7 +12,7 @@ import { import { CheerioAPI } from "cheerio"; import { load } from "cheerio/slim"; import { getFetcher } from "./fetcher"; -import { ArchiveId } from "types/entities"; +import { ArchiveId, TagSearchFilters } from "types/entities"; // This is a wrapper around the fetch function that loads the page into a CheerioAPI // instance and returns the type of the page. @@ -61,6 +62,19 @@ export const loadTagPage = async ({ tagName }: { tagName: string }) => { }); }; +export interface TagSearchPage extends CheerioAPI { + kind: "TagSearchPage"; +} +export const loadTagSearchPage = async ({ + tagSearchFilters, +}: { + tagSearchFilters: TagSearchFilters; +}) => { + return await fetchPage({ + url: getSearchUrlFromTagFilters(tagSearchFilters), + }); +}; + // Atom feed of the most recent works featuring a tag. // Sample: https://archiveofourown.org/tags/91247110/feed.atom export interface TagWorksAtomFeed extends CheerioAPI { diff --git a/src/tags/index.ts b/src/tags/index.ts index 0828aa8..8095bbb 100644 --- a/src/tags/index.ts +++ b/src/tags/index.ts @@ -12,10 +12,49 @@ import { getTagId, getTagNameFromFeed } from "./works-feed-getters"; import { loadTagFeedAtomPage, loadTagPage, + loadTagSearchPage, loadTagWorksFeed, } from "src/page-loaders"; -import type { Tag } from "types/entities"; +import type { + Tag, + TagSearchFilters, + TagSearchResultSummary, +} from "types/entities"; +import { + getPagesCount, + getTagsSearchResults, + getTotalResults, +} from "./search-getters"; + +export const searchTags = async ( + tagSearchFilters: Partial +): Promise => { + // We normalize the filters to ensure they have the required properties. + const normalizedFilters: TagSearchFilters = { + tagName: tagSearchFilters.tagName ?? null, + fandoms: tagSearchFilters.fandoms ?? [], + type: tagSearchFilters.type ?? "any", + wranglingStatus: tagSearchFilters.wranglingStatus ?? "any", + sortColumn: tagSearchFilters.sortColumn ?? "name", + sortDirection: tagSearchFilters.sortDirection ?? "asc", + page: tagSearchFilters.page ?? 1, + }; + + const page = await loadTagSearchPage({ tagSearchFilters: normalizedFilters }); + + return { + // We return the filters as is because they are already normalized + // and the API expects them to be in this format. + filters: normalizedFilters, + totalResults: getTotalResults(page), + pages: { + total: getPagesCount(page), + current: normalizedFilters.page, + }, + tags: getTagsSearchResults(page), + }; +}; export const getTag = async ({ tagName, @@ -43,6 +82,7 @@ export const getTag = async ({ }; }; +// TODO: this is really getCanonicalTagNameById export const getTagNameById = async ({ tagId }: { tagId: string }) => { return getTagNameFromFeed(await loadTagFeedAtomPage({ tagId })); }; diff --git a/src/tags/search-getters.ts b/src/tags/search-getters.ts new file mode 100644 index 0000000..4f31281 --- /dev/null +++ b/src/tags/search-getters.ts @@ -0,0 +1,67 @@ +import { TagSearchPage } from "src/page-loaders"; +import { TagSearchResultSummary } from "types/entities"; + +const parseIntOrThrow = (text: string) => { + const match = text.trim().match(/^(\d+)/); + if (!match) { + throw new Error(`Invalid integer: ${text}`); + } + return parseInt(match[1].trim(), 10); +}; + +export const getTotalResults = (page: TagSearchPage) => { + const totalResultsMatch = page("h3.heading") + .first() + .text() + .match(/(\d+)\s+Found/); + return totalResultsMatch ? parseIntOrThrow(totalResultsMatch[1]) : 0; +}; + +export const getPagesCount = (page: TagSearchPage) => { + const lastPageMatch = page(".pagination.actions li:not(.next, .previous)") + .last() + .text(); + return lastPageMatch ? parseIntOrThrow(lastPageMatch) : 0; +}; + +export const getTagsSearchResults = (page: TagSearchPage) => { + return page("ol.tag.index.group > li") + .map((_, li) => { + const $li = page(li); + const link = $li.find("a.tag").first(); + if (!link.length) { + return null; + } + + const name = link.text().trim(); + + // Tags are in the format: "Type: Name (Works Count)" + // Here we extract the works count. + const worksMatch = $li.text().match(/\((\d+)\)\s*$/); + const worksCount = parseIntOrThrow(worksMatch![1]); + + // Tags are in the format: "Type: Name (Works Count)" + // Here we extract the type. + const typeMatch = $li.text().match(/^([^:]+):/); + if (!typeMatch) { + throw new Error(`Invalid tag type: ${$li.text()}`); + } + const type = typeMatch[1].trim().toLowerCase(); + + const classes = new Set( + ($li.find("span").attr("class") ?? "").split(/\s+/).filter(Boolean) + ); + + return { + name, + type: + type == "unsortedtag" + ? "unsorted" + : (type as TagSearchResultSummary["tags"][number]["type"]), + canonical: classes.has("canonical"), + worksCount, + } as const; + }) + .get() + .filter((tag) => tag !== null); +}; diff --git a/src/urls.ts b/src/urls.ts index a3ce5cf..762afbb 100644 --- a/src/urls.ts +++ b/src/urls.ts @@ -3,7 +3,7 @@ import { isValidArchiveIdOrNullish, parseArchiveId, } from "./utils"; -import { WorkSummary } from "types/entities"; +import { TagSearchFilters, WorkSummary } from "types/entities"; declare global { var archiveBaseUrl: string; @@ -195,3 +195,46 @@ export const getWorkDetailsFromUrl = ({ collectionName: url.match(/collections\/(\w+)/)?.[1], }; }; + +const getSearchParamsFromTagFilters = ( + searchFilters: Partial +) => { + // Prepare the parameters for the search as a map first. This makes them a bit + // more readable, since these parameters will all need to be wrapped with with + // "tag_search[]" in the URL. + const parameters = { + name: searchFilters.tagName ?? "", + fandoms: searchFilters.fandoms?.join(",") ?? "", + type: searchFilters.type?.toLowerCase() ?? "", + wrangling_status: + searchFilters.wranglingStatus + // We remove the _or_ and _and_ that we added for readability + // so that the values match the expected values for the API. + ?.replaceAll("_or_", "_") + .replaceAll("_and_", "_") ?? "any", + sort_column: + searchFilters.sortColumn === "works_count" + ? "uses" + : searchFilters.sortColumn ?? "name", + sort_direction: searchFilters.sortDirection ?? "asc", + }; + + const searchParams = new URLSearchParams(); + if (searchFilters.page) { + searchParams.set("page", String(searchFilters.page)); + } + searchParams.set("commit", "Search Tags"); + + // Now add the parameters to the search params, wrapped with "tag_search[]" + for (const [key, value] of Object.entries(parameters)) { + searchParams.set(`tag_search[${key}]`, value); + } + + return searchParams; +}; + +export const getSearchUrlFromTagFilters = (searchFilters: TagSearchFilters) => { + const url = new URL(`tags/search`, getArchiveBaseUrl()); + url.search = getSearchParamsFromTagFilters(searchFilters).toString(); + return url.href; +}; diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html new file mode 100644 index 0000000..96f963d --- /dev/null +++ b/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html @@ -0,0 +1,449 @@ + + + + + + + + + + + + + + + Tags Matching 'an unusual' | Archive of Our Own + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/mocks/scripts/utils.mts b/tests/mocks/scripts/utils.mts index 795dd0b..3e77b96 100644 --- a/tests/mocks/scripts/utils.mts +++ b/tests/mocks/scripts/utils.mts @@ -87,26 +87,21 @@ export function getArchiveDataDir(archive: "ao3" | "superlove" = "ao3") { return path.join(getRootDataDir(), archive); } -// These parameters are excluded from the tag search folder name. -// Page is used for the page number instead (i.e. 01.html, 02.html, etc.), -// while view_adult is excluded because we simply add it to the all pages -// to make sure we bypass the adult banner. -const TAG_SEARCH_EXCLUDED_PARAMS = new Set(["page", "view_adult"]); - const getNormalizedTagSearchFolder = (searchParams: URLSearchParams) => { const entries: string[] = []; // Sort the parameters to make the folder name deterministic const keys = Array.from(new Set([...searchParams.keys()])).sort(); for (const key of keys) { - if (TAG_SEARCH_EXCLUDED_PARAMS.has(key)) { + // Page is used for the page number instead (i.e. 01.html, 02.html, etc.) + if (key === "page") { continue; } const values = searchParams.getAll(key); // Also sort the values of the same keys const sortedValues = values.length > 1 ? [...values].sort() : values; for (const value of sortedValues) { - const segment = `${key}=${value}`; + const segment = `${key}=${value == "any" ? "" : value}`; entries.push(filenamify(segment, { replacement: "_", maxLength: 100 })); } } diff --git a/tests/tag-search.test.ts b/tests/tag-search.test.ts new file mode 100644 index 0000000..c7d93b7 --- /dev/null +++ b/tests/tag-search.test.ts @@ -0,0 +1,265 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { getTagNameById, searchTags } from "src/index"; + +describe("Tags/search", () => { + it("should fetch canonical characters for first page with no param", async () => { + const result = await searchTags({ + type: "character", + wranglingStatus: "canonical", + fandoms: ["Hazbin Hotel (Cartoon)"], + }); + + expect(result.filters).toEqual({ + sortColumn: "name", + sortDirection: "asc", + tagName: null, + type: "character", + wranglingStatus: "canonical", + page: 1, + fandoms: ["Hazbin Hotel (Cartoon)"], + }); + + expect(result.pages).toEqual({ total: 3, current: 1 }); + + // We just look at the names to make the snapshot more readable, and + // because this fandom gets a lot of new works + expect(result.tags.map((tag) => tag.name)).toMatchInlineSnapshot(` + [ + "Abel (Hazbin Hotel)", + "Adam (Hazbin Hotel)", + "Alastor (Hazbin Hotel)", + "Alastor's Father (Hazbin Hotel)", + "Alastor's Grandmother (Hazbin Hotel)", + "Alastor's Microphone (Hazbin Hotel)", + "Alastor's Mother (Hazbin Hotel)", + "Alastor's Shadow (Hazbin Hotel)", + "Alastor's Sibling (Hazbin Hotel)", + "Angel Dust (Hazbin Hotel)", + "Angel Dust's Father (Hazbin Hotel)", + "Angel Dust's Mother (Hazbin Hotel)", + "Angel Dust's Shadow (Hazbin Hotel)", + "Angels (Hazbin Hotel)", + "Arackniss (Hazbin Hotel)", + "Baxter (Hazbin Hotel)", + "Belphegor (Helluva Boss)", + "Bethesda Von Eldritch", + "Bryrin (Hazbin Hotel)", + "Cannibal Child Vaggie Spares in Flashback (Hazbin Hotel)", + "Cannibal Town Ensemble (Hazbin Hotel)", + "Carmilla Carmine (Hazbin Hotel)", + "Carmilla Carmine's Daughters (Hazbin Hotel)", + "Charlie Magne | Morningstar", + "Cherri Bomb (Hazbin Hotel)", + "Clara (Hazbin Hotel)", + "Crymini (Hazbin Hotel)", + "Dazzle (Hazbin Hotel)", + "Dia (Hazbin Hotel)", + "Egg Boi #11 (Hazbin Hotel)", + "Egg Boi #23 (Hazbin Hotel)", + "Emily (Hazbin Hotel)", + "Essie (Hazbin Hotel)", + "Eve (Hazbin Hotel)", + "Ewe Demon (Hazbin Hotel)", + "Exorcist Angels (Hazbin Hotel)", + "Fat Nuggets (Hazbin Hotel)", + "Frank (Hazbin Hotel)", + "Franklin (Hazbin Hotel)", + "Frederick Von Eldritch", + "God (Hazbin Hotel)", + "Happy Hotel | Hazbin Hotel Residents", + "Harold Von Eldrich", + "Hazbin Hotel Ensemble", + "Helsa Von Eldritch", + "Henroin (Hazbin Hotel)", + "Husk (Hazbin Hotel)", + "Husk's Father (Hazbin Hotel)", + "Husk's Mother (Hazbin Hotel)", + "Izzi (Hazbin Hotel)", + ] + `); + }); + + it("Should fetch canonical characters for a middle page", async () => { + const result = await searchTags({ + type: "character", + wranglingStatus: "canonical", + fandoms: ["Hazbin Hotel (Cartoon)"], + page: 2, + }); + + expect(result.filters).toEqual({ + sortColumn: "name", + sortDirection: "asc", + tagName: null, + type: "character", + wranglingStatus: "canonical", + page: 2, + fandoms: ["Hazbin Hotel (Cartoon)"], + }); + + expect(result.pages).toEqual({ total: 3, current: 2 }); + + // We just look at the names to make the snapshot more readable, and + // because this fandom gets a lot of new works + expect(result.tags.map((tag) => tag.name)).toMatchInlineSnapshot(` + [ + "Katie Killjoy", + "KeeKee (Hazbin Hotel)", + "Kitty (Hazbin Hotel)", + "Leviathan (Helluva Boss)", + "Lilith Magne | Morningstar", + "Loan Shark Demon (Hazbin Hotel)", + "Lucifer Magne | Morningstar", + "Lucifer Magne | Morningstar's Siblings", + "Lute (Hazbin Hotel)", + "Melissa (Hazbin Hotel)", + "Mimzy (Hazbin Hotel)", + "Missi Zilla", + "Molly (Hazbin Hotel)", + "Monty Python (Hazbin Hotel)", + "Niffty (Hazbin Hotel)", + "Odette (Hazbin Hotel)", + "Original Children of Alastor and Angel Dust (Hazbin Hotel)", + "Original Children of Alastor and Lucifer Magne | Morningstar", + "Original Hazbin Hotel Character(s)", + "Original Magne | Morningstar Character(s)", + "Overlords (Hazbin Hotel)", + "Razzle (Hazbin Hotel)", + "Roo (Hazbin Hotel)", + "Rosie (Hazbin Hotel)", + "Rosie's First Husband (Hazbin Hotel)", + "Satan (Helluva Boss)", + "Sera (Hazbin Hotel)", + "Seviathan Von Eldritch", + "Sir Pentious (Hazbin Hotel)", + "Sir Pentious's Hat (Hazbin Hotel)", + "St. Peter (Hazbin Hotel)", + "Summer (Hazbin Hotel)", + "Susan (Hazbin Hotel)", + "The Egg Bois (Hazbin Hotel)", + "The Seven Deadly Sins (Hazbin Hotel & Helluva Boss)", + "The Three V's (Hazbin Hotel)", + "Tiffany Titfucker (Hazbin Hotel)", + "Tom Trench", + "Travis (Hazbin Hotel)", + "Vaggie (Hazbin Hotel)", + "Vaggie's Father (Hazbin Hotel)", + "Vaggie's Mother (Hazbin Hotel)", + "Valentino (Hazbin Hotel)", + "Vark | Vox's Shark (Hazbin Hotel)", + "Vax (Hazbin Hotel)", + "Velvette (Hazbin Hotel)", + "Victor (Daisies - Black Gryph0n & Baasik)", + "Vinny Shortcake (Hazbin Hotel)", + "Vox (Hazbin Hotel)", + "Vox's Assistant (Hazbin Hotel)", + ] + `); + }); + + it("should fetch canonical characters for last page", async () => { + const result = await searchTags({ + type: "character", + wranglingStatus: "canonical", + fandoms: ["Hazbin Hotel (Cartoon)"], + page: 3, + }); + + expect(result.filters).toEqual({ + sortColumn: "name", + sortDirection: "asc", + tagName: null, + type: "character", + wranglingStatus: "canonical", + page: 3, + fandoms: ["Hazbin Hotel (Cartoon)"], + }); + + expect(result.pages).toEqual({ total: 3, current: 3 }); + + // We just look at the names to make the snapshot more readable, and + // because this fandom gets a lot of new works + expect(result.tags.map((tag) => tag.name)).toMatchInlineSnapshot(` + [ + "Zeezi (Hazbin Hotel)", + "Zestial (Hazbin Hotel)", + ] + `); + }); + + it("should return empty results when the search has no matches", async () => { + const result = await searchTags({ + tagName: "asdfasdfasdfasdfasdfasdf", + sortColumn: "name", + sortDirection: "asc", + type: "fandom", + wranglingStatus: "canonical", + }); + + expect(result.filters).toEqual({ + sortColumn: "name", + sortDirection: "asc", + tagName: "asdfasdfasdfasdfasdfasdf", + type: "fandom", + wranglingStatus: "canonical", + page: 1, + fandoms: [], + }); + + expect(result.totalResults).toBe(0); + expect(result.pages).toEqual({ total: 0, current: 1 }); + expect(result.tags).toEqual([]); + }); + + it("should return different results for different tag types", async () => { + const result = await searchTags({ + // Chosen because somehow it has a bunch of diverse results. + tagName: "an unusual", + page: 2, + }); + expect(result.filters).toEqual({ + tagName: "an unusual", + fandoms: [], + type: "any", + wranglingStatus: "any", + sortColumn: "name", + sortDirection: "asc", + page: 2, + }); + + expect(result.totalResults).toBe(199); + + // Some tags we should find to make sure the parser is working correctly. + const UNSORTED_TAG = "an unusual ship that actually works very well! :)"; + const FREEFORM_TAG = "Eames projects his lust in an unusual way"; + const CHARACTER_TAG = "An Unusual Stained Glass Window"; + const CANONICAL_TAG = + "Episode: s02e23 Facing an Unusual Past (Saiki Kusuo no Sai-nan)"; + + expect(result.tags.find((tag) => tag.name === UNSORTED_TAG)).toEqual({ + name: UNSORTED_TAG, + type: "unsorted", + canonical: false, + worksCount: 1, + }); + expect(result.tags.find((tag) => tag.name === FREEFORM_TAG)).toEqual({ + name: FREEFORM_TAG, + type: "freeform", + canonical: false, + worksCount: 1, + }); + expect(result.tags.find((tag) => tag.name === CHARACTER_TAG)).toEqual({ + name: CHARACTER_TAG, + type: "character", + canonical: false, + worksCount: 0, + }); + expect(result.tags.find((tag) => tag.name === CANONICAL_TAG)).toEqual({ + name: CANONICAL_TAG, + type: "freeform", + canonical: true, + worksCount: 0, + }); + }); +}); diff --git a/types/entities.ts b/types/entities.ts index d69a2dc..6789a0f 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -30,6 +30,46 @@ export interface Tag { }>; } +export type TagSearchType = + | "fandom" + | "character" + | "relationship" + | "freeform" + | "any"; + +export type TagWranglingStatus = + | "canonical" + | "noncanonical" + | "synonymous" + | "canonical_or_synonymous" + | "noncanonical_and_nonsynonymous" + | "any"; + +export interface TagSearchFilters { + tagName: string | null; + fandoms: string[]; + type: TagSearchType; + wranglingStatus: TagWranglingStatus; + sortColumn: "name" | "created_at" | "works_count"; + sortDirection: "asc" | "desc"; + page: number; +} + +export interface TagSearchResultSummary { + filters: TagSearchFilters; + totalResults: number; + pages: { + total: number; + current: number; + }; + tags: Array<{ + name: string; + type: "fandom" | "character" | "relationship" | "freeform" | "unsorted"; + canonical: boolean; + worksCount: number; + }>; +} + export interface User { id: ArchiveId; username: string; From 315dfda02bf7be875e7ae1e7997670ec441284c1 Mon Sep 17 00:00:00 2001 From: Essential Randomness Date: Fri, 10 Oct 2025 00:39:34 -0700 Subject: [PATCH 3/4] fix up cases for serach folders --- .../02.html | 0 .../01.html | 0 .../01.html | 0 .../02.html | 0 .../03.html | 0 .../04.html | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename tests/mocks/data/ao3/tag-search/{commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]= => commit=search tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=}/02.html (100%) rename tests/mocks/data/ao3/tag-search/{commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical => commit=search tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=fandom__tag_search[wrangling_status]=canonical}/01.html (100%) rename tests/mocks/data/ao3/tag-search/{commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical => commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical}/01.html (100%) rename tests/mocks/data/ao3/tag-search/{commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical => commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical}/02.html (100%) rename tests/mocks/data/ao3/tag-search/{commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical => commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical}/03.html (100%) rename tests/mocks/data/ao3/tag-search/{commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical => commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical}/04.html (100%) diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html b/tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html similarity index 100% rename from tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html rename to tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=__tag_search[name]=an unusual__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=__tag_search[wrangling_status]=/02.html diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical/01.html b/tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=fandom__tag_search[wrangling_status]=canonical/01.html similarity index 100% rename from tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Fandom__tag_search[wrangling_status]=canonical/01.html rename to tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=__tag_search[name]=asdfasdfasdfasdfasdfasdf__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=fandom__tag_search[wrangling_status]=canonical/01.html diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/01.html b/tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/01.html similarity index 100% rename from tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/01.html rename to tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/01.html diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/02.html b/tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/02.html similarity index 100% rename from tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/02.html rename to tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/02.html diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/03.html b/tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/03.html similarity index 100% rename from tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/03.html rename to tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/03.html diff --git a/tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/04.html b/tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/04.html similarity index 100% rename from tests/mocks/data/ao3/tag-search/commit=Search Tags__tag_search[fandoms]=Hazbin Hotel (Cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=Character__tag_search[wrangling_status]=canonical/04.html rename to tests/mocks/data/ao3/tag-search/commit=search tags__tag_search[fandoms]=hazbin hotel (cartoon)__tag_search[name]=__tag_search[sort_column]=name__tag_search[sort_direction]=asc__tag_search[type]=character__tag_search[wrangling_status]=canonical/04.html From a8aa9cd4d93fdfb662a8b748ddc3e77bfdbaf7b0 Mon Sep 17 00:00:00 2001 From: "Ms. Boba" Date: Fri, 10 Oct 2025 07:40:40 +0000 Subject: [PATCH 4/4] make all the tag search folder paths lowercase --- tests/mocks/scripts/utils.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mocks/scripts/utils.mts b/tests/mocks/scripts/utils.mts index 3e77b96..07012dc 100644 --- a/tests/mocks/scripts/utils.mts +++ b/tests/mocks/scripts/utils.mts @@ -102,7 +102,7 @@ const getNormalizedTagSearchFolder = (searchParams: URLSearchParams) => { const sortedValues = values.length > 1 ? [...values].sort() : values; for (const value of sortedValues) { const segment = `${key}=${value == "any" ? "" : value}`; - entries.push(filenamify(segment, { replacement: "_", maxLength: 100 })); + entries.push(filenamify(segment.toLowerCase(), { replacement: "_", maxLength: 100 })); } }