✨ automatically migrate outdated configs

adminSiteServer/apiRouter.ts

@@ -106,6 +106,7 @@ import {
 import { uuidv7 } from "uuidv7"
 import {
     defaultGrapherConfig,
+    migrateToLatestSchemaVersion,
     getVariableDataRoute,
     getVariableMetadataRoute,
 } from "@ourworldindata/grapher"
@@ -579,20 +580,12 @@ const saveGrapher = async (
         newConfig.version += 1
     else newConfig.version = 1
 
-    // if the schema version is missing, assume it's the latest
+    // if the schema version is missing, assume it's the latest.
+    // if it's outdated, migrate it to the latest version
     if (newConfig.$schema === undefined) {
         newConfig.$schema = defaultGrapherConfig.$schema
-    } else if (
-        newConfig.$schema ===
-        "https://files.ourworldindata.org/schemas/grapher-schema.004.json"
-    ) {
-        // TODO: find a more principled way to do schema upgrades
-
-        // grapher-schema.004 -> grapher-schema.005 removed the obsolete hideLinesOutsideTolerance field
-        const configForMigration = newConfig as any
-        delete configForMigration.hideLinesOutsideTolerance
-        configForMigration.$schema = defaultGrapherConfig.$schema
-        newConfig = configForMigration
+    } else if (newConfig.$schema !== defaultGrapherConfig.$schema) {
+        newConfig = migrateToLatestSchemaVersion(newConfig)
     }
 
     // add the isPublished field if is missing

devTools/schema/generate-default-object-from-schema.ts

@@ -2,6 +2,15 @@
 
 import parseArgs from "minimist"
 import fs from "fs-extra"
+import { range } from "lodash"
+
+const schemaVersionRegex =
+    /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m
+const getSchemaVersion = (config: Record<string, any>): string =>
+    config.$schema?.match(schemaVersionRegex)?.groups?.version ?? "000"
+
+const toArrayString = (arr: string[]) =>
+    `[${arr.map((v) => `"${v}"`).join(", ")}]`
 
 function generateDefaultObjectFromSchema(
     schema: Record<string, any>,
@@ -49,13 +58,22 @@ async function main(parsedArgs: parseArgs.ParsedArgs) {
 
     // save as ts file if requested
     if (parsedArgs["save-ts"]) {
+        const currentVersion = getSchemaVersion(defaultConfig)
+        const outdatedVersionsAsInts = range(1, parseInt(currentVersion))
+        const outdatedVersions = outdatedVersionsAsInts.map((versionNumber) =>
+            versionNumber.toString().padStart(3, "0")
+        )
+
         const out = parsedArgs["save-ts"]
         const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
 
 // GENERATED BY devTools/schema/generate-default-object-from-schema.ts
 
 import { GrapherInterface } from "@ourworldindata/types"
 
+export const currentSchemaVersion = "${currentVersion}" as const
+export const outdatedSchemaVersions = ${toArrayString(outdatedVersions)} as const
+
 export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface`
         fs.outputFileSync(out, content)
     }

packages/@ourworldindata/grapher/src/index.ts

@@ -80,3 +80,4 @@ export {
     SlideShowController,
 } from "./slideshowController/SlideShowController"
 export { defaultGrapherConfig } from "./schema/defaultGrapherConfig"
+export { migrateToLatestSchemaVersion } from "./schema/migrations/migrate"

packages/@ourworldindata/grapher/src/schema/README.md

@@ -19,6 +19,7 @@ Checklist for breaking changes:
 -   Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version
 -   Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below)
 -   Write a migration to update the `chart_configs.full` column in the database for all stand-alone charts
+-   Write a migration to update configs in code (migrations live in `migrations/migrations.ts`)
 
 To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run:
 

packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts

@@ -4,6 +4,9 @@
 
 import { GrapherInterface } from "@ourworldindata/types"
 
+export const currentSchemaVersion = "005" as const
+export const outdatedSchemaVersions = ["001", "002", "003", "004"] as const
+
 export const defaultGrapherConfig = {
     $schema: "https://files.ourworldindata.org/schemas/grapher-schema.005.json",
     map: {

packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts

@@ -0,0 +1,36 @@
+import {
+    currentSchemaVersion,
+    outdatedSchemaVersions,
+} from "../defaultGrapherConfig"
+
+type CurrentSchemaVersion = typeof currentSchemaVersion
+type OutdatedSchemaVersion = (typeof outdatedSchemaVersions)[number]
+type SchemaVersion = OutdatedSchemaVersion | CurrentSchemaVersion
+
+// we are missing migrations for the first two schema versions for now.
+// if we encounter them at some point, we should add migrations for them as well.
+const outdatedSchemaVersionsWithMigration = outdatedSchemaVersions.filter(
+    (v) => v !== "001" && v !== "002"
+)
+export type OutdatedSchemaVersionWithMigration = Exclude<
+    OutdatedSchemaVersion,
+    "001" | "002"
+>
+
+// we can't type outdated configs as we don't know what they look like
+export type OutdatedConfig = Record<string, any>
+
+const schemaVersionRegex =
+    /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m
+
+export const getSchemaVersion = (config: OutdatedConfig): SchemaVersion =>
+    config.$schema?.match(schemaVersionRegex)?.groups?.version as SchemaVersion
+
+export const createSchemaForVersion = (version: SchemaVersion): string =>
+    `https://files.ourworldindata.org/schemas/grapher-schema.${version}.json`
+
+export const isLatestVersion = (version: SchemaVersion) =>
+    version === currentSchemaVersion
+
+export const isOutdatedVersionThatCanBeMigrated = (version: SchemaVersion) =>
+    outdatedSchemaVersionsWithMigration.includes(version as any)

packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts

@@ -0,0 +1,26 @@
+import { GrapherInterface } from "@ourworldindata/types"
+import {
+    getSchemaVersion,
+    isLatestVersion,
+    isOutdatedVersionThatCanBeMigrated,
+    type OutdatedConfig,
+} from "./helpers"
+import { runMigration } from "./migrations"
+
+const recursivelyUpgradeConfigToNextSchemaVersion = (
+    config: OutdatedConfig
+): OutdatedConfig => {
+    const version = getSchemaVersion(config)
+    if (isLatestVersion(version)) {
+        return config
+    } else if (isOutdatedVersionThatCanBeMigrated(version)) {
+        return recursivelyUpgradeConfigToNextSchemaVersion(runMigration(config))
+    } else {
+        throw new Error(`Missing migration: ${version}`)
+    }
+}
+
+export const migrateToLatestSchemaVersion = (
+    config: OutdatedConfig
+): GrapherInterface =>
+    recursivelyUpgradeConfigToNextSchemaVersion(config) as GrapherInterface

packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts

@@ -0,0 +1,38 @@
+import { match } from "ts-pattern"
+import {
+    type OutdatedConfig,
+    type OutdatedSchemaVersionWithMigration,
+    createSchemaForVersion,
+    getSchemaVersion,
+} from "./helpers"
+
+//
+// schema migrations
+//
+// add a migration if introducing a breaking change.
+// make sure to update the $schema field to the next version!
+//
+
+// TODO: write migrations for the first two schema versions
+
+const migrateFrom003To004 = (config: OutdatedConfig): OutdatedConfig => {
+    delete config.data
+    config.$schema = createSchemaForVersion("004")
+    return config
+}
+
+const migrateFrom004To005 = (config: OutdatedConfig): OutdatedConfig => {
+    delete config.hideLinesOutsideTolerance
+    config.$schema = createSchemaForVersion("005")
+    return config
+}
+
+export const runMigration = (config: OutdatedConfig): OutdatedConfig => {
+    const version = getSchemaVersion(
+        config
+    ) as OutdatedSchemaVersionWithMigration
+    return match(version)
+        .with("003", () => migrateFrom003To004(config))
+        .with("004", () => migrateFrom004To005(config))
+        .exhaustive()
+}