✨ automatically migrate outdated configs

adminSiteServer/apiRouter.ts

@@ -106,6 +106,7 @@ import {
 import { uuidv7 } from "uuidv7"
 import {
     defaultGrapherConfig,
+    makeConfigValidAgainstLatestSchema,
     getVariableDataRoute,
     getVariableMetadataRoute,
 } from "@ourworldindata/grapher"
@@ -493,7 +494,7 @@ const saveGrapher = async (
         referencedVariablesMightChange = true,
     }: {
         user: DbPlainUser
-        newConfig: GrapherInterface // Note that it is valid for newConfig to be of an older schema version which means that GrapherInterface as a type is slightly misleading
+        newConfig: GrapherInterface
         existingConfig?: GrapherInterface
         // if undefined, keep inheritance as is.
         // if true or false, enable or disable inheritance
@@ -579,22 +580,6 @@ const saveGrapher = async (
         newConfig.version += 1
     else newConfig.version = 1
 
-    // if the schema version is missing, assume it's the latest
-    if (newConfig.$schema === undefined) {
-        newConfig.$schema = defaultGrapherConfig.$schema
-    } else if (
-        newConfig.$schema ===
-        "https://files.ourworldindata.org/schemas/grapher-schema.004.json"
-    ) {
-        // TODO: find a more principled way to do schema upgrades
-
-        // grapher-schema.004 -> grapher-schema.005 removed the obsolete hideLinesOutsideTolerance field
-        const configForMigration = newConfig as any
-        delete configForMigration.hideLinesOutsideTolerance
-        configForMigration.$schema = defaultGrapherConfig.$schema
-        newConfig = configForMigration
-    }
-
     // add the isPublished field if is missing
     if (newConfig.isPublished === undefined) {
         newConfig.isPublished = false
@@ -1034,9 +1019,21 @@ postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => {
         shouldInherit = req.query.inheritance === "enable"
     }
 
+    let validConfig: GrapherInterface
+    try {
+        validConfig = makeConfigValidAgainstLatestSchema(req.body)
+    } catch (err) {
+        await logErrorAndMaybeSendToBugsnag(err, req)
+        return {
+            success: false,
+            message: "Config invalid. Check the $schema field.",
+            error: String(err),
+        }
+    }
+
     const { chartId } = await saveGrapher(trx, {
         user: res.locals.user,
-        newConfig: req.body,
+        newConfig: validConfig,
         shouldInherit,
     })
 
@@ -1064,11 +1061,23 @@ putRouteWithRWTransaction(
             shouldInherit = req.query.inheritance === "enable"
         }
 
+        let validConfig: GrapherInterface
+        try {
+            validConfig = makeConfigValidAgainstLatestSchema(req.body)
+        } catch (err) {
+            await logErrorAndMaybeSendToBugsnag(err, req)
+            return {
+                success: false,
+                message: "Config invalid. Check the $schema field.",
+                error: String(err),
+            }
+        }
+
         const existingConfig = await expectChartById(trx, req.params.chartId)
 
         const { chartId, savedPatch } = await saveGrapher(trx, {
             user: res.locals.user,
-            newConfig: req.body,
+            newConfig: validConfig,
             existingConfig,
             shouldInherit,
         })
@@ -1614,8 +1623,20 @@ putRouteWithRWTransaction(
             throw new JsonError(`Variable with id ${variableId} not found`, 500)
         }
 
+        let validConfig: GrapherInterface
+        try {
+            validConfig = makeConfigValidAgainstLatestSchema(req.body)
+        } catch (err) {
+            await logErrorAndMaybeSendToBugsnag(err, req)
+            return {
+                success: false,
+                message: "Config invalid. Check the $schema field.",
+                error: String(err),
+            }
+        }
+
         const { savedPatch, updatedCharts } =
-            await updateGrapherConfigETLOfVariable(trx, variable, req.body)
+            await updateGrapherConfigETLOfVariable(trx, variable, validConfig)
 
         // trigger build if any published chart has been updated
         if (updatedCharts.some((chart) => chart.isPublished)) {
@@ -1705,8 +1726,20 @@ putRouteWithRWTransaction(
             throw new JsonError(`Variable with id ${variableId} not found`, 500)
         }
 
+        let validConfig: GrapherInterface
+        try {
+            validConfig = makeConfigValidAgainstLatestSchema(req.body)
+        } catch (err) {
+            await logErrorAndMaybeSendToBugsnag(err, req)
+            return {
+                success: false,
+                message: "Config invalid. Check the $schema field.",
+                error: String(err),
+            }
+        }
+
         const { savedPatch, updatedCharts } =
-            await updateGrapherConfigAdminOfVariable(trx, variable, req.body)
+            await updateGrapherConfigAdminOfVariable(trx, variable, validConfig)
 
         // trigger build if any published chart has been updated
         if (updatedCharts.some((chart) => chart.isPublished)) {

db/model/Variable.ts

@@ -12,6 +12,7 @@ import {
     getVariableDataRoute,
     getVariableMetadataRoute,
     defaultGrapherConfig,
+    makeConfigValidAgainstLatestSchema,
 } from "@ourworldindata/grapher"
 import pl from "nodejs-polars"
 import { uuidv7 } from "uuidv7"
@@ -189,11 +190,6 @@ function makeConfigValidForIndicator({
 }): GrapherInterface {
     const updatedConfig = { ...config }
 
-    // if no schema is given, assume it's the latest
-    if (!updatedConfig.$schema) {
-        updatedConfig.$schema = defaultGrapherConfig.$schema
-    }
-
     // validate the given y-dimensions
     const defaultDimension = { property: DimensionProperty.y, variableId }
     const [yDimensions, otherDimensions] = _.partition(

devTools/schema/generate-default-object-from-schema.ts

@@ -2,6 +2,15 @@
 
 import parseArgs from "minimist"
 import fs from "fs-extra"
+import { range } from "lodash"
+
+const schemaVersionRegex =
+    /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m
+const getSchemaVersion = (config: Record<string, any>): string =>
+    config.$schema?.match(schemaVersionRegex)?.groups?.version ?? "000"
+
+const toArrayString = (arr: string[]) =>
+    `[${arr.map((v) => `"${v}"`).join(", ")}]`
 
 function generateDefaultObjectFromSchema(
     schema: Record<string, any>,
@@ -49,13 +58,22 @@ async function main(parsedArgs: parseArgs.ParsedArgs) {
 
     // save as ts file if requested
     if (parsedArgs["save-ts"]) {
+        const currentVersion = getSchemaVersion(defaultConfig)
+        const outdatedVersionsAsInts = range(1, parseInt(currentVersion))
+        const outdatedVersions = outdatedVersionsAsInts.map((versionNumber) =>
+            versionNumber.toString().padStart(3, "0")
+        )
+
         const out = parsedArgs["save-ts"]
         const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
 
 // GENERATED BY devTools/schema/generate-default-object-from-schema.ts
 
 import { GrapherInterface } from "@ourworldindata/types"
 
+export const currentSchemaVersion = "${currentVersion}" as const
+export const outdatedSchemaVersions = ${toArrayString(outdatedVersions)} as const
+
 export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface`
         fs.outputFileSync(out, content)
     }

packages/@ourworldindata/grapher/src/index.ts

@@ -80,3 +80,4 @@ export {
     SlideShowController,
 } from "./slideshowController/SlideShowController"
 export { defaultGrapherConfig } from "./schema/defaultGrapherConfig"
+export { makeConfigValidAgainstLatestSchema } from "./schema/migrations/migrate"

packages/@ourworldindata/grapher/src/schema/README.md

@@ -19,6 +19,7 @@ Checklist for breaking changes:
 -   Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version
 -   Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below)
 -   Write a migration to update the `chart_configs.full` column in the database for all stand-alone charts
+-   Write a migration to update configs in code (migrations live in `migrations/migrations.ts`)
 
 To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run:
 

packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts

@@ -4,6 +4,9 @@
 
 import { GrapherInterface } from "@ourworldindata/types"
 
+export const currentSchemaVersion = "005" as const
+export const outdatedSchemaVersions = ["001", "002", "003", "004"] as const
+
 export const defaultGrapherConfig = {
     $schema: "https://files.ourworldindata.org/schemas/grapher-schema.005.json",
     map: {

packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts

@@ -0,0 +1,53 @@
+import {
+    currentSchemaVersion,
+    outdatedSchemaVersions,
+} from "../defaultGrapherConfig"
+
+const allSchemaVersions = [...outdatedSchemaVersions, currentSchemaVersion]
+
+type CurrentSchemaVersion = typeof currentSchemaVersion
+type OutdatedSchemaVersion = (typeof outdatedSchemaVersions)[number]
+type SchemaVersion = OutdatedSchemaVersion | CurrentSchemaVersion
+
+type Schema =
+    `https://files.ourworldindata.org/schemas/grapher-schema.${SchemaVersion}.json`
+
+// we can't type configs that don't adhere to the latest schema as we don't know what they look like
+export type AnyConfig = Record<string, any>
+export type AnyConfigWithSchema = Record<string, any> & {
+    $schema?: Schema
+}
+
+const schemaVersionRegex =
+    /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m
+
+const isValidSchemaVersion = (version: string): version is SchemaVersion =>
+    allSchemaVersions.includes(version as any)
+
+export function getSchemaVersion(config: AnyConfigWithSchema): SchemaVersion
+export function getSchemaVersion(config: AnyConfig): SchemaVersion | null {
+    const version = config.$schema?.match(schemaVersionRegex)?.groups?.version
+    if (!version || !isValidSchemaVersion(version)) return null
+    return version
+}
+
+export function createSchemaForVersion(version: SchemaVersion): Schema {
+    return `https://files.ourworldindata.org/schemas/grapher-schema.${version}.json`
+}
+
+export const isLatestVersion = (version: SchemaVersion) =>
+    version === currentSchemaVersion
+
+export const isOutdatedVersion = (version: SchemaVersion) =>
+    outdatedSchemaVersions.includes(version as any)
+
+export const isSchemaMissing = (config: AnyConfigWithSchema) =>
+    config.$schema === undefined
+
+export const isSchemaOutdated = (
+    config: AnyConfig
+): config is AnyConfigWithSchema => {
+    const version = getSchemaVersion(config)
+    if (!version) return false
+    return isOutdatedVersion(version)
+}

packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts

@@ -0,0 +1,44 @@
+import { GrapherInterface } from "@ourworldindata/types"
+import {
+    getSchemaVersion,
+    isLatestVersion,
+    isSchemaMissing,
+    type AnyConfig,
+    type AnyConfigWithSchema,
+} from "./helpers"
+import { runMigration } from "./migrations"
+import { defaultGrapherConfig } from "../defaultGrapherConfig"
+import { isSchemaOutdated } from "./helpers"
+
+const recursivelyUpgradeConfigToNextSchemaVersion = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    const version = getSchemaVersion(config)
+    if (isLatestVersion(version)) return config
+    return recursivelyUpgradeConfigToNextSchemaVersion(runMigration(config))
+}
+
+const migrateToLatestSchemaVersion = (
+    config: AnyConfigWithSchema
+): GrapherInterface =>
+    recursivelyUpgradeConfigToNextSchemaVersion(config) as GrapherInterface
+
+export const makeConfigValidAgainstLatestSchema = (config: AnyConfig) => {
+    // the config adheres to the latest schema
+    if (config.$schema === defaultGrapherConfig.$schema) return config
+
+    // if the schema field is missing, assume it's the latest version
+    // TODO: remove?
+    if (isSchemaMissing(config)) {
+        config.$schema = defaultGrapherConfig.$schema
+        return config
+    }
+
+    // if the schema version is outdated, migrate it to the latest version
+    if (isSchemaOutdated(config)) {
+        return migrateToLatestSchemaVersion(config)
+    }
+
+    // else, the schema is invalid
+    throw new Error(`Invalid schema: ${config.$schema}`)
+}

packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts

@@ -0,0 +1,71 @@
+import { match } from "ts-pattern"
+import {
+    type AnyConfigWithSchema,
+    createSchemaForVersion,
+    getSchemaVersion,
+    isLatestVersion,
+} from "./helpers"
+
+//
+// schema migrations
+//
+// add a migration if introducing a breaking change.
+// make sure to update the $schema field to the next version!
+//
+
+// see https://github.com/owid/owid-grapher/commit/26f2a0d1790c71bdda7e12f284ca552945d2f6ef
+const migrateFrom001To002 = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    delete config.selectedData
+    config.$schema = createSchemaForVersion("002")
+    return config
+}
+
+// see https://github.com/owid/owid-grapher/commit/4525ad81fb7064709ffab83677a8b0354b324dfb
+const migrateFrom002To003 = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    if (config.hideTitelAnnotation) {
+        config.hideTitleAnnotations = {
+            entity: true,
+            time: true,
+            change: true,
+        }
+    }
+    delete config.hideTitleAnnotation
+
+    config.$schema = createSchemaForVersion("003")
+    return config
+}
+
+// see https://github.com/owid/owid-grapher/commit/1776721253cf61d7f1e24ebadeaf7a7ca2f43ced
+const migrateFrom003To004 = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    delete config.data
+    config.$schema = createSchemaForVersion("004")
+    return config
+}
+
+// see https://github.com/owid/owid-grapher/commit/1d67de3174764a413bc5055fbdf34efb2b49e079
+const migrateFrom004To005 = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    delete config.hideLinesOutsideTolerance
+    config.$schema = createSchemaForVersion("005")
+    return config
+}
+
+export const runMigration = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    const version = getSchemaVersion(config)
+    if (isLatestVersion(version)) return config
+    return match(version)
+        .with("001", () => migrateFrom001To002(config))
+        .with("002", () => migrateFrom002To003(config))
+        .with("003", () => migrateFrom003To004(config))
+        .with("004", () => migrateFrom004To005(config))
+        .exhaustive()
+}