✨ automatically migrate outdated configs

adminSiteServer/apiRouter.ts

@@ -106,6 +106,7 @@ import {
 import { uuidv7 } from "uuidv7"
 import {
     defaultGrapherConfig,
+    makeConfigValidAgainstLatestSchema,
     getVariableDataRoute,
     getVariableMetadataRoute,
 } from "@ourworldindata/grapher"
@@ -493,7 +494,7 @@ const saveGrapher = async (
         referencedVariablesMightChange = true,
     }: {
         user: DbPlainUser
-        newConfig: GrapherInterface // Note that it is valid for newConfig to be of an older schema version which means that GrapherInterface as a type is slightly misleading
+        newConfig: GrapherInterface
         existingConfig?: GrapherInterface
         // if undefined, keep inheritance as is.
         // if true or false, enable or disable inheritance
@@ -579,22 +580,6 @@ const saveGrapher = async (
         newConfig.version += 1
     else newConfig.version = 1
 
-    // if the schema version is missing, assume it's the latest
-    if (newConfig.$schema === undefined) {
-        newConfig.$schema = defaultGrapherConfig.$schema
-    } else if (
-        newConfig.$schema ===
-        "https://files.ourworldindata.org/schemas/grapher-schema.004.json"
-    ) {
-        // TODO: find a more principled way to do schema upgrades
-
-        // grapher-schema.004 -> grapher-schema.005 removed the obsolete hideLinesOutsideTolerance field
-        const configForMigration = newConfig as any
-        delete configForMigration.hideLinesOutsideTolerance
-        configForMigration.$schema = defaultGrapherConfig.$schema
-        newConfig = configForMigration
-    }
-
     // add the isPublished field if is missing
     if (newConfig.isPublished === undefined) {
         newConfig.isPublished = false
@@ -1034,9 +1019,21 @@ postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => {
         shouldInherit = req.query.inheritance === "enable"
     }
 
+    let validConfig: GrapherInterface
+    try {
+        validConfig = makeConfigValidAgainstLatestSchema(req.body)
+    } catch (err) {
+        await logErrorAndMaybeSendToBugsnag(err, req)
+        return {
+            success: false,
+            message: "Config invalid. Check the $schema field.",
+            error: String(err),
+        }
+    }
+
     const { chartId } = await saveGrapher(trx, {
         user: res.locals.user,
-        newConfig: req.body,
+        newConfig: validConfig,
         shouldInherit,
     })
 
@@ -1064,11 +1061,23 @@ putRouteWithRWTransaction(
             shouldInherit = req.query.inheritance === "enable"
         }
 
+        let validConfig: GrapherInterface
+        try {
+            validConfig = makeConfigValidAgainstLatestSchema(req.body)
+        } catch (err) {
+            await logErrorAndMaybeSendToBugsnag(err, req)
+            return {
+                success: false,
+                message: "Config invalid. Check the $schema field.",
+                error: String(err),
+            }
+        }
+
         const existingConfig = await expectChartById(trx, req.params.chartId)
 
         const { chartId, savedPatch } = await saveGrapher(trx, {
             user: res.locals.user,
-            newConfig: req.body,
+            newConfig: validConfig,
             existingConfig,
             shouldInherit,
         })
@@ -1614,8 +1623,20 @@ putRouteWithRWTransaction(
             throw new JsonError(`Variable with id ${variableId} not found`, 500)
         }
 
+        let validConfig: GrapherInterface
+        try {
+            validConfig = makeConfigValidAgainstLatestSchema(req.body)
+        } catch (err) {
+            await logErrorAndMaybeSendToBugsnag(err, req)
+            return {
+                success: false,
+                message: "Config invalid. Check the $schema field.",
+                error: String(err),
+            }
+        }
+
         const { savedPatch, updatedCharts } =
-            await updateGrapherConfigETLOfVariable(trx, variable, req.body)
+            await updateGrapherConfigETLOfVariable(trx, variable, validConfig)
 
         // trigger build if any published chart has been updated
         if (updatedCharts.some((chart) => chart.isPublished)) {
@@ -1705,8 +1726,20 @@ putRouteWithRWTransaction(
             throw new JsonError(`Variable with id ${variableId} not found`, 500)
         }
 
+        let validConfig: GrapherInterface
+        try {
+            validConfig = makeConfigValidAgainstLatestSchema(req.body)
+        } catch (err) {
+            await logErrorAndMaybeSendToBugsnag(err, req)
+            return {
+                success: false,
+                message: "Config invalid. Check the $schema field.",
+                error: String(err),
+            }
+        }
+
         const { savedPatch, updatedCharts } =
-            await updateGrapherConfigAdminOfVariable(trx, variable, req.body)
+            await updateGrapherConfigAdminOfVariable(trx, variable, validConfig)
 
         // trigger build if any published chart has been updated
         if (updatedCharts.some((chart) => chart.isPublished)) {

db/migration/1726588731621-MigrateOutdatedConfigsToLatestVersion.ts

@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class MigrateOutdatedConfigsToLatestVersion1726588731621
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        // we have v3 etl configs in the database; turn these into v5 configs
+        // by removing the `data` and `hideLinesOutsideTolerance` properties
+        await queryRunner.query(
+            `-- sql
+              UPDATE chart_configs
+              SET
+                  patch = JSON_SET(
+                      JSON_REMOVE(patch, '$.data', '$.hideLinesOutsideTolerance'),
+                      '$.$schema',
+                      'https://files.ourworldindata.org/schemas/grapher-schema.005.json'
+                  ),
+                  full = JSON_SET(
+                      JSON_REMOVE(full, '$.data', '$.hideLinesOutsideTolerance'),
+                      '$.$schema',
+                      'https://files.ourworldindata.org/schemas/grapher-schema.005.json'
+                  )
+              WHERE patch ->> '$.$schema' = 'https://files.ourworldindata.org/schemas/grapher-schema.003.json'
+            `
+        )
+    }
+
+    public async down(): Promise<void> {
+        throw new Error(
+            "Can't revert migration MigrateOutdatedConfigsToLatestVersion1726588731621"
+        )
+    }
+}

db/model/Variable.ts

@@ -189,11 +189,6 @@ function makeConfigValidForIndicator({
 }): GrapherInterface {
     const updatedConfig = { ...config }
 
-    // if no schema is given, assume it's the latest
-    if (!updatedConfig.$schema) {
-        updatedConfig.$schema = defaultGrapherConfig.$schema
-    }
-
     // validate the given y-dimensions
     const defaultDimension = { property: DimensionProperty.y, variableId }
     const [yDimensions, otherDimensions] = _.partition(

devTools/schema/generate-default-object-from-schema.ts

@@ -2,6 +2,15 @@
 
 import parseArgs from "minimist"
 import fs from "fs-extra"
+import { range } from "lodash"
+
+const schemaVersionRegex =
+    /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m
+const getSchemaVersion = (config: Record<string, any>): string =>
+    config.$schema?.match(schemaVersionRegex)?.groups?.version ?? "000"
+
+const toArrayString = (arr: string[]) =>
+    `[${arr.map((v) => `"${v}"`).join(", ")}]`
 
 function generateDefaultObjectFromSchema(
     schema: Record<string, any>,
@@ -49,13 +58,22 @@ async function main(parsedArgs: parseArgs.ParsedArgs) {
 
     // save as ts file if requested
     if (parsedArgs["save-ts"]) {
+        const latestVersion = getSchemaVersion(defaultConfig)
+        const outdatedVersionsAsInts = range(1, parseInt(latestVersion))
+        const outdatedVersions = outdatedVersionsAsInts.map((versionNumber) =>
+            versionNumber.toString().padStart(3, "0")
+        )
+
         const out = parsedArgs["save-ts"]
         const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
 
 // GENERATED BY devTools/schema/generate-default-object-from-schema.ts
 
 import { GrapherInterface } from "@ourworldindata/types"
 
+export const latestSchemaVersion = "${latestVersion}" as const
+export const outdatedSchemaVersions = ${toArrayString(outdatedVersions)} as const
+
 export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface`
         fs.outputFileSync(out, content)
     }

packages/@ourworldindata/grapher/src/index.ts

@@ -80,3 +80,4 @@ export {
     SlideShowController,
 } from "./slideshowController/SlideShowController"
 export { defaultGrapherConfig } from "./schema/defaultGrapherConfig"
+export { makeConfigValidAgainstLatestSchema } from "./schema/migrations/migrate"

packages/@ourworldindata/grapher/src/schema/README.md

@@ -19,6 +19,7 @@ Checklist for breaking changes:
 -   Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version
 -   Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below)
 -   Write a migration to update the `chart_configs.full` column in the database for all stand-alone charts
+-   Write a migration to update configs in code (see `migrations/migrations.ts`)
 
 To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run:
 

packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts

@@ -4,6 +4,9 @@
 
 import { GrapherInterface } from "@ourworldindata/types"
 
+export const latestSchemaVersion = "005" as const
+export const outdatedSchemaVersions = ["001", "002", "003", "004"] as const
+
 export const defaultGrapherConfig = {
     $schema: "https://files.ourworldindata.org/schemas/grapher-schema.005.json",
     map: {

packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts

@@ -0,0 +1,50 @@
+import {
+    latestSchemaVersion,
+    outdatedSchemaVersions,
+} from "../defaultGrapherConfig"
+
+const allSchemaVersions = [...outdatedSchemaVersions, latestSchemaVersion]
+
+type LatestSchemaVersion = typeof latestSchemaVersion
+type OutdatedSchemaVersion = (typeof outdatedSchemaVersions)[number]
+type SchemaVersion = OutdatedSchemaVersion | LatestSchemaVersion
+
+type Schema =
+    `https://files.ourworldindata.org/schemas/grapher-schema.${SchemaVersion}.json`
+
+// we can't type configs that don't adhere to the latest schema as we don't know what they look like
+export type AnyConfig = Record<string, any>
+export type AnyConfigWithSchema = Record<string, any> & {
+    $schema?: Schema
+}
+
+const schemaVersionRegex =
+    /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m
+
+const isValidSchemaVersion = (version: string): version is SchemaVersion =>
+    allSchemaVersions.includes(version as any)
+
+export function getSchemaVersion(config: AnyConfigWithSchema): SchemaVersion
+export function getSchemaVersion(config: AnyConfig): SchemaVersion | null {
+    const version = config.$schema?.match(schemaVersionRegex)?.groups?.version
+    if (!version || !isValidSchemaVersion(version)) return null
+    return version
+}
+
+export function createSchemaForVersion(version: SchemaVersion): Schema {
+    return `https://files.ourworldindata.org/schemas/grapher-schema.${version}.json`
+}
+
+export const isLatestVersion = (version: SchemaVersion) =>
+    version === latestSchemaVersion
+
+export const isOutdatedVersion = (version: SchemaVersion) =>
+    outdatedSchemaVersions.includes(version as any)
+
+export const isSchemaOutdated = (
+    config: AnyConfig
+): config is AnyConfigWithSchema => {
+    const version = getSchemaVersion(config)
+    if (!version) return false
+    return isOutdatedVersion(version)
+}

packages/@ourworldindata/grapher/src/schema/migrations/migrate.test.ts

@@ -0,0 +1,53 @@
+#! /usr/bin/env jest
+
+import { defaultGrapherConfig } from "../defaultGrapherConfig"
+import { makeConfigValidAgainstLatestSchema } from "./migrate"
+
+it("keeps valid configs as is", () => {
+    const validConfig = {
+        $schema: defaultGrapherConfig.$schema,
+        title: "Test",
+    }
+    expect(makeConfigValidAgainstLatestSchema(validConfig)).toEqual(validConfig)
+})
+
+it("if the schema field is missing, assumes the config adheres to latest schema version", () => {
+    expect(makeConfigValidAgainstLatestSchema({})).toEqual({
+        $schema: defaultGrapherConfig.$schema,
+    })
+})
+
+it("throws if the schema is invalid", () => {
+    expect(() =>
+        makeConfigValidAgainstLatestSchema({
+            $schema: "invalid",
+        })
+    ).toThrow()
+})
+
+it("runs multiple migrations if necessary", () => {
+    const outdatedConfig = {
+        $schema:
+            "https://files.ourworldindata.org/schemas/grapher-schema.003.json",
+        data: { availableEntities: [] }, // removed in v4
+        hideLinesOutsideTolerance: true, // removed in v5
+    }
+    const validConfig = makeConfigValidAgainstLatestSchema(outdatedConfig)
+    expect(validConfig).not.toHaveProperty("data")
+    expect(validConfig).not.toHaveProperty("hideLinesOutsideTolerance")
+})
+
+it("doesn't mutate the given config", () => {
+    const outdatedConfig = {
+        $schema:
+            "https://files.ourworldindata.org/schemas/grapher-schema.004.json",
+        hideLinesOutsideTolerance: true,
+    }
+    const validConfig = makeConfigValidAgainstLatestSchema(outdatedConfig)
+    expect(validConfig).not.toHaveProperty("hideLinesOutsideTolerance")
+    expect(outdatedConfig).toEqual({
+        $schema:
+            "https://files.ourworldindata.org/schemas/grapher-schema.004.json",
+        hideLinesOutsideTolerance: true,
+    })
+})

packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts

@@ -0,0 +1,53 @@
+import { GrapherInterface } from "@ourworldindata/types"
+import { cloneDeep } from "@ourworldindata/utils"
+
+import { defaultGrapherConfig } from "../defaultGrapherConfig"
+import {
+    getSchemaVersion,
+    isLatestVersion,
+    isSchemaOutdated,
+    type AnyConfig,
+    type AnyConfigWithSchema,
+} from "./helpers"
+import { runMigration } from "./migrations"
+
+const recursivelyUpgradeConfigToNextSchemaVersion = (
+    config: AnyConfigWithSchema
+): AnyConfigWithSchema => {
+    const version = getSchemaVersion(config)
+    if (isLatestVersion(version)) return config
+    return recursivelyUpgradeConfigToNextSchemaVersion(runMigration(config))
+}
+
+const migrateToLatestSchemaVersion = (
+    config: AnyConfigWithSchema
+): GrapherInterface =>
+    recursivelyUpgradeConfigToNextSchemaVersion(config) as GrapherInterface
+
+/**
+ * Makes a config valid against the latest schema version.
+ *
+ * Note that the given config is not actually validated against the schema.
+ * Instead, we rely on the schema version to determine if the config is outdated.
+ * If the config is outdated, we migrate it to the latest version by
+ * applying a series of predefined migrations.
+ */
+export const makeConfigValidAgainstLatestSchema = (config: AnyConfig) => {
+    // the config adheres to the latest schema
+    if (config.$schema === defaultGrapherConfig.$schema) return config
+
+    // if the schema field is missing, assume it's the latest version
+    // TODO: remove? this is a bit risky
+    if (!config.$schema) {
+        config.$schema = defaultGrapherConfig.$schema
+        return config
+    }
+
+    // if the schema version is outdated, migrate it to the latest version
+    if (isSchemaOutdated(config)) {
+        return migrateToLatestSchemaVersion(cloneDeep(config))
+    }
+
+    // else, the schema is invalid
+    throw new Error(`Invalid schema: ${config.$schema}`)
+}