From e3c1c28cc0a5cc507a1f38799eacd3001bae9016 Mon Sep 17 00:00:00 2001 From: sophiamersmann Date: Tue, 17 Sep 2024 14:31:05 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20automatically=20migrate=20outdated?= =?UTF-8?q?=20configs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRouter.ts | 17 +++------ .../generate-default-object-from-schema.ts | 18 ++++++++++ packages/@ourworldindata/grapher/src/index.ts | 1 + .../grapher/src/schema/README.md | 1 + .../src/schema/defaultGrapherConfig.ts | 3 ++ .../grapher/src/schema/migrations/helpers.ts | 36 +++++++++++++++++++ .../grapher/src/schema/migrations/migrate.ts | 34 ++++++++++++++++++ .../src/schema/migrations/migrations.ts | 34 ++++++++++++++++++ 8 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 5d845c205bb..5413c2a9ac6 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -106,6 +106,7 @@ import { import { uuidv7 } from "uuidv7" import { defaultGrapherConfig, + migrateToLatestSchemaVersion, getVariableDataRoute, getVariableMetadataRoute, } from "@ourworldindata/grapher" @@ -579,20 +580,12 @@ const saveGrapher = async ( newConfig.version += 1 else newConfig.version = 1 - // if the schema version is missing, assume it's the latest + // if the schema version is missing, assume it's the latest. + // if it's outdated, migrate it to the latest version if (newConfig.$schema === undefined) { newConfig.$schema = defaultGrapherConfig.$schema - } else if ( - newConfig.$schema === - "https://files.ourworldindata.org/schemas/grapher-schema.004.json" - ) { - // TODO: find a more principled way to do schema upgrades - - // grapher-schema.004 -> grapher-schema.005 removed the obsolete hideLinesOutsideTolerance field - const configForMigration = newConfig as any - delete configForMigration.hideLinesOutsideTolerance - configForMigration.$schema = defaultGrapherConfig.$schema - newConfig = configForMigration + } else if (newConfig.$schema !== defaultGrapherConfig.$schema) { + newConfig = migrateToLatestSchemaVersion(newConfig) } // add the isPublished field if is missing diff --git a/devTools/schema/generate-default-object-from-schema.ts b/devTools/schema/generate-default-object-from-schema.ts index 3ef1f66ca00..4ad8f64e75d 100644 --- a/devTools/schema/generate-default-object-from-schema.ts +++ b/devTools/schema/generate-default-object-from-schema.ts @@ -2,6 +2,15 @@ import parseArgs from "minimist" import fs from "fs-extra" +import { range } from "lodash" + +const schemaVersionRegex = + /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?\d{3})\.json/m +const getSchemaVersion = (config: Record): string => + config.$schema?.match(schemaVersionRegex)?.groups?.version ?? "000" + +const toArrayString = (arr: string[]) => + `[${arr.map((v) => `"${v}"`).join(", ")}]` function generateDefaultObjectFromSchema( schema: Record, @@ -49,6 +58,12 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { // save as ts file if requested if (parsedArgs["save-ts"]) { + const currentVersion = getSchemaVersion(defaultConfig) + const outdatedVersionsAsInts = range(1, parseInt(currentVersion)) + const outdatedVersions = outdatedVersionsAsInts.map((versionNumber) => + versionNumber.toString().padStart(3, "0") + ) + const out = parsedArgs["save-ts"] const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY @@ -56,6 +71,9 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { import { GrapherInterface } from "@ourworldindata/types" +export const currentSchemaVersion = "${currentVersion}" as const +export const outdatedSchemaVersions = ${toArrayString(outdatedVersions)} as const + export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface` fs.outputFileSync(out, content) } diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index 9e9d08b9902..4c17d44077e 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -80,3 +80,4 @@ export { SlideShowController, } from "./slideshowController/SlideShowController" export { defaultGrapherConfig } from "./schema/defaultGrapherConfig" +export { migrateToLatestSchemaVersion } from "./schema/migrations/migrate" diff --git a/packages/@ourworldindata/grapher/src/schema/README.md b/packages/@ourworldindata/grapher/src/schema/README.md index cad2a6eb595..6826869712d 100644 --- a/packages/@ourworldindata/grapher/src/schema/README.md +++ b/packages/@ourworldindata/grapher/src/schema/README.md @@ -19,6 +19,7 @@ Checklist for breaking changes: - Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version - Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below) - Write a migration to update the `chart_configs.full` column in the database for all stand-alone charts +- Write a migration to update configs in code (migrations live in `migrations/migrations.ts`) To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run: diff --git a/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts b/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts index c45ec0d1224..6ac0132decf 100644 --- a/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts +++ b/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts @@ -4,6 +4,9 @@ import { GrapherInterface } from "@ourworldindata/types" +export const currentSchemaVersion = "005" as const +export const outdatedSchemaVersions = ["001", "002", "003", "004"] as const + export const defaultGrapherConfig = { $schema: "https://files.ourworldindata.org/schemas/grapher-schema.005.json", map: { diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts b/packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts new file mode 100644 index 00000000000..50b03d057b9 --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts @@ -0,0 +1,36 @@ +import { + currentSchemaVersion, + outdatedSchemaVersions, +} from "../defaultGrapherConfig" + +type CurrentSchemaVersion = typeof currentSchemaVersion +type OutdatedSchemaVersion = (typeof outdatedSchemaVersions)[number] +type SchemaVersion = OutdatedSchemaVersion | CurrentSchemaVersion + +// we are missing migrations for the first two schema versions for now. +// if we encounter them at some point, we should add migrations for them as well. +const outdatedSchemaVersionsWithMigration = outdatedSchemaVersions.filter( + (v) => v !== "001" && v !== "002" +) +export type OutdatedSchemaVersionWithMigration = Exclude< + OutdatedSchemaVersion, + "001" | "002" +> + +// we can't type outdated configs as we don't know what they look like +export type OutdatedConfig = Record + +const schemaVersionRegex = + /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?\d{3})\.json/m + +export const getSchemaVersion = (config: OutdatedConfig): SchemaVersion => + config.$schema?.match(schemaVersionRegex)?.groups?.version as SchemaVersion + +export const createSchemaForVersion = (version: SchemaVersion): string => + `https://files.ourworldindata.org/schemas/grapher-schema.${version}.json` + +export const isLatestVersion = (version: SchemaVersion) => + version === currentSchemaVersion + +export const isOutdatedVersionThatCanBeMigrated = (version: SchemaVersion) => + outdatedSchemaVersionsWithMigration.includes(version as any) diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts b/packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts new file mode 100644 index 00000000000..cae1c91ec94 --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts @@ -0,0 +1,34 @@ +import { GrapherInterface } from "@ourworldindata/types" +import { + getSchemaVersion, + isLatestVersion, + isOutdatedVersionThatCanBeMigrated, + type OutdatedConfig, + type OutdatedSchemaVersionWithMigration, +} from "./helpers" +import { migrations } from "./migrations" + +const migrate = (config: OutdatedConfig): OutdatedConfig => { + const version = getSchemaVersion( + config + ) as OutdatedSchemaVersionWithMigration + return migrations[version](config) +} + +const recursivelyUpgradeConfigToNextSchemaVersion = ( + config: OutdatedConfig +): OutdatedConfig => { + const version = getSchemaVersion(config) + if (isLatestVersion(version)) { + return config + } else if (isOutdatedVersionThatCanBeMigrated(version)) { + return recursivelyUpgradeConfigToNextSchemaVersion(migrate(config)) + } else { + throw new Error(`Missing migration: ${version}`) + } +} + +export const migrateToLatestSchemaVersion = ( + config: OutdatedConfig +): GrapherInterface => + recursivelyUpgradeConfigToNextSchemaVersion(config) as GrapherInterface diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts b/packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts new file mode 100644 index 00000000000..676cb5399b8 --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts @@ -0,0 +1,34 @@ +import { + type OutdatedConfig, + type OutdatedSchemaVersionWithMigration, + createSchemaForVersion, +} from "./helpers" + +// +// schema migrations +// +// add a migration if introducing a breaking change. +// make sure to update the $schema field to the next version! +// + +// TODO: write migrations for the first two schema versions + +const migrateFrom003To004 = (config: OutdatedConfig): OutdatedConfig => { + delete config.data + config.$schema = createSchemaForVersion("004") + return config +} + +const migrateFrom004To005 = (config: OutdatedConfig): OutdatedConfig => { + delete config.hideLinesOutsideTolerance + config.$schema = createSchemaForVersion("005") + return config +} + +export const migrations: Record< + OutdatedSchemaVersionWithMigration, + (config: OutdatedConfig) => OutdatedConfig +> = { + "003": migrateFrom003To004, + "004": migrateFrom004To005, +} as const