-
-
Notifications
You must be signed in to change notification settings - Fork 229
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
✨ automatically migrate outdated configs
- Loading branch information
1 parent
3ab39cb
commit a78ab14
Showing
11 changed files
with
340 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
33 changes: 33 additions & 0 deletions
33
db/migration/1726588731621-MigrateOutdatedConfigsToLatestVersion.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
import { MigrationInterface, QueryRunner } from "typeorm" | ||
|
||
export class MigrateOutdatedConfigsToLatestVersion1726588731621 | ||
implements MigrationInterface | ||
{ | ||
public async up(queryRunner: QueryRunner): Promise<void> { | ||
// we have v3 etl configs in the database; turn these into v5 configs | ||
// by removing the `data` and `hideLinesOutsideTolerance` properties | ||
await queryRunner.query( | ||
`-- sql | ||
UPDATE chart_configs | ||
SET | ||
patch = JSON_SET( | ||
JSON_REMOVE(patch, '$.data', '$.hideLinesOutsideTolerance'), | ||
'$.$schema', | ||
'https://files.ourworldindata.org/schemas/grapher-schema.005.json' | ||
), | ||
full = JSON_SET( | ||
JSON_REMOVE(full, '$.data', '$.hideLinesOutsideTolerance'), | ||
'$.$schema', | ||
'https://files.ourworldindata.org/schemas/grapher-schema.005.json' | ||
) | ||
WHERE patch ->> '$.$schema' = 'https://files.ourworldindata.org/schemas/grapher-schema.003.json' | ||
` | ||
) | ||
} | ||
|
||
public async down(): Promise<void> { | ||
throw new Error( | ||
"Can't revert migration MigrateOutdatedConfigsToLatestVersion1726588731621" | ||
) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
53 changes: 53 additions & 0 deletions
53
packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
import { | ||
latestSchemaVersion, | ||
outdatedSchemaVersions, | ||
} from "../defaultGrapherConfig" | ||
|
||
const allSchemaVersions = [...outdatedSchemaVersions, latestSchemaVersion] | ||
|
||
type LatestSchemaVersion = typeof latestSchemaVersion | ||
type OutdatedSchemaVersion = (typeof outdatedSchemaVersions)[number] | ||
type SchemaVersion = OutdatedSchemaVersion | LatestSchemaVersion | ||
|
||
type Schema = | ||
`https://files.ourworldindata.org/schemas/grapher-schema.${SchemaVersion}.json` | ||
|
||
// we can't type configs that don't adhere to the latest schema as we don't know what they look like | ||
export type AnyConfig = Record<string, any> | ||
export type AnyConfigWithSchema = Record<string, any> & { | ||
$schema?: Schema | ||
} | ||
|
||
const schemaVersionRegex = | ||
/https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?<version>\d{3})\.json/m | ||
|
||
const isValidSchemaVersion = (version: string): version is SchemaVersion => | ||
allSchemaVersions.includes(version as any) | ||
|
||
export function getSchemaVersion(config: AnyConfigWithSchema): SchemaVersion | ||
export function getSchemaVersion(config: AnyConfig): SchemaVersion | null { | ||
const version = config.$schema?.match(schemaVersionRegex)?.groups?.version | ||
if (!version || !isValidSchemaVersion(version)) return null | ||
return version | ||
} | ||
|
||
export function createSchemaForVersion(version: SchemaVersion): Schema { | ||
return `https://files.ourworldindata.org/schemas/grapher-schema.${version}.json` | ||
} | ||
|
||
export const isLatestVersion = (version: SchemaVersion) => | ||
version === latestSchemaVersion | ||
|
||
export const isOutdatedVersion = (version: SchemaVersion) => | ||
outdatedSchemaVersions.includes(version as any) | ||
|
||
export const isSchemaMissing = (config: AnyConfigWithSchema) => | ||
config.$schema === undefined | ||
|
||
export const isSchemaOutdated = ( | ||
config: AnyConfig | ||
): config is AnyConfigWithSchema => { | ||
const version = getSchemaVersion(config) | ||
if (!version) return false | ||
return isOutdatedVersion(version) | ||
} |
53 changes: 53 additions & 0 deletions
53
packages/@ourworldindata/grapher/src/schema/migrations/migrate.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
#! /usr/bin/env jest | ||
|
||
import { defaultGrapherConfig } from "../defaultGrapherConfig" | ||
import { makeConfigValidAgainstLatestSchema } from "./migrate" | ||
|
||
it("keeps valid configs as is", () => { | ||
const validConfig = { | ||
$schema: defaultGrapherConfig.$schema, | ||
title: "Test", | ||
} | ||
expect(makeConfigValidAgainstLatestSchema(validConfig)).toEqual(validConfig) | ||
}) | ||
|
||
it("if the schema field is missing, assumes the config adheres to latest schema version", () => { | ||
expect(makeConfigValidAgainstLatestSchema({})).toEqual({ | ||
$schema: defaultGrapherConfig.$schema, | ||
}) | ||
}) | ||
|
||
it("throws if the schema is invalid", () => { | ||
expect(() => | ||
makeConfigValidAgainstLatestSchema({ | ||
$schema: "invalid", | ||
}) | ||
).toThrow() | ||
}) | ||
|
||
it("runs multiple migrations if necessary", () => { | ||
const outdatedConfig = { | ||
$schema: | ||
"https://files.ourworldindata.org/schemas/grapher-schema.003.json", | ||
data: { availableEntities: [] }, // removed in v4 | ||
hideLinesOutsideTolerance: true, // removed in v5 | ||
} | ||
const validConfig = makeConfigValidAgainstLatestSchema(outdatedConfig) | ||
expect(validConfig).not.toHaveProperty("data") | ||
expect(validConfig).not.toHaveProperty("hideLinesOutsideTolerance") | ||
}) | ||
|
||
it("doesn't mutate the given config", () => { | ||
const outdatedConfig = { | ||
$schema: | ||
"https://files.ourworldindata.org/schemas/grapher-schema.004.json", | ||
hideLinesOutsideTolerance: true, | ||
} | ||
const validConfig = makeConfigValidAgainstLatestSchema(outdatedConfig) | ||
expect(validConfig).not.toHaveProperty("hideLinesOutsideTolerance") | ||
expect(outdatedConfig).toEqual({ | ||
$schema: | ||
"https://files.ourworldindata.org/schemas/grapher-schema.004.json", | ||
hideLinesOutsideTolerance: true, | ||
}) | ||
}) |
Oops, something went wrong.