From 7388f1c25d1c0b792d0b0fd4031bfbfb114e4130 Mon Sep 17 00:00:00 2001
From: Richard Herndon <richardmherndon@gmail.com>
Date: Thu, 14 Nov 2024 14:40:35 -0800
Subject: [PATCH 1/4] update the cli-migrations docker images so that their
 entrypoints allow use of the --disallow-inconsistent-metadata flag when
 applying metadata (fixes #10599)

---
 .../auto-apply-migrations.mdx                        |  7 +++++++
 packaging/cli-migrations/v2/README.md                |  8 ++++++++
 packaging/cli-migrations/v2/docker-entrypoint.sh     | 12 +++++++++++-
 packaging/cli-migrations/v3/README.md                |  8 ++++++++
 packaging/cli-migrations/v3/docker-entrypoint.sh     | 12 +++++++++++-
 5 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/docs/docs/migrations-metadata-seeds/auto-apply-migrations.mdx b/docs/docs/migrations-metadata-seeds/auto-apply-migrations.mdx
index ab2d41687b446..0dcfce2032353 100644
--- a/docs/docs/migrations-metadata-seeds/auto-apply-migrations.mdx
+++ b/docs/docs/migrations-metadata-seeds/auto-apply-migrations.mdx
@@ -105,3 +105,10 @@ If you're managing Migrations with a different tool and want to use this image t
 `metadata` directory of your Hasura Project at the `/hasura-metadata`
 [path of this Docker container the container's entry point script](https://github.com/hasura/graphql-engine/blob/master/packaging/cli-migrations/v3/docker-entrypoint.sh#L13)
 will apply the Metadata before starting the server.
+
+## Disallowing Inconsistent Metadata
+
+If you set the `HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA` environment variable to `true`, the entrypoint script will
+reject inconsistent metadata and exit with a non-zero exit code. This has the same behavior as the
+`--disallow-inconsistent-metadata` flag in the `hasura metadata apply` command documented
+[here](/hasura-cli/commands/hasura_metadata_apply.mdx).
diff --git a/packaging/cli-migrations/v2/README.md b/packaging/cli-migrations/v2/README.md
index 04254b3f2e1bb..6e97b8fd49e7b 100644
--- a/packaging/cli-migrations/v2/README.md
+++ b/packaging/cli-migrations/v2/README.md
@@ -134,3 +134,11 @@ At least one of these **must** be configured to migrate the database successfull
 - `HASURA_GRAPHQL_MIGRATIONS_SERVER_TIMEOUT` (default=`30s`)
 
   Specify the server timeout threshold.
+
+### Disallow Inconsistent Metadata (Optional)
+
+- `HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA` (default=`false`)
+
+  If set to true, inconsistent metadata will be rejected, causing the entrypoint script to exit with a non-zero exit code.
+  Has the same behavior as the `--disallow-inconsistent-metadata` flag in the `hasura metadata apply` command documented 
+  [here](https://hasura.io/docs/latest/hasura-cli/commands/hasura_metadata_apply).
diff --git a/packaging/cli-migrations/v2/docker-entrypoint.sh b/packaging/cli-migrations/v2/docker-entrypoint.sh
index 81548b240f25e..d1f50841a6ecb 100755
--- a/packaging/cli-migrations/v2/docker-entrypoint.sh
+++ b/packaging/cli-migrations/v2/docker-entrypoint.sh
@@ -11,6 +11,7 @@ log() {
 
 DEFAULT_MIGRATIONS_DIR="/hasura-migrations"
 DEFAULT_METADATA_DIR="/hasura-metadata"
+DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG=""
 TEMP_PROJECT_DIR="/tmp/hasura-project"
 
 # configure the target database for migrations
@@ -72,6 +73,15 @@ if [ -z ${HASURA_GRAPHQL_METADATA_DIR+x} ]; then
     HASURA_GRAPHQL_METADATA_DIR="$DEFAULT_METADATA_DIR"
 fi
 
+# check if disallow metadata inconsistency is set to true (case insensitive), default otherwise
+if [ "$(echo "$HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is set to true, disallowing inconsistent metadata"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="--disallow-inconsistent-metadata"
+else
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is not true, falling back to default CLI behavior"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="$DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG"
+fi
+
 # apply migrations if the directory exist
 if [ -d "$HASURA_GRAPHQL_MIGRATIONS_DIR" ]; then
     log "migrations-apply" "applying migrations from $HASURA_GRAPHQL_MIGRATIONS_DIR"
@@ -95,7 +105,7 @@ if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
     echo "version: 2" > config.yaml
     echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
     echo "metadata_directory: metadata" >> config.yaml
-    hasura-cli metadata apply 
+    hasura-cli metadata apply $HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA
 else
     log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
 fi
diff --git a/packaging/cli-migrations/v3/README.md b/packaging/cli-migrations/v3/README.md
index f4303978b25d0..d81e451a48e63 100644
--- a/packaging/cli-migrations/v3/README.md
+++ b/packaging/cli-migrations/v3/README.md
@@ -71,3 +71,11 @@ Optional configuration for the server which boots during migrations.
 - `HASURA_GRAPHQL_MIGRATIONS_SERVER_TIMEOUT` (default=`30s`)
 
   Specify the server timeout threshold.
+
+### Disallow Inconsistent Metadata (Optional)
+
+- `HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA` (default=`false`)
+
+  If set to true, inconsistent metadata will be rejected, causing the entrypoint script to exit with a non-zero exit code.
+  Has the same behavior as the `--disallow-inconsistent-metadata` flag in the `hasura metadata apply` command documented 
+  [here](https://hasura.io/docs/latest/hasura-cli/commands/hasura_metadata_apply).
diff --git a/packaging/cli-migrations/v3/docker-entrypoint.sh b/packaging/cli-migrations/v3/docker-entrypoint.sh
index ec6d1c78699e4..f32c5faf487e4 100755
--- a/packaging/cli-migrations/v3/docker-entrypoint.sh
+++ b/packaging/cli-migrations/v3/docker-entrypoint.sh
@@ -11,6 +11,7 @@ log() {
 
 DEFAULT_MIGRATIONS_DIR="/hasura-migrations"
 DEFAULT_METADATA_DIR="/hasura-metadata"
+DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG=""
 TEMP_PROJECT_DIR="/tmp/hasura-project"
 
 if [ -z ${HGE_BINARY+x} ]; then
@@ -67,6 +68,15 @@ if [ -z ${HASURA_GRAPHQL_METADATA_DIR+x} ]; then
     HASURA_GRAPHQL_METADATA_DIR="$DEFAULT_METADATA_DIR"
 fi
 
+# check if disallow metadata inconsistency is set to true (case insensitive), default otherwise
+if [ "$(echo "$HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is set to true, disallowing inconsistent metadata"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="--disallow-inconsistent-metadata"
+else
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is not true, falling back to default CLI behavior"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="$DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG"
+fi
+
 # apply metadata if the directory exist
 if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
     rm -rf "$TEMP_PROJECT_DIR"
@@ -77,7 +87,7 @@ if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
     echo "version: 3" > config.yaml
     echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
     echo "metadata_directory: metadata" >> config.yaml
-    hasura-cli metadata apply 
+    hasura-cli metadata apply $HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA
 else
     log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
 fi

From a02bffe66ce936089d59a76baeaa6efa100f554b Mon Sep 17 00:00:00 2001
From: Richard Herndon <richardmherndon@gmail.com>
Date: Fri, 15 Nov 2024 09:36:12 -0800
Subject: [PATCH 2/4] sketch out tests, but need to figure out how to run them

---
 .../v2/test/bad_metadata/public_test.yaml              | 10 ++++++++++
 packaging/cli-migrations/v2/test/test.sh               |  8 ++++++++
 .../v3/test/bad_metadata/public_test.yaml              | 10 ++++++++++
 packaging/cli-migrations/v3/test/docker-compose.yaml   |  3 ++-
 packaging/cli-migrations/v3/test/test.sh               |  8 ++++++++
 5 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 packaging/cli-migrations/v2/test/bad_metadata/public_test.yaml
 create mode 100644 packaging/cli-migrations/v3/test/bad_metadata/public_test.yaml

diff --git a/packaging/cli-migrations/v2/test/bad_metadata/public_test.yaml b/packaging/cli-migrations/v2/test/bad_metadata/public_test.yaml
new file mode 100644
index 0000000000000..2d7dbac9b7230
--- /dev/null
+++ b/packaging/cli-migrations/v2/test/bad_metadata/public_test.yaml
@@ -0,0 +1,10 @@
+# this file contains metadata that is intentionally inconsistent with the database, to test
+# the behavior of the entrypoint script when inconsistent metadata is detected.
+# the fake_table table does not exist.
+- table:
+    schema: public
+    name: test
+  object_relationships:
+    - name: fake_table
+      using:
+        foreign_key_constraint_on: test_row_id
diff --git a/packaging/cli-migrations/v2/test/test.sh b/packaging/cli-migrations/v2/test/test.sh
index 69a9011e7ca0a..d394d772191c6 100755
--- a/packaging/cli-migrations/v2/test/test.sh
+++ b/packaging/cli-migrations/v2/test/test.sh
@@ -41,5 +41,13 @@ wait_for_server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f   -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json -
 # get list of migrations applied from graphql-engine server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f   -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json -
+
+# stop the graphql-engine container
+docker compose stop graphql-engine
+# overwrite existing metadata with intentionally inconsistent metadata 
+docker cp bad_metadata/tables.yaml graphql-engine:/hasura-metadata/tables.yaml
+
+# TODO: start up the container with inconsistent metadata disallowed, verify that it fails to start
+
 # delete postgres and graphql-engine
 docker compose down -v
diff --git a/packaging/cli-migrations/v3/test/bad_metadata/public_test.yaml b/packaging/cli-migrations/v3/test/bad_metadata/public_test.yaml
new file mode 100644
index 0000000000000..485d898181b59
--- /dev/null
+++ b/packaging/cli-migrations/v3/test/bad_metadata/public_test.yaml
@@ -0,0 +1,10 @@
+# this file contains metadata that is intentionally inconsistent with the database, to test
+# the behavior of the entrypoint script when inconsistent metadata is detected.
+# the fake_table table does not exist.
+table:
+  name: test
+  schema: public
+object_relationships:
+  - name: fake_table
+    using:
+      foreign_key_constraint_on: test_row_id
diff --git a/packaging/cli-migrations/v3/test/docker-compose.yaml b/packaging/cli-migrations/v3/test/docker-compose.yaml
index 674d07aea680d..ca8572765889b 100644
--- a/packaging/cli-migrations/v3/test/docker-compose.yaml
+++ b/packaging/cli-migrations/v3/test/docker-compose.yaml
@@ -16,4 +16,5 @@ services:
     environment:
       HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
       HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console
-      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
\ No newline at end of file
+      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA: "true"
\ No newline at end of file
diff --git a/packaging/cli-migrations/v3/test/test.sh b/packaging/cli-migrations/v3/test/test.sh
index 38376511f5034..76cbc009c37a8 100755
--- a/packaging/cli-migrations/v3/test/test.sh
+++ b/packaging/cli-migrations/v3/test/test.sh
@@ -38,5 +38,13 @@ wait_for_server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/metadata | jq -j '.' | diff validation/metadata.json -
 # get list of migrations applied from graphql-engine server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f -d'{"type" : "get_catalog_state", "args" : {} }' localhost:8080/v1/metadata | jq .cli_state | diff validation/catalog_cli_state.json -
+
+# stop the graphql-engine container
+docker compose stop graphql-engine
+# overwrite existing metadata with intentionally inconsistent metadata 
+docker cp bad_metadata/public_test.yaml graphql-engine:/hasura-metadata/databases/default/tables/public_test.yaml
+
+# TODO: start up the container with inconsistent metadata disallowed, verify that it fails to start
+
 # delete postgres and graphql-engine
 docker compose down -v

From b68a4075c5e579acbe4cc03e2deed237ea64bea5 Mon Sep 17 00:00:00 2001
From: Richard Herndon <richardmherndon@gmail.com>
Date: Fri, 15 Nov 2024 13:39:40 -0800
Subject: [PATCH 3/4] use consistent migrations-vs-metadata ordering in v3 and
 v2 entrypoint scripts

---
 .../cli-migrations/v3/docker-entrypoint.sh    | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/packaging/cli-migrations/v3/docker-entrypoint.sh b/packaging/cli-migrations/v3/docker-entrypoint.sh
index f32c5faf487e4..92d863e0efb50 100755
--- a/packaging/cli-migrations/v3/docker-entrypoint.sh
+++ b/packaging/cli-migrations/v3/docker-entrypoint.sh
@@ -77,21 +77,6 @@ else
     HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="$DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG"
 fi
 
-# apply metadata if the directory exist
-if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
-    rm -rf "$TEMP_PROJECT_DIR"
-    log "migrations-apply" "applying metadata from $HASURA_GRAPHQL_METADATA_DIR"
-    mkdir -p "$TEMP_PROJECT_DIR"
-    cp -a "$HASURA_GRAPHQL_METADATA_DIR/." "$TEMP_PROJECT_DIR/metadata/"
-    cd "$TEMP_PROJECT_DIR"
-    echo "version: 3" > config.yaml
-    echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
-    echo "metadata_directory: metadata" >> config.yaml
-    hasura-cli metadata apply $HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA
-else
-    log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
-fi
-
 # apply migrations if the directory exist
 if [ -d "$HASURA_GRAPHQL_MIGRATIONS_DIR" ]; then
     log "migrations-apply" "applying migrations from $HASURA_GRAPHQL_MIGRATIONS_DIR"
@@ -107,6 +92,21 @@ else
     log "migrations-apply" "directory $HASURA_GRAPHQL_MIGRATIONS_DIR does not exist, skipping migrations"
 fi
 
+# apply metadata if the directory exist
+if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
+    rm -rf "$TEMP_PROJECT_DIR"
+    log "migrations-apply" "applying metadata from $HASURA_GRAPHQL_METADATA_DIR"
+    mkdir -p "$TEMP_PROJECT_DIR"
+    cp -a "$HASURA_GRAPHQL_METADATA_DIR/." "$TEMP_PROJECT_DIR/metadata/"
+    cd "$TEMP_PROJECT_DIR"
+    echo "version: 3" > config.yaml
+    echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
+    echo "metadata_directory: metadata" >> config.yaml
+    hasura-cli metadata apply $HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA
+else
+    log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
+fi
+
 # kill graphql engine that we started earlier
 log "migrations-shutdown" "killing temporary server"
 kill $PID

From 034fd38ef76e3b134ac288ee8548c6261485e7c8 Mon Sep 17 00:00:00 2001
From: Richard Herndon <richardmherndon@gmail.com>
Date: Fri, 15 Nov 2024 13:40:09 -0800
Subject: [PATCH 4/4] modify tests to verify that inconsistent metadata
 prevents startup when new flag is set

---
 .../{public_test.yaml => tables.yaml}          |  0
 .../cli-migrations/v2/test/docker-compose.yaml |  3 ++-
 packaging/cli-migrations/v2/test/test.sh       | 18 ++++++++++++++++--
 packaging/cli-migrations/v3/test/test.sh       | 18 ++++++++++++++++--
 4 files changed, 34 insertions(+), 5 deletions(-)
 rename packaging/cli-migrations/v2/test/bad_metadata/{public_test.yaml => tables.yaml} (100%)

diff --git a/packaging/cli-migrations/v2/test/bad_metadata/public_test.yaml b/packaging/cli-migrations/v2/test/bad_metadata/tables.yaml
similarity index 100%
rename from packaging/cli-migrations/v2/test/bad_metadata/public_test.yaml
rename to packaging/cli-migrations/v2/test/bad_metadata/tables.yaml
diff --git a/packaging/cli-migrations/v2/test/docker-compose.yaml b/packaging/cli-migrations/v2/test/docker-compose.yaml
index 674d07aea680d..ca8572765889b 100644
--- a/packaging/cli-migrations/v2/test/docker-compose.yaml
+++ b/packaging/cli-migrations/v2/test/docker-compose.yaml
@@ -16,4 +16,5 @@ services:
     environment:
       HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
       HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console
-      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
\ No newline at end of file
+      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA: "true"
\ No newline at end of file
diff --git a/packaging/cli-migrations/v2/test/test.sh b/packaging/cli-migrations/v2/test/test.sh
index d394d772191c6..0e4728ab76e40 100755
--- a/packaging/cli-migrations/v2/test/test.sh
+++ b/packaging/cli-migrations/v2/test/test.sh
@@ -44,10 +44,24 @@ docker run --rm --network container:graphql-engine curlimages/curl -s -f   -d'{"
 
 # stop the graphql-engine container
 docker compose stop graphql-engine
+
 # overwrite existing metadata with intentionally inconsistent metadata 
 docker cp bad_metadata/tables.yaml graphql-engine:/hasura-metadata/tables.yaml
-
-# TODO: start up the container with inconsistent metadata disallowed, verify that it fails to start
+# start the container with new, inconsistent metadata
+docker compose up -d --no-recreate graphql-engine
+# confirm that the service does not become available for at least 10 seconds
+for _ in $(seq 1 10); do
+    if docker run --rm --network container:graphql-engine curlimages/curl -s -f http://127.0.0.1:8080/v1/version > /dev/null 2>&1; then
+        echo "Server became available when it should not have"
+        exit 1
+    fi
+    sleep 1
+done
+# confirm that the service's log contain the expected error
+if ! docker compose logs graphql-engine | grep -qi "error applying metadata"; then
+    echo "Expected error message not found in logs"
+    exit 1
+fi
 
 # delete postgres and graphql-engine
 docker compose down -v
diff --git a/packaging/cli-migrations/v3/test/test.sh b/packaging/cli-migrations/v3/test/test.sh
index 76cbc009c37a8..c08ea98bd86c7 100755
--- a/packaging/cli-migrations/v3/test/test.sh
+++ b/packaging/cli-migrations/v3/test/test.sh
@@ -41,10 +41,24 @@ docker run --rm --network container:graphql-engine curlimages/curl -s -f -d'{"ty
 
 # stop the graphql-engine container
 docker compose stop graphql-engine
+
 # overwrite existing metadata with intentionally inconsistent metadata 
 docker cp bad_metadata/public_test.yaml graphql-engine:/hasura-metadata/databases/default/tables/public_test.yaml
-
-# TODO: start up the container with inconsistent metadata disallowed, verify that it fails to start
+# start the container with new, inconsistent metadata
+docker compose up -d --no-recreate graphql-engine
+# confirm that the service does not become available for at least 10 seconds
+for _ in $(seq 1 10); do
+    if docker run --rm --network container:graphql-engine curlimages/curl -s -f http://127.0.0.1:8080/v1/version > /dev/null 2>&1; then
+        echo "Server became available when it should not have"
+        exit 1
+    fi
+    sleep 1
+done
+# confirm that the service's log contain the expected error
+if ! docker compose logs graphql-engine | grep -qi "error applying metadata"; then
+    echo "Expected error message not found in logs"
+    exit 1
+fi
 
 # delete postgres and graphql-engine
 docker compose down -v