Support disallowing inconsistent metadata in cli-migrations images (close #10599)

docs/docs/migrations-metadata-seeds/auto-apply-migrations.mdx

@@ -105,3 +105,10 @@ If you're managing Migrations with a different tool and want to use this image t
 `metadata` directory of your Hasura Project at the `/hasura-metadata`
 [path of this Docker container the container's entry point script](https://github.com/hasura/graphql-engine/blob/master/packaging/cli-migrations/v3/docker-entrypoint.sh#L13)
 will apply the Metadata before starting the server.
+
+## Disallowing Inconsistent Metadata
+
+If you set the `HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA` environment variable to `true`, the entrypoint script will
+reject inconsistent metadata and exit with a non-zero exit code. This has the same behavior as the
+`--disallow-inconsistent-metadata` flag in the `hasura metadata apply` command documented
+[here](/hasura-cli/commands/hasura_metadata_apply.mdx).

packaging/cli-migrations/v2/README.md

@@ -134,3 +134,11 @@ At least one of these **must** be configured to migrate the database successfull
 - `HASURA_GRAPHQL_MIGRATIONS_SERVER_TIMEOUT` (default=`30s`)
 
   Specify the server timeout threshold.
+
+### Disallow Inconsistent Metadata (Optional)
+
+- `HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA` (default=`false`)
+
+  If set to true, inconsistent metadata will be rejected, causing the entrypoint script to exit with a non-zero exit code.
+  Has the same behavior as the `--disallow-inconsistent-metadata` flag in the `hasura metadata apply` command documented 
+  [here](https://hasura.io/docs/latest/hasura-cli/commands/hasura_metadata_apply).

packaging/cli-migrations/v2/docker-entrypoint.sh

@@ -11,6 +11,7 @@ log() {
 
 DEFAULT_MIGRATIONS_DIR="/hasura-migrations"
 DEFAULT_METADATA_DIR="/hasura-metadata"
+DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG=""
 TEMP_PROJECT_DIR="/tmp/hasura-project"
 
 # configure the target database for migrations
@@ -72,6 +73,15 @@ if [ -z ${HASURA_GRAPHQL_METADATA_DIR+x} ]; then
     HASURA_GRAPHQL_METADATA_DIR="$DEFAULT_METADATA_DIR"
 fi
 
+# check if disallow metadata inconsistency is set to true (case insensitive), default otherwise
+if [ "$(echo "$HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is set to true, disallowing inconsistent metadata"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="--disallow-inconsistent-metadata"
+else
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is not true, falling back to default CLI behavior"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="$DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG"
+fi
+
 # apply migrations if the directory exist
 if [ -d "$HASURA_GRAPHQL_MIGRATIONS_DIR" ]; then
     log "migrations-apply" "applying migrations from $HASURA_GRAPHQL_MIGRATIONS_DIR"
@@ -95,7 +105,7 @@ if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
     echo "version: 2" > config.yaml
     echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
     echo "metadata_directory: metadata" >> config.yaml
-    hasura-cli metadata apply 
+    hasura-cli metadata apply $HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA
 else
     log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
 fi

packaging/cli-migrations/v2/test/bad_metadata/tables.yaml

@@ -0,0 +1,10 @@
+# this file contains metadata that is intentionally inconsistent with the database, to test
+# the behavior of the entrypoint script when inconsistent metadata is detected.
+# the fake_table table does not exist.
+- table:
+    schema: public
+    name: test
+  object_relationships:
+    - name: fake_table
+      using:
+        foreign_key_constraint_on: test_row_id

packaging/cli-migrations/v2/test/docker-compose.yaml

@@ -16,4 +16,5 @@ services:
     environment:
       HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
       HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console
-      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA: "true"

packaging/cli-migrations/v2/test/test.sh

@@ -41,5 +41,27 @@ wait_for_server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f   -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json -
 # get list of migrations applied from graphql-engine server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f   -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json -
+
+# stop the graphql-engine container
+docker compose stop graphql-engine
+
+# overwrite existing metadata with intentionally inconsistent metadata 
+docker cp bad_metadata/tables.yaml graphql-engine:/hasura-metadata/tables.yaml
+# start the container with new, inconsistent metadata
+docker compose up -d --no-recreate graphql-engine
+# confirm that the service does not become available for at least 10 seconds
+for _ in $(seq 1 10); do
+    if docker run --rm --network container:graphql-engine curlimages/curl -s -f http://127.0.0.1:8080/v1/version > /dev/null 2>&1; then
+        echo "Server became available when it should not have"
+        exit 1
+    fi
+    sleep 1
+done
+# confirm that the service's log contain the expected error
+if ! docker compose logs graphql-engine | grep -qi "error applying metadata"; then
+    echo "Expected error message not found in logs"
+    exit 1
+fi
+
 # delete postgres and graphql-engine
 docker compose down -v

packaging/cli-migrations/v3/README.md

@@ -71,3 +71,11 @@ Optional configuration for the server which boots during migrations.
 - `HASURA_GRAPHQL_MIGRATIONS_SERVER_TIMEOUT` (default=`30s`)
 
   Specify the server timeout threshold.
+
+### Disallow Inconsistent Metadata (Optional)
+
+- `HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA` (default=`false`)
+
+  If set to true, inconsistent metadata will be rejected, causing the entrypoint script to exit with a non-zero exit code.
+  Has the same behavior as the `--disallow-inconsistent-metadata` flag in the `hasura metadata apply` command documented 
+  [here](https://hasura.io/docs/latest/hasura-cli/commands/hasura_metadata_apply).

packaging/cli-migrations/v3/docker-entrypoint.sh

@@ -11,6 +11,7 @@ log() {
 
 DEFAULT_MIGRATIONS_DIR="/hasura-migrations"
 DEFAULT_METADATA_DIR="/hasura-metadata"
+DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG=""
 TEMP_PROJECT_DIR="/tmp/hasura-project"
 
 if [ -z ${HGE_BINARY+x} ]; then
@@ -67,19 +68,13 @@ if [ -z ${HASURA_GRAPHQL_METADATA_DIR+x} ]; then
     HASURA_GRAPHQL_METADATA_DIR="$DEFAULT_METADATA_DIR"
 fi
 
-# apply metadata if the directory exist
-if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
-    rm -rf "$TEMP_PROJECT_DIR"
-    log "migrations-apply" "applying metadata from $HASURA_GRAPHQL_METADATA_DIR"
-    mkdir -p "$TEMP_PROJECT_DIR"
-    cp -a "$HASURA_GRAPHQL_METADATA_DIR/." "$TEMP_PROJECT_DIR/metadata/"
-    cd "$TEMP_PROJECT_DIR"
-    echo "version: 3" > config.yaml
-    echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
-    echo "metadata_directory: metadata" >> config.yaml
-    hasura-cli metadata apply 
+# check if disallow metadata inconsistency is set to true (case insensitive), default otherwise
+if [ "$(echo "$HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is set to true, disallowing inconsistent metadata"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="--disallow-inconsistent-metadata"
 else
-    log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
+    log "migrations-startup" "env var HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA is not true, falling back to default CLI behavior"
+    HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA="$DEFAULT_DISALLOW_INCONSISTENT_METADATA_FLAG"
 fi
 
 # apply migrations if the directory exist
@@ -97,6 +92,21 @@ else
     log "migrations-apply" "directory $HASURA_GRAPHQL_MIGRATIONS_DIR does not exist, skipping migrations"
 fi
 
+# apply metadata if the directory exist
+if [ -d "$HASURA_GRAPHQL_METADATA_DIR" ]; then
+    rm -rf "$TEMP_PROJECT_DIR"
+    log "migrations-apply" "applying metadata from $HASURA_GRAPHQL_METADATA_DIR"
+    mkdir -p "$TEMP_PROJECT_DIR"
+    cp -a "$HASURA_GRAPHQL_METADATA_DIR/." "$TEMP_PROJECT_DIR/metadata/"
+    cd "$TEMP_PROJECT_DIR"
+    echo "version: 3" > config.yaml
+    echo "endpoint: http://localhost:$HASURA_GRAPHQL_MIGRATIONS_SERVER_PORT" >> config.yaml
+    echo "metadata_directory: metadata" >> config.yaml
+    hasura-cli metadata apply $HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA
+else
+    log "migrations-apply" "directory $HASURA_GRAPHQL_METADATA_DIR does not exist, skipping metadata"
+fi
+
 # kill graphql engine that we started earlier
 log "migrations-shutdown" "killing temporary server"
 kill $PID

packaging/cli-migrations/v3/test/bad_metadata/public_test.yaml

@@ -0,0 +1,10 @@
+# this file contains metadata that is intentionally inconsistent with the database, to test
+# the behavior of the entrypoint script when inconsistent metadata is detected.
+# the fake_table table does not exist.
+table:
+  name: test
+  schema: public
+object_relationships:
+  - name: fake_table
+    using:
+      foreign_key_constraint_on: test_row_id

packaging/cli-migrations/v3/test/docker-compose.yaml

@@ -16,4 +16,5 @@ services:
     environment:
       HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
       HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console
-      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_DISALLOW_INCONSISTENT_METADATA: "true"

packaging/cli-migrations/v3/test/test.sh

@@ -38,5 +38,27 @@ wait_for_server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/metadata | jq -j '.' | diff validation/metadata.json -
 # get list of migrations applied from graphql-engine server
 docker run --rm --network container:graphql-engine curlimages/curl -s -f -d'{"type" : "get_catalog_state", "args" : {} }' localhost:8080/v1/metadata | jq .cli_state | diff validation/catalog_cli_state.json -
+
+# stop the graphql-engine container
+docker compose stop graphql-engine
+
+# overwrite existing metadata with intentionally inconsistent metadata 
+docker cp bad_metadata/public_test.yaml graphql-engine:/hasura-metadata/databases/default/tables/public_test.yaml
+# start the container with new, inconsistent metadata
+docker compose up -d --no-recreate graphql-engine
+# confirm that the service does not become available for at least 10 seconds
+for _ in $(seq 1 10); do
+    if docker run --rm --network container:graphql-engine curlimages/curl -s -f http://127.0.0.1:8080/v1/version > /dev/null 2>&1; then
+        echo "Server became available when it should not have"
+        exit 1
+    fi
+    sleep 1
+done
+# confirm that the service's log contain the expected error
+if ! docker compose logs graphql-engine | grep -qi "error applying metadata"; then
+    echo "Expected error message not found in logs"
+    exit 1
+fi
+
 # delete postgres and graphql-engine
 docker compose down -v