DO NOT MERGE: focused Trino perf run for Hoglake changes

tests/mw-dev/run.sh

@@ -65,7 +65,9 @@ else
 fi
 TRINO_TLS_PASSWORD="${TRINO_TLS_PASSWORD:-duckgres-e2e-keystore}"
 # Managed admission requires atomic-table-creation-v1, including frozen perf.
-HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:fcd2bdc2b17cbe7bdf4b52b19ebe1c901c853c5ec925cf52a94609e464e96a02}"
+# Targeted run (do not merge): a hoglake build with /scan column_stats
+# (PostHog/hoglake#189, main 27a705ed), so the connector can prune files.
+HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:12d713e688902ea3f46aeee7266139abdc7eeb30bb52f22e67d03c1ad9f65b76}"
 HOGLAKE_DATA_PATH="${HOGLAKE_DATA_PATH:-}"
 if [ "${GITHUB_ACTIONS:-}" = true ] && [ -n "$HOGLAKE_DATA_PATH" ]; then
   hoglake_bucket="${HOGLAKE_DATA_PATH#s3://}"

tests/mw-dev/scenario/runner_test.go

@@ -525,6 +525,7 @@ func TestFrozenPerfScenarioUsesSupportedStepsAndOriginalCatalog(t *testing.T) {
 }
 
 func TestFrozenPerfScenarioRunsOptionalPropertiesAfterOriginalPerf(t *testing.T) {
+	t.Skip("targeted Hoglake footer-cache run drops the properties comparison")
 	t.Setenv("DUCKGRES_SCENARIO_PROPERTIES_S3_URI", "s3://example/properties/")
 	t.Setenv("DUCKGRES_SCENARIO_HOGLAKE_URI", "http://hoglake:8080")
 	t.Setenv("DUCKGRES_SCENARIO_FROZEN_S3_URI", "s3://example-frozen/frozen_v1/")
@@ -715,8 +716,9 @@ func assertPerfTargetsOnlyPGWire(t *testing.T, step core.Step) {
 func assertPerfTargetsPGWireTrinoAndAthena(t *testing.T, step core.Step) {
 	t.Helper()
 	targets, ok := step.With["targets"].([]any)
-	if !ok || len(targets) != 5 || targets[0] != "pgwire_uncached" || targets[1] != "pgwire_cached" || targets[2] != "trino" || targets[3] != "trino_cached" || targets[4] != "athena" {
-		t.Fatalf("perf step %s targets = %#v, want [pgwire_uncached pgwire_cached trino trino_cached athena]", step.ID, step.With["targets"])
+	// Targeted Hoglake footer-cache run: Trino targets only.
+	if !ok || len(targets) != 2 || targets[0] != "trino" || targets[1] != "trino_cached" {
+		t.Fatalf("perf step %s targets = %#v, want [trino trino_cached]", step.ID, step.With["targets"])
 	}
 }
 

tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml

@@ -98,7 +98,7 @@ steps:
       org_id: ${env:DUCKGRES_SCENARIO_ORG_ID}
       catalog: ducklake
       catalog_file: ../../../perf/queries/ducklake_posthog_tables.yaml
-      targets: [pgwire_uncached, pgwire_cached, trino, trino_cached, athena]
+      targets: [trino, trino_cached]
       trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen
       trino_ca_cert_file: ${env:DUCKGRES_SCENARIO_TRINO_CA_CERT}
       trino_startup_timeout: 2m
@@ -116,36 +116,6 @@ steps:
       worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST}
       worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST}
 
-  - id: properties_comparison
-    type: properties_comparison
-    depends_on: [perf_queries]
-    with:
-      org_id: ${env:DUCKGRES_SCENARIO_ORG_ID}
-      catalog: ducklake
-      hoglake_uri: ${env:DUCKGRES_SCENARIO_HOGLAKE_URI}
-      hoglake_file: ../perf/setup_hoglake.py
-      targets: [pgwire_uncached, pgwire_cached, trino, trino_cached, athena]
-      trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen
-      hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen
-      trino_schema: properties_perf
-      trino_ca_cert_file: ${env:DUCKGRES_SCENARIO_TRINO_CA_CERT}
-      trino_startup_timeout: 2m
-      trino_startup_poll_interval: 2s
-      athena_region: ${env:DUCKGRES_SCENARIO_ATHENA_REGION}
-      athena_workgroup: ${env:DUCKGRES_SCENARIO_ATHENA_WORKGROUP}
-      athena_catalog: AwsDataCatalog
-      athena_database: ${env:DUCKGRES_SCENARIO_ATHENA_DATABASE}
-      athena_output_location: ${env:DUCKGRES_SCENARIO_ATHENA_RESULTS_S3_URI}
-      athena_poll_interval: 500ms
-      athena_query_timeout: 30m
-      run_id: ${run_id}
-      # Same dataset as perf_queries: both suites of one nightly publish together.
-      dataset_version: *nightly_dataset
-      output_subdir: perf-properties
-      fail_on_query_errors: true
-      worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST}
-      worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST}
-
 # Managed Hoglake retains tenant ownership and rejects public deprovision.
 # The workflow's always-run isolated-stack teardown owns cleanup of the
 # Duckling, namespace, metadata store and dedicated managed storage prefix.

tests/perf/queries/ducklake_posthog_tables.yaml

@@ -2,14 +2,13 @@ name: posthog-frozen-ducklake-golden-v5
 description: Identical query shapes with protocol-isolated execution over frozen Parquet files registered as DuckLake tables on PGWire, Hoglake tables on Trino, and Athena external tables.
 seed: 42
 dataset_scale: 1
+# Targeted Hoglake connector run (do not merge): every nightly query, on the
+# two Trino targets only, with fewer measured iterations than the nightly.
 targets:
-  - pgwire_uncached
-  - pgwire_cached
   - trino
   - trino_cached
-  - athena
 warmup_iterations: 1
-measure_iterations: 4
+measure_iterations: 2
 
 relation_variants:
   ducklake_table:
@@ -18,9 +17,6 @@ relation_variants:
   hoglake_table:
     events: posthog.events
     persons: posthog.persons
-  athena_external:
-    events: events
-    persons: persons
 
 paired_queries:
   - query_id_base: q_events_total_v5