Skip to content

Commit abcc103

Browse files
committed
fix: provision frozen perf scenarios with managed Hoglake
1 parent 8424729 commit abcc103

10 files changed

Lines changed: 186 additions & 43 deletions

File tree

‎.github/workflows/scenario-dev.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ jobs:
104104
go-version-file: go.mod
105105

106106
- name: Test scenario workflow scripts
107-
run: go test -count=1 ./tests/mw-dev/scenario ./tests/mw-dev ./tests/perf/publishercli
107+
run: go test -count=1 ./tests/mw-dev/scenario/... ./tests/mw-dev ./tests/perf/publishercli
108108

109109
- name: Configure AWS credentials (OIDC)
110110
uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2
@@ -115,6 +115,10 @@ jobs:
115115
# collect artifacts and tear down the isolated namespace.
116116
role-duration-seconds: 16200
117117

118+
- name: Discover managed Hoglake configuration
119+
if: env.E2E_SUITE == 'trino'
120+
run: bash tests/mw-dev/discover-hoglake.sh
121+
118122
- name: Connect to Tailscale
119123
uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2
120124
with:

‎tests/mw-dev/README.md‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,7 +1064,21 @@ for fixture writers to terminate and deletes only the numeric PR's exact prefix.
10641064
The initialized metadata-loss protection has unit regression coverage; this
10651065
lane does not corrupt the server database to simulate metadata loss.
10661066

1067-
The frozen performance workflow remains a separate, deferred migration. Its
1068-
external read-only catalog bootstrap is not proof of managed Hoglake onboarding
1069-
and must be adapted independently before relying on it with the new backend
1070-
ownership rules. This change does not copy or rewrite the frozen dataset.
1067+
The frozen performance workflow also discovers the dedicated managed storage
1068+
base and configures managed Hoglake admission. It uses the same Hoglake server
1069+
pin, which supports atomic table creation, but keeps the frozen-data read-only
1070+
Pod Identity for fixture reads. The control plane owns the tenant's managed
1071+
catalog; the importer registers immutable Parquet in a separate `<org>-frozen`
1072+
catalog (and `<org>-properties` for the optional suite). The isolated benchmark
1073+
selector uses the read-only Pod Identity instead of assuming the managed tenant
1074+
storage role. It does not copy, rewrite or grant writes to the frozen dataset.
1075+
1076+
For local frozen runs, supply `HOGLAKE_DATA_PATH` with the dedicated
1077+
`s3://<bucket>/trino/` base before deployment. Missing configuration fails before
1078+
namespace mutation. Managed Hoglake rejects public deprovision by design, so the
1079+
frozen scenario leaves cleanup to the workflow's always-run `run.sh teardown`.
1080+
Direct/local scenario invocations must run that teardown even after failures;
1081+
it removes the isolated Duckling and namespace before cleaning the scoped managed
1082+
storage prefix. Do not use the frozen source prefix as `HOGLAKE_DATA_PATH`.
1083+
Collect `run.sh diagnostics` and preserve scenario artifacts before teardown, then
1084+
redeploy a fresh isolated stack before retrying.

‎tests/mw-dev/run.sh‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,8 @@ else
6161
TRINO_IMAGE="${TRINO_IMAGE:-ghcr.io/posthog/trino:86468a7955788b90fe2072f80d86d548972ff28b@sha256:64927a71d2870802a56b671828c6052e7aa37317a7c3a50bd50a93960402d67b}"
6262
fi
6363
TRINO_TLS_PASSWORD="${TRINO_TLS_PASSWORD:-duckgres-e2e-keystore}"
64-
if [ "$SCENARIO_NAME" = posthog_frozen_perf ]; then
65-
HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:f10c34f9c779e2794fca662d5302f97dc26e48a6b2a601ae344cad945e70483c}"
66-
else
67-
HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:fcd2bdc2b17cbe7bdf4b52b19ebe1c901c853c5ec925cf52a94609e464e96a02}"
68-
fi
64+
# Managed admission requires atomic-table-creation-v1, including frozen perf.
65+
HOGLAKE_IMAGE="${HOGLAKE_IMAGE:-ghcr.io/posthog/hoglake-server@sha256:fcd2bdc2b17cbe7bdf4b52b19ebe1c901c853c5ec925cf52a94609e464e96a02}"
6966
HOGLAKE_DATA_PATH="${HOGLAKE_DATA_PATH:-}"
7067
if [ "${GITHUB_ACTIONS:-}" = true ] && [ -n "$HOGLAKE_DATA_PATH" ]; then
7168
hoglake_bucket="${HOGLAKE_DATA_PATH#s3://}"
@@ -542,8 +539,10 @@ cleanup_hoglake_storage() {
542539
# END isolated Hoglake cleanup
543540

544541
cmd_deploy() {
545-
if [ "$E2E_SUITE" = trino ] && ! hoglake_perf_enabled; then
546-
: "${HOGLAKE_CI_POD_IDENTITY_ROLE:?Dedicated Hoglake CI role is required}"
542+
if [ "$E2E_SUITE" = trino ]; then
543+
if ! hoglake_perf_enabled; then
544+
: "${HOGLAKE_CI_POD_IDENTITY_ROLE:?Dedicated Hoglake CI role is required}"
545+
fi
547546
[[ "$HOGLAKE_DATA_PATH" =~ ^s3://[a-z0-9][a-z0-9.-]+/trino/$ ]] || { echo "HOGLAKE_DATA_PATH must be the configured dedicated bucket/trino/ base" >&2; return 1; }
548547
fi
549548
reset_pr_stack
@@ -577,13 +576,10 @@ cmd_deploy() {
577576
sleep 15
578577
"${KUBECTL[@]}" -n "$NS" patch deployment duckgres-hoglake --type=merge -p '{"spec":{"replicas":1}}'
579578
"${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-hoglake --timeout=300s
580-
if hoglake_perf_enabled; then
581-
"${KUBECTL[@]}" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \
582-
"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"controlplane\",\"env\":[{\"name\":\"DUCKGRES_TRINO_HOGLAKE_URI\",\"value\":\"http://duckgres-hoglake.$NS.svc:8080\"}]}]}}}}"
583-
else
584-
patch="$(jq -cn --arg uri "http://duckgres-hoglake.$NS.svc:8080" --arg path "$HOGLAKE_DATA_PATH" '{spec:{template:{spec:{containers:[{name:"controlplane",env:[{name:"DUCKGRES_TRINO_MANAGED_HOGLAKE_URI",value:$uri},{name:"DUCKGRES_TRINO_HOGLAKE_DATA_PATH",value:$path},{name:"DUCKGRES_TRINO_HOGLAKE_NAMESPACE",value:"main"}]}]}}}}')"
585-
"${KUBECTL[@]}" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p "$patch"
586-
fi
579+
# Fresh tenants use managed Hoglake admission in every Trino lane.
580+
# Frozen fixture catalogs are imported separately and keep read-only identity.
581+
patch="$(jq -cn --arg uri "http://duckgres-hoglake.$NS.svc:8080" --arg path "$HOGLAKE_DATA_PATH" '{spec:{template:{spec:{containers:[{name:"controlplane",env:[{name:"DUCKGRES_TRINO_MANAGED_HOGLAKE_URI",value:$uri},{name:"DUCKGRES_TRINO_HOGLAKE_DATA_PATH",value:$path},{name:"DUCKGRES_TRINO_HOGLAKE_NAMESPACE",value:"main"}]}]}}}}')"
582+
"${KUBECTL[@]}" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p "$patch"
587583
if trino_multicell_enabled; then
588584
NS="$TRINO_CELL_NS" ensure_trino_pod_identity
589585
"${KUBECTL[@]}" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \

‎tests/mw-dev/scenario/core/catalog_test.go‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ steps:
213213
}
214214
}
215215

216-
func TestFrozenPerfScenarioCombinesAllTargetsAndCleanup(t *testing.T) {
216+
func TestFrozenPerfScenarioCombinesAllTargets(t *testing.T) {
217217
for _, tc := range []struct {
218218
file string
219219
targets []string
@@ -246,10 +246,6 @@ func TestFrozenPerfScenarioCombinesAllTargetsAndCleanup(t *testing.T) {
246246
if !found {
247247
t.Fatal("missing perf queries")
248248
}
249-
last := scenario.Steps[len(scenario.Steps)-1]
250-
if last.ID != "deprovision" || !last.AlwaysRun {
251-
t.Fatal("missing unconditional cleanup")
252-
}
253249
})
254250
}
255251
}

‎tests/mw-dev/scenario/perf/hoglake_catalog.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@ import (
1616
trinodriver "github.com/posthog/duckgres/tests/perf/drivers/trino"
1717
)
1818

19-
// selectHoglakeCatalog switches only the backend dataset of the disposable
20-
// tenant catalog. Its name, principal, credentials, cache mode and OPA grant
21-
// remain identical. Callers select the original dataset explicitly on retries.
19+
// selectHoglakeCatalog switches the disposable tenant catalog to a fixture
20+
// dataset using the read-only storage Pod Identity. Tenant authentication,
21+
// cache mode and OPA grant remain identical. Callers select the original
22+
// dataset explicitly on retries.
2223
func (f defaultDriverFactory) selectHoglakeCatalog(ctx context.Context, connection trinodriver.ConnectionConfig) error {
2324
if f.trinoCatalogStoreDSN == "" || connection.CatalogStoreCellID == "" {
2425
return errors.New("dataset selection requires an explicit isolated benchmark catalog store and cell")
@@ -44,11 +45,10 @@ func (f defaultDriverFactory) selectHoglakeCatalog(ctx context.Context, connecti
4445
if original == nil {
4546
return errors.New("no benchmark catalog in the configured catalog-store cell; verify DUCKGRES_SCENARIO_TRINO_CATALOG_STORE_CELL_ID (not the public API cell ID)")
4647
}
47-
if original["connector.name"] != "hoglake" || original["hoglake.catalog"] == "" || original["hoglake.uri"] == "" || original["fs.cache.enabled"] != "false" {
48-
return errors.New("dataset selection requires an existing uncached Hoglake benchmark catalog")
48+
desired, err := fixtureHoglakeProperties(original, connection.HoglakeCatalog)
49+
if err != nil {
50+
return err
4951
}
50-
desired := maps.Clone(original)
51-
desired["hoglake.catalog"] = connection.HoglakeCatalog
5252
if maps.Equal(original, desired) {
5353
return waitHoglakeSchema(ctx, connection)
5454
}
@@ -121,6 +121,21 @@ func (f defaultDriverFactory) selectHoglakeCatalog(ctx context.Context, connecti
121121
}
122122
}
123123

124+
func fixtureHoglakeProperties(original map[string]string, catalog string) (map[string]string, error) {
125+
if original["connector.name"] != "hoglake" || original["hoglake.catalog"] == "" || original["hoglake.uri"] == "" || original["fs.cache.enabled"] != "false" || original["s3.auth-type"] != "IAM_ROLE" {
126+
return nil, errors.New("dataset selection requires an existing uncached Hoglake benchmark catalog using IAM_ROLE authentication")
127+
}
128+
if original["s3.iam-role"] != "" && original["hoglake.catalog"] == catalog {
129+
return nil, errors.New("read-only benchmark fixtures require a separate Hoglake catalog from the managed tenant")
130+
}
131+
desired := maps.Clone(original)
132+
desired["hoglake.catalog"] = catalog
133+
// Managed onboarding assumes a tenant's writable storage role. Immutable
134+
// benchmark fixtures are read through the isolated Trino Pod Identity instead.
135+
delete(desired, "s3.iam-role")
136+
return desired, nil
137+
}
138+
124139
// Metadata access establishes that the connector loaded the selected dataset;
125140
// SELECT 1 only establishes that the coordinator accepts SQL.
126141
func waitHoglakeSchema(ctx context.Context, connection trinodriver.ConnectionConfig) error {
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package perf
2+
3+
import (
4+
"context"
5+
"maps"
6+
"strings"
7+
"testing"
8+
9+
trinodriver "github.com/posthog/duckgres/tests/perf/drivers/trino"
10+
)
11+
12+
func TestFixtureHoglakeCatalogUsesPodIdentity(t *testing.T) {
13+
managed := map[string]string{
14+
"connector.name": "hoglake", "hoglake.uri": "http://hoglake.example:8080",
15+
"hoglake.catalog": "org-fixture", "fs.cache.enabled": "false",
16+
"fs.s3.enabled": "true", "s3.auth-type": "IAM_ROLE",
17+
"s3.iam-role": "arn:aws:iam::123456789012:role/tenant-fixture",
18+
"s3.region": "us-east-1", "s3.max-connections": "500",
19+
}
20+
before := maps.Clone(managed)
21+
for _, catalog := range []string{"org-fixture-frozen", "org-fixture-properties", "org-fixture-frozen"} {
22+
selected, err := fixtureHoglakeProperties(managed, catalog)
23+
if err != nil {
24+
t.Fatal(err)
25+
}
26+
want := maps.Clone(before)
27+
delete(want, "s3.iam-role")
28+
want["hoglake.catalog"] = catalog
29+
if !maps.Equal(selected, want) {
30+
t.Fatalf("fixture properties = %v, want %v", selected, want)
31+
}
32+
if !maps.Equal(managed, before) {
33+
t.Fatal("mutated source catalog properties")
34+
}
35+
managed, before = selected, maps.Clone(selected)
36+
}
37+
}
38+
39+
func TestFixtureHoglakeCatalogRejectsInvalidBaseline(t *testing.T) {
40+
for _, change := range []map[string]string{
41+
{"connector.name": "ducklake"}, {"hoglake.uri": ""}, {"hoglake.catalog": ""},
42+
{"fs.cache.enabled": "true"}, {"s3.auth-type": "STATIC"},
43+
} {
44+
original := map[string]string{"connector.name": "hoglake", "hoglake.uri": "http://hoglake.example:8080", "hoglake.catalog": "org-fixture", "fs.cache.enabled": "false", "s3.auth-type": "IAM_ROLE"}
45+
maps.Copy(original, change)
46+
if _, err := fixtureHoglakeProperties(original, "org-fixture-frozen"); err == nil {
47+
t.Fatalf("accepted invalid baseline %v", change)
48+
}
49+
}
50+
}
51+
52+
func TestFixtureHoglakeCatalogRejectsManagedDataset(t *testing.T) {
53+
original := map[string]string{"connector.name": "hoglake", "hoglake.uri": "http://hoglake.example:8080", "hoglake.catalog": "org-fixture", "fs.cache.enabled": "false", "s3.auth-type": "IAM_ROLE", "s3.iam-role": "tenant-role"}
54+
if _, err := fixtureHoglakeProperties(original, "org-fixture"); err == nil {
55+
t.Fatal("accepted managed dataset as a read-only fixture")
56+
}
57+
}
58+
59+
func TestSelectHoglakeCatalogRequiresIsolatedStore(t *testing.T) {
60+
for _, factory := range []defaultDriverFactory{{}, {trinoCatalogStoreDSN: "postgres://fixture.example/db"}} {
61+
err := factory.selectHoglakeCatalog(context.Background(), trinodriver.ConnectionConfig{Catalog: "org_fixture", HoglakeCatalog: "org-fixture-frozen"})
62+
if err == nil || !strings.Contains(err.Error(), "explicit isolated benchmark catalog store and cell") {
63+
t.Fatalf("got %v, want isolated store guard", err)
64+
}
65+
}
66+
}

‎tests/mw-dev/scenario/runner_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1158,3 +1158,25 @@ func containsTemplate(value any) bool {
11581158
}
11591159
return false
11601160
}
1161+
1162+
func TestFrozenPerfSeparatesManagedAndImmutableCatalogs(t *testing.T) {
1163+
scenario, err := core.LoadScenario(filepath.Join("scenarios", "posthog_frozen_perf.yaml"))
1164+
if err != nil {
1165+
t.Fatal(err)
1166+
}
1167+
var fixture, selected string
1168+
for _, step := range scenario.Steps {
1169+
switch step.ID {
1170+
case "setup_hoglake":
1171+
fixture, _ = step.With["hoglake_catalog"].(string)
1172+
case "perf_queries":
1173+
selected, _ = step.With["trino_hoglake_catalog"].(string)
1174+
}
1175+
if step.Type == "deprovision_warehouse" {
1176+
t.Fatal("managed ownership requires isolated workflow teardown, not public deprovision")
1177+
}
1178+
}
1179+
if fixture != "${env:DUCKGRES_SCENARIO_ORG_ID}-frozen" || selected != fixture {
1180+
t.Fatalf("immutable fixture and selection must share separate catalog: %q %q", fixture, selected)
1181+
}
1182+
}

‎tests/mw-dev/scenario/scenarios/posthog_frozen_perf.yaml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ steps:
8686
org_id: ${env:DUCKGRES_SCENARIO_ORG_ID}
8787
uri: ${env:DUCKGRES_SCENARIO_HOGLAKE_URI}
8888
source: ${env:DUCKGRES_SCENARIO_FROZEN_S3_URI}
89+
hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen
8990
file: ../perf/setup_hoglake.py
9091

9192
- id: perf_queries
@@ -96,7 +97,7 @@ steps:
9697
catalog: ducklake
9798
catalog_file: ../../../perf/queries/ducklake_posthog_tables.yaml
9899
targets: [pgwire_uncached, pgwire_cached, trino, trino_cached, athena]
99-
trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}
100+
trino_hoglake_catalog: ${env:DUCKGRES_SCENARIO_ORG_ID}-frozen
100101
trino_ca_cert_file: ${env:DUCKGRES_SCENARIO_TRINO_CA_CERT}
101102
trino_startup_timeout: 2m
102103
trino_startup_poll_interval: 2s
@@ -141,12 +142,6 @@ steps:
141142
worker_cpu: ${env:DUCKGRES_K8S_WORKER_CPU_REQUEST}
142143
worker_memory: ${env:DUCKGRES_K8S_WORKER_MEMORY_REQUEST}
143144

144-
- id: deprovision
145-
type: deprovision_warehouse
146-
depends_on: [properties_comparison]
147-
always_run: true
148-
with:
149-
org_id: ${env:DUCKGRES_SCENARIO_ORG_ID}
150-
verify_deleted: true
151-
cleanup_timeout: 15m
152-
poll_interval: 10s
145+
# Managed Hoglake retains tenant ownership and rejects public deprovision.
146+
# The workflow's always-run isolated-stack teardown owns cleanup of the
147+
# Duckling, namespace, metadata store and dedicated managed storage prefix.

‎tests/mw-dev/scenario/script_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,15 +114,16 @@ func TestDevScenarioWorkflowUsesUnifiedMwDevHarness(t *testing.T) {
114114
"scenario_summary.md",
115115
"$GITHUB_STEP_SUMMARY",
116116
"tests/mw-dev/run.sh diagnostics",
117-
"tests/mw-dev/run.sh teardown",
117+
"- name: Teardown\n if: always()\n run: tests/mw-dev/run.sh teardown",
118+
"- name: Discover managed Hoglake configuration\n if: env.E2E_SUITE == 'trino'\n run: bash tests/mw-dev/discover-hoglake.sh",
118119
"- name: Publish scenario perf results",
119120
"github.ref == 'refs/heads/main'",
120121
"MW_DEV_SCENARIO_PERF_SECRET_ID: ${{ vars.MW_DEV_SCENARIO_PERF_SECRET_ID }}",
121122
"aws secretsmanager get-secret-value",
122123
"go run ./cmd/duckgres-perf-publisher",
123124
"--connection-secret-stdin",
124125
"--schema duckgres_scenario_perf",
125-
"go test -count=1 ./tests/mw-dev/scenario ./tests/mw-dev ./tests/perf/publishercli",
126+
"go test -count=1 ./tests/mw-dev/scenario/... ./tests/mw-dev ./tests/perf/publishercli",
126127
} {
127128
if !strings.Contains(workflow, required) {
128129
t.Fatalf("workflow missing %q", required)

‎tests/mw-dev/trino_hoglake_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package e2emwdev_test
33
import (
44
"os"
55
"os/exec"
6+
"path/filepath"
67
"strings"
78
"testing"
89
)
@@ -133,3 +134,36 @@ fi
133134
})
134135
}
135136
}
137+
138+
func TestFrozenPerfDeployConfiguresManagedAdmission(t *testing.T) {
139+
f := newRunSHFakes(t)
140+
for _, name := range []string{"duckgres-ci-trino-ca.crt", "duckgres-ci-trino-server.p12"} {
141+
if err := os.WriteFile(filepath.Join(filepath.Dir(f.binDir), "secrets", name), []byte("fixture TLS"), 0600); err != nil {
142+
t.Fatal(err)
143+
}
144+
}
145+
out, err := runSHCommand(t, f.binDir, "deploy", "E2E_SUITE=trino", "SCENARIO_NAME=posthog_frozen_perf", "SCENARIO_POD_IDENTITY_ROLE=arn:aws:iam::123456789012:role/athena-perf", "TRINO_POD_IDENTITY_ROLE=arn:aws:iam::123456789012:role/trino-perf", "SCENARIO_DEV_ALLOW_DUCKLING_DELETE=1").CombinedOutput()
146+
if err != nil {
147+
t.Fatalf("deploy: %v %s", err, out)
148+
}
149+
calls := f.calls(t)
150+
for _, want := range []string{"DUCKGRES_TRINO_MANAGED_HOGLAKE_URI", "DUCKGRES_TRINO_HOGLAKE_DATA_PATH", "s3://example-hoglake/trino/"} {
151+
if !strings.Contains(calls, want) {
152+
t.Errorf("frozen deployment missing %s", want)
153+
}
154+
}
155+
if strings.Contains(calls, `"name":"DUCKGRES_TRINO_HOGLAKE_URI"`) {
156+
t.Fatal("deprecated benchmark switch cannot admit a new tenant")
157+
}
158+
}
159+
160+
func TestFrozenPerfMissingManagedPathFailsBeforeMutation(t *testing.T) {
161+
f := newRunSHFakes(t)
162+
out, err := runSHCommand(t, f.binDir, "deploy", "E2E_SUITE=trino", "SCENARIO_NAME=posthog_frozen_perf", "HOGLAKE_DATA_PATH=", "SCENARIO_DEV_ALLOW_DUCKLING_DELETE=1").CombinedOutput()
163+
if err == nil {
164+
t.Fatalf("accepted missing managed path: %s", out)
165+
}
166+
if calls := f.calls(t); strings.Contains(calls, "kubectl") || strings.Contains(calls, "aws") {
167+
t.Fatal("missing prerequisite mutated existing fixture")
168+
}
169+
}

0 commit comments

Comments
 (0)