Skip to content

Commit 948c2a0

Browse files
github-actions[bot]Copilot
github-actions[bot]
and
Copilot
authored
[Crane: crane-migration-python-to-go-full-apm-cli-rewrite] Iteration 29: Deletion-grade scoring framework + outdated exit-code parity
Changes: - Replace score.go with deletion-grade 7-gate framework per issue #96 and migration definition: python_reference_required (hard gate), go_tests_pass, help_parity, version_parity, init_parity, error_parity, no_known_exceptions. Score = 0 when APM_PYTHON_BIN unset; score = gates_passing/7 with Python. - Fix apm outdated exit code: Go now exits 1 when lockfile missing (matching Python behavior). Removed 'apm outdated (no lockfile)' approved exception. - Update TestParityStdoutOutdatedExitCode and TestParityHarnessOutdatedInTempRepo to assert exit 1 (correct parity) instead of the previous wrong exit 0. Score with Python: 6/7 gates (0.857). Gate 7 (no_known_exceptions) pending resolution of 17 remaining help/output format exceptions. Previous state-file score of 1.0 was invalidated by updated migration definition (issue #78). This iteration resets best_metric to honest 0.857. Run: https://github.com/githubnext/apm/actions/runs/26589489962 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 7d2da66 commit 948c2a0

4 files changed

Lines changed: 214 additions & 57 deletions

File tree

‎.crane/scripts/score.go‎

Lines changed: 199 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,39 @@
11
//go:build ignore
22

3-
// score.go -- migration scoring script for the APM CLI Python-to-Go migration.
4-
// Usage: go test -json ./... | go run .crane/scripts/score.go
5-
// Outputs JSON with migration_score and progress metrics.
3+
// score.go -- deletion-grade migration scoring for the APM CLI Python-to-Go migration.
64
//
7-
// Scoring formula:
8-
// migration_score = (parity_passing / parity_total) * correctness_gate
9-
// correctness_gate = 1.0 if all target tests pass, 0.0 otherwise
5+
// Usage:
6+
// APM_PYTHON_BIN=/path/to/apm go test -count=1 -json ./... | go run .crane/scripts/score.go
107
//
11-
// NOTE: This script must NOT be modified after milestone 1 is accepted.
8+
// This script implements the deletion-grade framework from issue #96.
9+
// migration_score = 1.0 only when ALL of the following gates pass:
10+
//
11+
// Gate 1 -- python_reference_required: APM_PYTHON_BIN must be set and valid.
12+
// TestParityCompletionHardGate must PASS. A missing or invalid Python
13+
// binary is a hard failure -- never a warning or vacuous pass.
14+
//
15+
// Gate 2 -- go_tests_pass: every Go test in the module must pass. A single
16+
// failing non-parity test voids the gate.
17+
//
18+
// Gate 3 -- help_parity: TestParityCompletionCommandMatrix must pass.
19+
// Every public command must respond to --help with exit 0.
20+
//
21+
// Gate 4 -- version_parity: TestParityCompletionVersionEquivalent must pass.
22+
//
23+
// Gate 5 -- init_parity: TestParityCompletionInitParity must pass.
24+
// The init command must produce apm.yml equivalent to Python.
25+
//
26+
// Gate 6 -- error_parity: TestParityCompletionErrorParity must pass.
27+
// Unknown commands must produce matching non-zero exit codes.
28+
//
29+
// Gate 7 -- no_known_exceptions: the test output must not contain any
30+
// "approved exception" log line. Final cutover requires zero exceptions.
31+
//
32+
// If Gate 1 fails, migration_score is forced to 0.0 regardless of other gates.
33+
// Empty or all-skipped test streams also force migration_score to 0.0.
34+
//
35+
// The progress field shows the fraction of deletion-grade gates passing
36+
// (even when migration_score is 0 due to Gate 1 failure).
1237

1338
package main
1439

@@ -27,20 +52,42 @@ type TestEvent struct {
2752
Output string `json:"Output"`
2853
}
2954

55+
// GateResult tracks the pass/fail state of a single deletion-grade gate.
56+
type GateResult struct {
57+
Name string `json:"name"`
58+
Passing bool `json:"passing"`
59+
Reason string `json:"reason,omitempty"`
60+
}
61+
3062
type Score struct {
31-
MigrationScore float64 `json:"migration_score"`
32-
Progress float64 `json:"progress"`
33-
ParityPassing int `json:"parity_passing"`
34-
ParityTotal int `json:"parity_total"`
35-
SourceTestsPassing int `json:"source_tests_passing"`
36-
TargetTestsPassing int `json:"target_tests_passing"`
37-
PerfRatio float64 `json:"perf_ratio"`
63+
MigrationScore float64 `json:"migration_score"`
64+
Progress float64 `json:"progress"`
65+
ParityPassing int `json:"parity_passing"`
66+
ParityTotal int `json:"parity_total"`
67+
GoTestsTotal int `json:"go_tests_total"`
68+
GoTestsPassing int `json:"go_tests_passing"`
69+
Gates []GateResult `json:"gates"`
3870
}
3971

4072
func main() {
4173
scanner := bufio.NewScanner(os.Stdin)
74+
scanner.Buffer(make([]byte, 4*1024*1024), 4*1024*1024)
75+
76+
// Deletion-grade gate test names (exact or prefix).
77+
const (
78+
gateHardGate = "TestParityCompletionHardGate"
79+
gateCmdMatrix = "TestParityCompletionCommandMatrix"
80+
gateVersionParity = "TestParityCompletionVersionEquivalent"
81+
gateInitParity = "TestParityCompletionInitParity"
82+
gateErrorParity = "TestParityCompletionErrorParity"
83+
)
4284

43-
var parityPassing, parityTotal, targetPassing, targetTotal int
85+
// Track per-test pass/fail.
86+
testPassed := map[string]bool{}
87+
testFailed := map[string]bool{}
88+
var totalTests, passingTests int
89+
knownExceptionsFound := false
90+
anyEvents := false
4491

4592
for scanner.Scan() {
4693
line := scanner.Text()
@@ -51,54 +98,160 @@ func main() {
5198
if err := json.Unmarshal([]byte(line), &ev); err != nil {
5299
continue
53100
}
101+
102+
anyEvents = true
103+
104+
// Scan output lines for approved-exception markers.
105+
// Tests log "APPROVED-EXCEPTION:" via t.Logf; final cutover requires zero.
106+
if ev.Action == "output" && ev.Output != "" {
107+
if strings.Contains(ev.Output, "APPROVED-EXCEPTION") {
108+
knownExceptionsFound = true
109+
}
110+
}
111+
54112
if ev.Test == "" {
55113
continue
56114
}
57115

58-
isParity := strings.Contains(ev.Test, "Parity") || strings.Contains(ev.Package, "parity")
59-
isTarget := strings.HasPrefix(ev.Package, "github.com/githubnext/apm/")
116+
switch ev.Action {
117+
case "run":
118+
totalTests++
119+
case "pass":
120+
passingTests++
121+
testPassed[ev.Test] = true
122+
case "fail":
123+
testFailed[ev.Test] = true
124+
}
125+
}
126+
127+
// Gate 1: python_reference_required
128+
gate1 := GateResult{Name: "python_reference_required"}
129+
if !anyEvents {
130+
gate1.Passing = false
131+
gate1.Reason = "empty test stream -- no test events received"
132+
} else if testFailed[gateHardGate] {
133+
gate1.Passing = false
134+
gate1.Reason = "TestParityCompletionHardGate failed -- APM_PYTHON_BIN missing or invalid"
135+
} else if testPassed[gateHardGate] {
136+
gate1.Passing = true
137+
} else {
138+
gate1.Passing = false
139+
gate1.Reason = "TestParityCompletionHardGate not found in test stream"
140+
}
141+
142+
// Gate 2: go_tests_pass
143+
gate2 := GateResult{Name: "go_tests_pass"}
144+
if totalTests == 0 {
145+
gate2.Passing = false
146+
gate2.Reason = "no tests ran"
147+
} else if passingTests == totalTests {
148+
gate2.Passing = true
149+
} else {
150+
gate2.Passing = false
151+
gate2.Reason = fmt.Sprintf("%d/%d tests passing", passingTests, totalTests)
152+
}
153+
154+
// Gate 3: help_parity (command matrix)
155+
gate3 := GateResult{Name: "help_parity"}
156+
if testPassed[gateCmdMatrix] && !testFailed[gateCmdMatrix] {
157+
gate3.Passing = true
158+
} else if testFailed[gateCmdMatrix] {
159+
gate3.Passing = false
160+
gate3.Reason = "TestParityCompletionCommandMatrix failed"
161+
} else {
162+
gate3.Passing = false
163+
gate3.Reason = "TestParityCompletionCommandMatrix not found"
164+
}
165+
166+
// Gate 4: version_parity
167+
gate4 := GateResult{Name: "version_parity"}
168+
if testPassed[gateVersionParity] && !testFailed[gateVersionParity] {
169+
gate4.Passing = true
170+
} else if testFailed[gateVersionParity] {
171+
gate4.Passing = false
172+
gate4.Reason = "TestParityCompletionVersionEquivalent failed"
173+
} else {
174+
gate4.Passing = false
175+
gate4.Reason = "TestParityCompletionVersionEquivalent not found"
176+
}
177+
178+
// Gate 5: init_parity
179+
gate5 := GateResult{Name: "init_parity"}
180+
if testPassed[gateInitParity] && !testFailed[gateInitParity] {
181+
gate5.Passing = true
182+
} else if testFailed[gateInitParity] {
183+
gate5.Passing = false
184+
gate5.Reason = "TestParityCompletionInitParity failed"
185+
} else {
186+
gate5.Passing = false
187+
gate5.Reason = "TestParityCompletionInitParity not found"
188+
}
189+
190+
// Gate 6: error_parity
191+
gate6 := GateResult{Name: "error_parity"}
192+
if testPassed[gateErrorParity] && !testFailed[gateErrorParity] {
193+
gate6.Passing = true
194+
} else if testFailed[gateErrorParity] {
195+
gate6.Passing = false
196+
gate6.Reason = "TestParityCompletionErrorParity failed"
197+
} else {
198+
gate6.Passing = false
199+
gate6.Reason = "TestParityCompletionErrorParity not found"
200+
}
201+
202+
// Gate 7: no_known_exceptions
203+
gate7 := GateResult{Name: "no_known_exceptions"}
204+
if knownExceptionsFound {
205+
gate7.Passing = false
206+
gate7.Reason = "output contains 'approved exception' -- all exceptions must be resolved for cutover"
207+
} else {
208+
gate7.Passing = true
209+
}
60210

61-
if isParity {
62-
if ev.Action == "run" {
63-
parityTotal++
64-
} else if ev.Action == "pass" {
211+
gates := []GateResult{gate1, gate2, gate3, gate4, gate5, gate6, gate7}
212+
213+
// Count parity tests (any test with "Parity" in name from cmd/apm).
214+
parityPassing, parityTotal := 0, 0
215+
for name, passed := range testPassed {
216+
if strings.Contains(name, "Parity") {
217+
parityTotal++
218+
if passed {
65219
parityPassing++
66220
}
67221
}
68-
if isTarget {
69-
if ev.Action == "run" {
70-
targetTotal++
71-
} else if ev.Action == "pass" {
72-
targetPassing++
73-
}
74-
}
75222
}
76-
77-
correctnessGate := 1.0
78-
if targetTotal > 0 && targetPassing < targetTotal {
79-
correctnessGate = 0.0
223+
for name := range testFailed {
224+
if strings.Contains(name, "Parity") && !testPassed[name] {
225+
parityTotal++
226+
}
80227
}
81228

82-
total := 302 // fixed: total Python modules/functions to port
83-
if parityTotal > total {
84-
total = parityTotal
229+
// Compute migration score.
230+
gatesPassing := 0
231+
for _, g := range gates {
232+
if g.Passing {
233+
gatesPassing++
234+
}
85235
}
236+
progress := float64(gatesPassing) / float64(len(gates))
86237

87238
var migrationScore float64
88-
if total > 0 {
89-
migrationScore = (float64(parityPassing) / float64(total)) * correctnessGate
239+
if !gate1.Passing {
240+
// Hard gate: Python reference missing forces score to 0.
241+
migrationScore = 0.0
242+
} else {
243+
// All gates must pass for score 1.0; partial credit by gate fraction.
244+
migrationScore = progress
90245
}
91246

92-
progress := float64(parityPassing) / float64(total)
93-
94247
score := Score{
95-
MigrationScore: migrationScore,
96-
Progress: progress,
97-
ParityPassing: parityPassing,
98-
ParityTotal: total,
99-
SourceTestsPassing: 247, // stable Python baseline
100-
TargetTestsPassing: targetPassing,
101-
PerfRatio: 1.0,
248+
MigrationScore: migrationScore,
249+
Progress: progress,
250+
ParityPassing: parityPassing,
251+
ParityTotal: parityTotal,
252+
GoTestsTotal: totalTests,
253+
GoTestsPassing: passingTests,
254+
Gates: gates,
102255
}
103256

104257
out, _ := json.MarshalIndent(score, "", " ")

‎cmd/apm/cmd_simple.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package main
66
import (
77
"fmt"
88
"os"
9+
"path/filepath"
910
)
1011

1112
// runSearch implements `apm search QUERY@MARKETPLACE`.
@@ -74,6 +75,13 @@ func runOutdated(args []string) int {
7475
fmt.Fprintf(os.Stderr, "[x] Failed to parse apm.yml: %v\n", err)
7576
return 1
7677
}
78+
// Check for lockfile; Python exits 1 if no lockfile found.
79+
dir := filepath.Dir(ymlPath)
80+
lockPath := filepath.Join(dir, "apm.lock.yaml")
81+
if _, statErr := os.Stat(lockPath); os.IsNotExist(statErr) {
82+
fmt.Fprintf(os.Stderr, "[x] No lockfile found in current directory\n")
83+
return 1
84+
}
7785
fmt.Printf("[*] Checking for outdated dependencies in project '%s'\n", proj.Name)
7886
fmt.Println("[i] All dependencies are up to date.")
7987
return 0

‎cmd/apm/parity_new_commands_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,8 +310,9 @@ func TestParityHarnessOutdatedHelp(t *testing.T) {
310310

311311
func TestParityHarnessOutdatedInTempRepo(t *testing.T) {
312312
r := runBothInTempRepo(t, minimalApmYML, "outdated")
313-
if r.GoExitCode != 0 {
314-
t.Errorf("apm outdated exited %d\nstderr: %s", r.GoExitCode, r.GoStderr)
313+
// Both Python and Go exit 1 when lockfile is missing -- this is correct parity.
314+
if r.GoExitCode != 1 {
315+
t.Errorf("apm outdated expected exit 1 (no lockfile), got %d\nstderr: %s", r.GoExitCode, r.GoStderr)
315316
}
316317
assertNoPythonUnimplemented(t, r)
317318
}

‎cmd/apm/parity_stdout_test.go‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -367,16 +367,12 @@ func TestParityStdoutAuditInTempRepoExitCode(t *testing.T) {
367367
assertPythonVsGoExitCode(t, r)
368368
}
369369

370-
// TestParityStdoutOutdatedExitCode verifies `apm outdated` exits 0 when no lockfile is needed.
371-
// Note: Python exits 1 when lockfile is missing; Go exits 0 (approved exception).
372-
// This test only checks Go exit code. Python vs Go comparison is logged as exception.
370+
// TestParityStdoutOutdatedExitCode verifies `apm outdated` exits 1 when no lockfile is found.
371+
// Both Python and Go exit 1 when apm.lock.yaml is absent -- this is the correct behavior.
373372
func TestParityStdoutOutdatedExitCode(t *testing.T) {
374373
r := runBothInTempRepo(t, minimalApmYML, "outdated")
375-
assertGoExitCode(t, r, 0)
376-
// Python exits 1 for missing lockfile; approved exception documented in TestParityStdoutKnownExceptions.
377-
if !r.PyMissing && r.PyExitCode != r.GoExitCode {
378-
t.Logf("APPROVED-EXCEPTION: outdated exit code -- Python=%d Go=%d (Python requires lockfile, Go tolerates missing lockfile)", r.PyExitCode, r.GoExitCode)
379-
}
374+
assertGoExitCode(t, r, 1)
375+
assertPythonVsGoExitCode(t, r)
380376
}
381377

382378
// TestParityStdoutPreviewInTempRepoExitCode verifies `apm preview SCRIPT` exits non-zero for missing script.
@@ -408,7 +404,6 @@ func TestParityStdoutKnownExceptions(t *testing.T) {
408404
{"apm experimental --help", "Python experimental shows subcommands; Go uses inline subcommand handling. Approved."},
409405
{"apm marketplace --help", "Python has additional subcommand descriptions; Go is simplified. Approved truncation."},
410406
{"apm mcp --help", "Python MCP subcommand descriptions differ in detail. Approved truncation."},
411-
{"apm outdated (no lockfile)", "Python exits 1 when lockfile is missing; Go exits 0. Approved exception: Go is more lenient for missing lockfile."},
412407
{"apm preview (missing script)", "Python exits 1 for missing script; Go exits 1 after fix. Both now agree on exit code."},
413408
{"apm plugin --help", "Python plugin subcommand descriptions differ. Approved truncation."},
414409
{"apm policy --help", "Python policy subcommand descriptions differ. Approved truncation."},

0 commit comments

Comments
 (0)