From f568f3db4e0d8581833c75bfcf807029ff2e5baf Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Sat, 17 May 2025 07:26:55 -0400 Subject: [PATCH 1/5] Filter non network tests --- pkg/test/ginkgo/cmd_runsuite.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index 93827f08a788..60be150cfd05 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "math/rand" "os" "os/signal" "path/filepath" @@ -17,7 +16,6 @@ import ( "syscall" "time" - "github.com/onsi/ginkgo/v2" configv1 "github.com/openshift/api/config/v1" "github.com/sirupsen/logrus" "github.com/spf13/pflag" @@ -246,15 +244,29 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, junitSuiteName string, mon // this ensures the tests are always run in random order to avoid // any intra-tests dependencies - suiteConfig, _ := ginkgo.GinkgoConfiguration() - r := rand.New(rand.NewSource(suiteConfig.RandomSeed)) - r.Shuffle(len(tests), func(i, j int) { tests[i], tests[j] = tests[j], tests[i] }) + //suiteConfig, _ := ginkgo.GinkgoConfiguration() + //r := rand.New(rand.NewSource(suiteConfig.RandomSeed)) + //r.Shuffle(len(tests), func(i, j int) { tests[i], tests[j] = tests[j], tests[i] }) tests = suite.Filter(tests) if len(tests) == 0 { return fmt.Errorf("suite %q does not contain any tests", suite.Name) } + filteredTests := []*testCase{} + for _, test := range tests { + checkName := strings.ToLower(test.name) + if strings.Contains(checkName, "network") { + filteredTests = append(filteredTests, test) + } + } + + tests = filteredTests + + if len(tests) == 0 { + return fmt.Errorf("Filtered suite %q does not contain any tests", suite.Name) + } + logrus.Infof("Found %d filtered tests", len(tests)) count := o.Count From cd83749cb5f5be8533225489459145ec2c4a9c33 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Sat, 17 May 2025 07:29:58 -0400 Subject: [PATCH 2/5] Filter network tests --- pkg/test/ginkgo/cmd_runsuite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index 60be150cfd05..51128122b4c2 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -256,7 +256,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, junitSuiteName string, mon filteredTests := []*testCase{} for _, test := range tests { checkName := strings.ToLower(test.name) - if strings.Contains(checkName, "network") { + if !strings.Contains(checkName, "network") { filteredTests = append(filteredTests, test) } } From 2741c98d3b2211158976fee94015ed1d32ea118c Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Sat, 17 May 2025 18:40:23 -0400 Subject: [PATCH 3/5] Filter network tests --- pkg/test/ginkgo/cmd_runsuite.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index 51128122b4c2..b9f4e1401b4b 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -253,16 +253,17 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, junitSuiteName string, mon return fmt.Errorf("suite %q does not contain any tests", suite.Name) } - filteredTests := []*testCase{} - for _, test := range tests { - checkName := strings.ToLower(test.name) - if !strings.Contains(checkName, "network") { - filteredTests = append(filteredTests, test) + if strings.Contains(suite.Name, "conformance") { + filteredTests := []*testCase{} + for _, test := range tests { + checkName := strings.ToLower(test.name) + if !strings.Contains(checkName, "network") { + filteredTests = append(filteredTests, test) + } } + tests = filteredTests } - tests = filteredTests - if len(tests) == 0 { return fmt.Errorf("Filtered suite %q does not contain any tests", suite.Name) } From e5391966c1a4cbbf67127f8997b188584609e4e5 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Sun, 18 May 2025 07:46:29 -0400 Subject: [PATCH 4/5] Skip retries --- pkg/test/ginkgo/cmd_runsuite.go | 131 ++++++++++++++++---------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index b9f4e1401b4b..e8922bab1d6e 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -10,7 +10,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strings" "sync" "syscall" @@ -513,71 +512,71 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, junitSuiteName string, mon pass, fail, skip, failing := summarizeTests(tests) // attempt to retry failures to do flake detection - if fail > 0 && fail <= suite.MaximumAllowedFlakes { - var retries []*testCase - - // Make a copy of the all failing tests (subject to the max allowed flakes) so we can have - // a list of tests to retry. - for _, test := range failing { - retry := test.Retry() - retries = append(retries, retry) - if len(retries) > suite.MaximumAllowedFlakes { - break - } - } - - logrus.Warningf("Retry count: %d", len(retries)) - - // Run the tests in the retries list. - q := newParallelTestQueue(testRunnerContext) - q.Execute(testCtx, retries, parallelism, testOutputConfig, abortFn) - - var flaky, skipped []string - var repeatFailures []*testCase - for _, test := range retries { - if test.success { - flaky = append(flaky, test.name) - } else if test.skipped { - skipped = append(skipped, test.name) - } else { - repeatFailures = append(repeatFailures, test) - } - } - - // Add the list of retries into the list of all tests. - for _, retry := range retries { - if retry.flake { - // Retry tests that flaked are omitted so that the original test is counted as a failure. - fmt.Fprintf(o.Out, "Ignoring retry that returned a flake, original failure is authoritative for test: %s\n", retry.name) - continue - } - tests = append(tests, retry) - } - if len(flaky) > 0 { - failing = repeatFailures - sort.Strings(flaky) - fmt.Fprintf(o.Out, "Flaky tests:\n\n%s\n\n", strings.Join(flaky, "\n")) - } - if len(skipped) > 0 { - // If a retry test got skipped, it means we very likely failed a precondition in the first failure, so - // we need to remove the failure case. - var withoutPreconditionFailures []*testCase - testLoop: - for _, t := range tests { - for _, st := range skipped { - if t.name == st && t.failed { - continue testLoop - } - withoutPreconditionFailures = append(withoutPreconditionFailures, t) - } - } - tests = withoutPreconditionFailures - failing = repeatFailures - sort.Strings(skipped) - fmt.Fprintf(o.Out, "Skipped tests that failed a precondition:\n\n%s\n\n", strings.Join(skipped, "\n")) - - } - } + //if fail > 0 && fail <= suite.MaximumAllowedFlakes { + // var retries []*testCase + // + // // Make a copy of the all failing tests (subject to the max allowed flakes) so we can have + // // a list of tests to retry. + // for _, test := range failing { + // retry := test.Retry() + // retries = append(retries, retry) + // if len(retries) > suite.MaximumAllowedFlakes { + // break + // } + // } + // + // logrus.Warningf("Retry count: %d", len(retries)) + // + // // Run the tests in the retries list. + // q := newParallelTestQueue(testRunnerContext) + // q.Execute(testCtx, retries, parallelism, testOutputConfig, abortFn) + // + // var flaky, skipped []string + // var repeatFailures []*testCase + // for _, test := range retries { + // if test.success { + // flaky = append(flaky, test.name) + // } else if test.skipped { + // skipped = append(skipped, test.name) + // } else { + // repeatFailures = append(repeatFailures, test) + // } + // } + // + // // Add the list of retries into the list of all tests. + // for _, retry := range retries { + // if retry.flake { + // // Retry tests that flaked are omitted so that the original test is counted as a failure. + // fmt.Fprintf(o.Out, "Ignoring retry that returned a flake, original failure is authoritative for test: %s\n", retry.name) + // continue + // } + // tests = append(tests, retry) + // } + // if len(flaky) > 0 { + // failing = repeatFailures + // sort.Strings(flaky) + // fmt.Fprintf(o.Out, "Flaky tests:\n\n%s\n\n", strings.Join(flaky, "\n")) + // } + // if len(skipped) > 0 { + // // If a retry test got skipped, it means we very likely failed a precondition in the first failure, so + // // we need to remove the failure case. + // var withoutPreconditionFailures []*testCase + // testLoop: + // for _, t := range tests { + // for _, st := range skipped { + // if t.name == st && t.failed { + // continue testLoop + // } + // withoutPreconditionFailures = append(withoutPreconditionFailures, t) + // } + // } + // tests = withoutPreconditionFailures + // failing = repeatFailures + // sort.Strings(skipped) + // fmt.Fprintf(o.Out, "Skipped tests that failed a precondition:\n\n%s\n\n", strings.Join(skipped, "\n")) + // + // } + //} // monitor the cluster while the tests are running and report any detected anomalies var syntheticTestResults []*junitapi.JUnitTestCase From 0a6c7907a1a4165c19b35ae02aa4f69100335bdf Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Sun, 18 May 2025 17:25:43 -0400 Subject: [PATCH 5/5] Output timestamp --- pkg/test/ginkgo/test_runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/test/ginkgo/test_runner.go b/pkg/test/ginkgo/test_runner.go index c07c5143bb8e..3bb9ec1a8d54 100644 --- a/pkg/test/ginkgo/test_runner.go +++ b/pkg/test/ginkgo/test_runner.go @@ -54,7 +54,7 @@ func (r *testSuiteRunnerImpl) RunOneTest(ctx context.Context, test *testCase) { defer recordTestResultInMonitor(testRunResult, r.testOutput.monitorRecorder) // log the results to systemout - r.testSuiteProgress.LogTestStart(r.testOutput.out, test.name) + r.testSuiteProgress.LogTestStart(r.testOutput.out, time.Now().Format(time.RFC3339)+"_"+test.name) defer r.testSuiteProgress.TestEnded(test.name, testRunResult) defer recordTestResultInLogWithoutOverlap(testRunResult, r.testOutput.testOutputLock, r.testOutput.out, r.testOutput.includeSuccessfulOutput)