From 80201739e2b17de3ac45dbecc8f72c7e29a436b4 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sun, 26 Mar 2023 10:38:19 +0200 Subject: [PATCH 1/6] Check if tests are coupled cf. https://github.com/r-lib/lintr/issues/1937 --- .../workflows/check-random-test-order.yaml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/check-random-test-order.yaml diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml new file mode 100644 index 000000000..ddfd4f75c --- /dev/null +++ b/.github/workflows/check-random-test-order.yaml @@ -0,0 +1,74 @@ +# Run tests in random order +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +name: check-random-test-order + +jobs: + check-random-test-order: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: "devel" + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + pak-version: devel + extra-packages: | + local::. + + - name: Run Tests in Random Order + run: | + options(crayon.enabled = TRUE) + withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") + library(cli) + library(glue) + library(testthat) + pkgload::load_all(".") + test_script_paths <- testthat::find_test_scripts("tests/testthat") + set.seed(123) + randomized_test_script_paths <- sample(test_script_paths) + any_test_failures <- FALSE + any_test_errors <- FALSE + test_path <- function(path) { + report <- as.data.frame(testthat::test_file(path, reporter = "silent")) + has_test_failures <- any(report$failed == 1L) + has_test_errors <- any(report$error == 1L) + if (has_test_failures) { + cli_alert_danger(glue("Tests in `{path}` are failing.")) + any_test_failures <<- TRUE + } + if (has_test_errors) { + cli_alert_danger(glue("There was error while running tests in `{path}`.")) + any_test_errors <<- TRUE + } + if (!has_test_failures && !has_test_errors) { + cli_alert_success(glue("All tests passing in `{path}`.")) + } + } + cli_rule() + cli_inform("Running tests in random order:") + cli_rule() + purrr::walk(randomized_test_script_paths, test_path) + cli_rule() + if (any_test_failures) { + cli_abort("Tests in some files are failing. Check the log.") + } + if (any_test_errors) { + cli_abort("There was error while running tests in some files. Check the log.") + } + if (!any_test_failures && !any_test_errors) { + cli_alert_success("Tests from all files are passing!") + } + cli_rule() + shell: Rscript {0} From d01a9e35e7b7c462ab3b1848f821a0e321cbe769 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sat, 15 Apr 2023 10:50:23 +0200 Subject: [PATCH 2/6] select a different seed each time --- .github/workflows/check-random-test-order.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index ddfd4f75c..fede8548d 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -36,7 +36,9 @@ jobs: library(testthat) pkgload::load_all(".") test_script_paths <- testthat::find_test_scripts("tests/testthat") - set.seed(123) + seed <- sample.int(1e6, 1L) + cli_inform("Chosen seed for the current test run: {seed}") + set.seed(seed) randomized_test_script_paths <- sample(test_script_paths) any_test_failures <- FALSE any_test_errors <- FALSE From c26668d178a401aa2d3923fd7e3156473d0385a7 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sat, 15 Apr 2023 10:53:52 +0200 Subject: [PATCH 3/6] show the exact errors --- .github/workflows/check-random-test-order.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index fede8548d..b4fb3dd50 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -33,7 +33,6 @@ jobs: withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") library(cli) library(glue) - library(testthat) pkgload::load_all(".") test_script_paths <- testthat::find_test_scripts("tests/testthat") seed <- sample.int(1e6, 1L) @@ -49,10 +48,14 @@ jobs: if (has_test_failures) { cli_alert_danger(glue("Tests in `{path}` are failing.")) any_test_failures <<- TRUE + failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) + print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) } if (has_test_errors) { cli_alert_danger(glue("There was error while running tests in `{path}`.")) any_test_errors <<- TRUE + errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) + print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) } if (!has_test_failures && !has_test_errors) { cli_alert_success(glue("All tests passing in `{path}`.")) @@ -64,10 +67,10 @@ jobs: purrr::walk(randomized_test_script_paths, test_path) cli_rule() if (any_test_failures) { - cli_abort("Tests in some files are failing. Check the log.") + cli_abort("Tests in some files are failing.") } if (any_test_errors) { - cli_abort("There was error while running tests in some files. Check the log.") + cli_abort("There was error while running tests in some files.") } if (!any_test_failures && !any_test_errors) { cli_alert_success("Tests from all files are passing!") From d462116d71fc1382927fb5aafa351417859d60b5 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Thu, 20 Apr 2023 10:43:04 +0200 Subject: [PATCH 4/6] extract code to a script --- .dev/run-test-files-in-random-order.R | 54 +++++++++++++++++++ .../workflows/check-random-test-order.yaml | 48 +---------------- 2 files changed, 56 insertions(+), 46 deletions(-) create mode 100644 .dev/run-test-files-in-random-order.R diff --git a/.dev/run-test-files-in-random-order.R b/.dev/run-test-files-in-random-order.R new file mode 100644 index 000000000..7c769bb05 --- /dev/null +++ b/.dev/run-test-files-in-random-order.R @@ -0,0 +1,54 @@ +library(cli) +library(glue) +withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") +pkgload::load_all(".") + +test_script_paths <- testthat::find_test_scripts("tests/testthat") +seed <- sample.int(1e6, 1L) +cli_inform("Chosen seed for the current test run: {seed}") +set.seed(seed) +randomized_test_script_paths <- sample(test_script_paths) + +any_test_failures <- FALSE +any_test_errors <- FALSE + +test_path <- function(path) { + report <- as.data.frame(testthat::test_file(path, reporter = "silent")) + has_test_failures <- any(report$failed == 1L) + has_test_errors <- any(report$error == 1L) + if (has_test_failures) { + cli_alert_danger(glue("Tests in `{path}` are failing.")) + any_test_failures <<- TRUE + failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) + print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) + } + if (has_test_errors) { + cli_alert_danger(glue("There was error while running tests in `{path}`.")) + any_test_errors <<- TRUE + errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) + print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) + } + if (!has_test_failures && !has_test_errors) { + cli_alert_success(glue("All tests passing in `{path}`.")) + } +} + +cli_rule() +cli_inform("Running tests in random order:") +cli_rule() + +purrr::walk(randomized_test_script_paths, test_path) + +cli_rule() +if (any_test_failures) { + cli_abort("Tests in some files are failing.") +} + +if (any_test_errors) { + cli_abort("There was error while running tests in some files.") +} + +if (!any_test_failures && !any_test_errors) { + cli_alert_success("Tests from all files are passing!") +} +cli_rule() diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index b4fb3dd50..af30b7eef 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -30,50 +30,6 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) - withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") - library(cli) - library(glue) - pkgload::load_all(".") - test_script_paths <- testthat::find_test_scripts("tests/testthat") - seed <- sample.int(1e6, 1L) - cli_inform("Chosen seed for the current test run: {seed}") - set.seed(seed) - randomized_test_script_paths <- sample(test_script_paths) - any_test_failures <- FALSE - any_test_errors <- FALSE - test_path <- function(path) { - report <- as.data.frame(testthat::test_file(path, reporter = "silent")) - has_test_failures <- any(report$failed == 1L) - has_test_errors <- any(report$error == 1L) - if (has_test_failures) { - cli_alert_danger(glue("Tests in `{path}` are failing.")) - any_test_failures <<- TRUE - failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) - print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) - } - if (has_test_errors) { - cli_alert_danger(glue("There was error while running tests in `{path}`.")) - any_test_errors <<- TRUE - errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) - print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) - } - if (!has_test_failures && !has_test_errors) { - cli_alert_success(glue("All tests passing in `{path}`.")) - } - } - cli_rule() - cli_inform("Running tests in random order:") - cli_rule() - purrr::walk(randomized_test_script_paths, test_path) - cli_rule() - if (any_test_failures) { - cli_abort("Tests in some files are failing.") - } - if (any_test_errors) { - cli_abort("There was error while running tests in some files.") - } - if (!any_test_failures && !any_test_errors) { - cli_alert_success("Tests from all files are passing!") - } - cli_rule() + callr::rscript("run-test-files-in-random-order.R") + shell: Rscript {0} From b084cac331d8d39351948568c1fbd93a42e731d2 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Thu, 20 Apr 2023 10:48:08 +0200 Subject: [PATCH 5/6] Update check-random-test-order.yaml --- .github/workflows/check-random-test-order.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index af30b7eef..66c2dca01 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -30,6 +30,6 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) - callr::rscript("run-test-files-in-random-order.R") + callr::rscript(".dev/run-test-files-in-random-order.R") shell: Rscript {0} From f6e3b2f31cdf3bdd21c97a0e3e3f46cd2eba4fd0 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Tue, 19 Sep 2023 08:42:55 +0200 Subject: [PATCH 6/6] run on release version --- .github/workflows/check-random-test-order.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index 66c2dca01..77dea2b53 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -18,7 +18,7 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: - r-version: "devel" + r-version: "release" use-public-rspm: true - uses: r-lib/actions/setup-r-dependencies@v2