From a4661b3ecc6a744b5a0ebd4f5d2ee970026076a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Mon, 5 Apr 2021 22:55:26 +0200 Subject: [PATCH 1/7] Add support for ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS environment variable --- CHANGELOG.md | 2 ++ REFERENCE.md | 17 +++++++++++++++++ lib/arduino_ci/cpp_library.rb | 14 ++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64257932..7165aebc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added +- Added support for `ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS` environment variable. + ### Changed ### Deprecated diff --git a/REFERENCE.md b/REFERENCE.md index 7767251e..cc163110 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -68,6 +68,23 @@ If set, testing will fail if no unit test files are detected (or if the director If set, testing will fail if no example sketches are detected. This is to avoid communicating a passing status in cases where a commit may have accidentally moved or deleted the examples. +### `ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS` environment variable + +If you want to pass on additional flags to the compiler when it runs unit tests +you can set this variable, e.g. + +```bash +export ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS="--coverage -g -O0" +``` + +By default the variable will be split up by space characters. If one of the +flags contain spaces use the `ARDUINO_CI_TEST_EXTRA_COMPILER_FLAGS_DELIMITER` +variable to chose a different delimiter, e.g. + +```bash +export ARDUINO_CI_TEST_EXTRA_COMPILER_FLAGS_DELIMITER="|" +export ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS="-Wall|-DGREETING='Hello world'" +``` ## Indirectly Overriding Build Behavior (medium term use), and Advanced Options diff --git a/lib/arduino_ci/cpp_library.rb b/lib/arduino_ci/cpp_library.rb index 8ee111ea..4e5317c2 100644 --- a/lib/arduino_ci/cpp_library.rb +++ b/lib/arduino_ci/cpp_library.rb @@ -485,6 +485,18 @@ def test_args(aux_libraries, ci_gcc_config) ret end + # Allow users to inject extra compiler flags though environment variable. + def extra_compiler_flags_for_unittest + return [] unless ENV["ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS"] + + delimiter = if ENV["ARDUINO_CI_TEST_EXTRA_COMPILER_FLAGS_DELIMITER"] + yield(ENV["ARDUINO_CI_TEST_EXTRA_COMPILER_FLAGS_DELIMITER"]) + else + " " + end + ENV["ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS"].split(delimiter) + end + # build a file for running a test of the given unit test file # # The dependent libraries configuration is appended with data from library.properties internal to the library under test @@ -505,6 +517,7 @@ def build_for_test(test_file, gcc_binary) ] end arg_sets << @test_args + arg_sets << extra_compiler_flags_for_unittest arg_sets << [test_file.to_s, "-l#{LIBRARY_NAME}"] args = arg_sets.flatten(1) return nil unless run_gcc(gcc_binary, *args) @@ -555,6 +568,7 @@ def build_shared_library(aux_libraries, gcc_binary, ci_gcc_config) @test_args = test_args(@full_dependencies, ci_gcc_config) # build full set of include directories to be cached for later arg_sets << @test_args + arg_sets << extra_compiler_flags_for_unittest arg_sets << cpp_files_arduino.map(&:to_s) # Arduino.cpp, Godmode.cpp, and stdlib.cpp arg_sets << cpp_files_unittest.map(&:to_s) # ArduinoUnitTests.cpp arg_sets << cpp_files.map(&:to_s) # CPP files for the primary application library under test From ad198305a08df48b34e0c216ccf3f89326e4b3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Fri, 4 Feb 2022 18:02:31 +0100 Subject: [PATCH 2/7] Break long line --- REFERENCE.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index cc163110..f1047bca 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -51,8 +51,14 @@ This specifies the minimum free SRAM memory for stack/heap, in bytes, that _must ### `CUSTOM_INIT_SCRIPT` environment variable -If set, testing will execute (using `/bin/sh`) the script referred to by this variable -- relative to the current working directory (i.e. the root directory of the library). The script will _run_ in the Arduino Libraries directory (changing to the Libraries directory, running the script, and returning to the individual library root afterward). This enables use cases like the GitHub action to install custom library versions (i.e. a version of a library that is different than what the library manager would automatically install by name) prior to CI test runs. - +If set, testing will execute (using `/bin/sh`) the script referred to by this +variable -- relative to the current working directory (i.e. the root directory +of the library). The script will _run_ in the Arduino Libraries directory +(changing to the Libraries directory, running the script, and returning to the +individual library root afterward). This enables use cases like the GitHub +action to install custom library versions (i.e. a version of a library that +is different than what the library manager would automatically install by name) +prior to CI test runs. ### `USE_SUBDIR` environment variable From f9cdf2236ba8a376f3e7fc63b95261a4da073935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Fri, 4 Feb 2022 19:09:09 +0100 Subject: [PATCH 3/7] Remove unused config argument to perform_custom_initialization --- exe/arduino_ci.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exe/arduino_ci.rb b/exe/arduino_ci.rb index 72d455ed..8e585f1a 100755 --- a/exe/arduino_ci.rb +++ b/exe/arduino_ci.rb @@ -333,8 +333,8 @@ def get_annotated_compilers(config, cpp_library) # # This feature is to drive GitHub actions / docker image installation where the container is # in a clean-slate state but needs some way to have custom library versions injected into it. -# In this case, the user provided script would fetch a git repo or some other method -def perform_custom_initialization(_config) +# In this case, the user provided script would fetch a git repo or some other method. +def perform_custom_initialization() script_path = ENV[VAR_CUSTOM_INIT_SCRIPT] inform("Environment variable #{VAR_CUSTOM_INIT_SCRIPT}") { "'#{script_path}'" } return if script_path.nil? @@ -567,7 +567,7 @@ def perform_example_compilation_tests(cpp_library, config) end # run any library init scripts from the library itself. -perform_custom_initialization(config) +perform_custom_initialization() # initialize library under test inform("Environment variable #{VAR_USE_SUBDIR}") { "'#{ENV[VAR_USE_SUBDIR]}'" } From 459419f7c1259bc928c38acec4f12ab81cc77a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Fri, 4 Feb 2022 18:11:21 +0100 Subject: [PATCH 4/7] Add support for overriding shell to execute CUSTOM_INIT_SCRIPT with --- CHANGELOG.md | 2 ++ REFERENCE.md | 8 +++++--- exe/arduino_ci.rb | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7165aebc..e7ae6a74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - Added support for `ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS` environment variable. +- Added support for overriding the shell used to execute `CUSTOM_INIT_SCRIPT` + by setting `CUSTOM_INIT_SCRIPT_SHELL` (defaults to `/bin/sh`). ### Changed diff --git a/REFERENCE.md b/REFERENCE.md index f1047bca..fb5f4417 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -51,9 +51,11 @@ This specifies the minimum free SRAM memory for stack/heap, in bytes, that _must ### `CUSTOM_INIT_SCRIPT` environment variable -If set, testing will execute (using `/bin/sh`) the script referred to by this -variable -- relative to the current working directory (i.e. the root directory -of the library). The script will _run_ in the Arduino Libraries directory +If set, testing will execute the script referred to by this variable -- relative +to the current working directory (i.e. the root directory of the library). By +default the script will be executed using `/bin/sh` but you can override by +setting `CUSTOM_INIT_SCRIPT_SHELL` (e.g to `powershell`, `/usr/bin/perl` etc). +The script will _run_ in the Arduino Libraries directory (changing to the Libraries directory, running the script, and returning to the individual library root afterward). This enables use cases like the GitHub action to install custom library versions (i.e. a version of a library that diff --git a/exe/arduino_ci.rb b/exe/arduino_ci.rb index 8e585f1a..50ed22ce 100755 --- a/exe/arduino_ci.rb +++ b/exe/arduino_ci.rb @@ -66,6 +66,8 @@ def self.parse(options) puts "Additionally, the following environment variables control the script:" puts " - #{VAR_CUSTOM_INIT_SCRIPT} - if set, this script will be run from the Arduino/libraries directory" puts " prior to any automated library installation or testing (e.g. to install unofficial libraries)" + puts " - #{VAR_CUSTOM_INIT_SCRIPT}_SHELL - if set, this will override the" + puts " default shell (/bin/sh) used to execute #{VAR_CUSTOM_INIT_SCRIPT} with." puts " - #{VAR_USE_SUBDIR} - if set, the script will install the library from this subdirectory of the cwd" puts " - #{VAR_EXPECT_EXAMPLES} - if set, testing will fail if no example sketches are present" puts " - #{VAR_EXPECT_UNITTESTS} - if set, testing will fail if no unit tests are present" @@ -336,6 +338,7 @@ def get_annotated_compilers(config, cpp_library) # In this case, the user provided script would fetch a git repo or some other method. def perform_custom_initialization() script_path = ENV[VAR_CUSTOM_INIT_SCRIPT] + script_shell = ENV[VAR_CUSTOM_INIT_SCRIPT + "_SHELL"] || "/bin/sh" inform("Environment variable #{VAR_CUSTOM_INIT_SCRIPT}") { "'#{script_path}'" } return if script_path.nil? return if script_path.empty? @@ -343,9 +346,9 @@ def perform_custom_initialization() script_pathname = Pathname.getwd + script_path assure("Script at #{VAR_CUSTOM_INIT_SCRIPT} exists") { script_pathname.exist? } - assure_multiline("Running #{script_pathname} with sh in libraries working dir") do + assure_multiline("Running #{script_pathname} with #{script_shell} in libraries working dir") do Dir.chdir(@backend.lib_dir) do - IO.popen(["/bin/sh", script_pathname.to_s], err: [:child, :out]) do |io| + IO.popen([script_shell, script_pathname.to_s], err: [:child, :out]) do |io| io.each_line { |line| puts " #{line}" } end end From 3cfc8bcf1a3535d9d8d8fcd81d27d5d306da9e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Fri, 4 Feb 2022 18:50:30 +0100 Subject: [PATCH 5/7] Generalize perform_custom_initialization into run_custom_script --- exe/arduino_ci.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exe/arduino_ci.rb b/exe/arduino_ci.rb index 50ed22ce..3310c04d 100755 --- a/exe/arduino_ci.rb +++ b/exe/arduino_ci.rb @@ -331,20 +331,20 @@ def get_annotated_compilers(config, cpp_library) compilers end -# Handle existence or nonexistence of custom initialization script -- run it if you have it +# Run custom custom script specified by user. # # This feature is to drive GitHub actions / docker image installation where the container is # in a clean-slate state but needs some way to have custom library versions injected into it. # In this case, the user provided script would fetch a git repo or some other method. -def perform_custom_initialization() - script_path = ENV[VAR_CUSTOM_INIT_SCRIPT] - script_shell = ENV[VAR_CUSTOM_INIT_SCRIPT + "_SHELL"] || "/bin/sh" - inform("Environment variable #{VAR_CUSTOM_INIT_SCRIPT}") { "'#{script_path}'" } +def run_custom_script(env_var) + script_path = ENV[env_var] + script_shell = ENV[env_var + "_SHELL"] || "/bin/sh" + inform("Environment variable #{env_var}") { "'#{script_path}'" } return if script_path.nil? return if script_path.empty? script_pathname = Pathname.getwd + script_path - assure("Script at #{VAR_CUSTOM_INIT_SCRIPT} exists") { script_pathname.exist? } + assure("Script at #{env_var} exists") { script_pathname.exist? } assure_multiline("Running #{script_pathname} with #{script_shell} in libraries working dir") do Dir.chdir(@backend.lib_dir) do @@ -570,7 +570,7 @@ def perform_example_compilation_tests(cpp_library, config) end # run any library init scripts from the library itself. -perform_custom_initialization() +run_custom_script(VAR_CUSTOM_INIT_SCRIPT) # initialize library under test inform("Environment variable #{VAR_USE_SUBDIR}") { "'#{ENV[VAR_USE_SUBDIR]}'" } From 4cd52f1c8c78de4742a9ef824f65ceda047c65b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Fri, 4 Feb 2022 19:06:00 +0100 Subject: [PATCH 6/7] Extend run_custom_script to accept arguments --- exe/arduino_ci.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exe/arduino_ci.rb b/exe/arduino_ci.rb index 3310c04d..0e3d1a6f 100755 --- a/exe/arduino_ci.rb +++ b/exe/arduino_ci.rb @@ -336,7 +336,7 @@ def get_annotated_compilers(config, cpp_library) # This feature is to drive GitHub actions / docker image installation where the container is # in a clean-slate state but needs some way to have custom library versions injected into it. # In this case, the user provided script would fetch a git repo or some other method. -def run_custom_script(env_var) +def run_custom_script(env_var, *args) script_path = ENV[env_var] script_shell = ENV[env_var + "_SHELL"] || "/bin/sh" inform("Environment variable #{env_var}") { "'#{script_path}'" } @@ -348,7 +348,7 @@ def run_custom_script(env_var) assure_multiline("Running #{script_pathname} with #{script_shell} in libraries working dir") do Dir.chdir(@backend.lib_dir) do - IO.popen([script_shell, script_pathname.to_s], err: [:child, :out]) do |io| + IO.popen([script_shell, script_pathname.to_s, *args], err: [:child, :out]) do |io| io.each_line { |line| puts " #{line}" } end end From 0887f9f41cb7a1785d310a4eb2189883bf8a74b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20L=C3=B8vdal?= Date: Fri, 4 Feb 2022 20:32:23 +0100 Subject: [PATCH 7/7] Add support for pre/post unit test run scripts --- CHANGELOG.md | 2 ++ REFERENCE.md | 25 +++++++++++++++++++++++++ exe/arduino_ci.rb | 6 ++++++ 3 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7ae6a74..dae3ab9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added support for `ARDUINO_CI_UNIT_TEST_EXTRA_COMPILER_FLAGS` environment variable. - Added support for overriding the shell used to execute `CUSTOM_INIT_SCRIPT` by setting `CUSTOM_INIT_SCRIPT_SHELL` (defaults to `/bin/sh`). +- Added support running scripts before and/or after each run of unit tests with + `ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT` and `ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT`. ### Changed diff --git a/REFERENCE.md b/REFERENCE.md index fb5f4417..5e368a93 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -62,6 +62,31 @@ action to install custom library versions (i.e. a version of a library that is different than what the library manager would automatically install by name) prior to CI test runs. +### `ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT` and `ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT` environment variables + +If set, the corresponding script will be run before/after each run of unit tests +for each configured platform and for each compiler, e.g. if `.arduino-ci.yaml` +contains + +```yaml +unittest: + compilers: + - g++-10 + - g++-11 + libraries: ~ + platforms: + - leonardo + - uno +``` + +the scripts will be invoked four times, with the current platform name being +tested as the first parameter to the script and the current compiler used as +the second parameter. It is not necessary to define both PRE and POST script; +if you only want to run something before or after unit tests that's fine. +By default the scripts are executed by `/bin/sh`, you can override by setting +`ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT_SHELL` and +`ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT_SHELL` respectively. + ### `USE_SUBDIR` environment variable If set, testing will be conducted in this subdirectory (relative to the working directory). This is for monorepos or other layouts where the library directory and project root directory are different. diff --git a/exe/arduino_ci.rb b/exe/arduino_ci.rb index 0e3d1a6f..c3146aec 100755 --- a/exe/arduino_ci.rb +++ b/exe/arduino_ci.rb @@ -15,6 +15,8 @@ VAR_USE_SUBDIR = "USE_SUBDIR".freeze VAR_EXPECT_EXAMPLES = "EXPECT_EXAMPLES".freeze VAR_EXPECT_UNITTESTS = "EXPECT_UNITTESTS".freeze +VAR_ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT = "ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT".freeze +VAR_ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT = "ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT".freeze @failure_count = 0 @passfail = proc { |result| result ? "✓" : "✗" } @@ -68,6 +70,8 @@ def self.parse(options) puts " prior to any automated library installation or testing (e.g. to install unofficial libraries)" puts " - #{VAR_CUSTOM_INIT_SCRIPT}_SHELL - if set, this will override the" puts " default shell (/bin/sh) used to execute #{VAR_CUSTOM_INIT_SCRIPT} with." + puts " - #{VAR_ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT} and/or #{VAR_ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT}" + puts " if set, run the script before/after each unit test run" puts " - #{VAR_USE_SUBDIR} - if set, the script will install the library from this subdirectory of the cwd" puts " - #{VAR_EXPECT_EXAMPLES} - if set, testing will fail if no example sketches are present" puts " - #{VAR_EXPECT_UNITTESTS} - if set, testing will fail if no unit tests are present" @@ -444,6 +448,7 @@ def perform_unit_tests(cpp_library, file_config) platforms.each do |p| puts compilers.each do |gcc_binary| + run_custom_script(VAR_ARDUINO_CI_PRE_UNIT_TEST_RUN_SCRIPT, p, gcc_binary) # before compiling the tests, build a shared library of everything except the test code next @failure_count += 1 unless build_shared_library(gcc_binary, p, config, cpp_library) @@ -463,6 +468,7 @@ def perform_unit_tests(cpp_library, file_config) cpp_library.run_test_file(exe) end end + run_custom_script(VAR_ARDUINO_CI_POST_UNIT_TEST_RUN_SCRIPT, p, gcc_binary) end end end