diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 008bd673..7c7187dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,60 +1,63 @@ stages: - build-and-test - - benchmark + - benchmark-completion variables: PYTHONUNBUFFERED: "true" +before_script: + - git submodule update --init + .build_and_test_job: &BUILD_AND_TEST stage: build-and-test tags: [benchmarks, infinity] script: - - git submodule update --init - - make test - - make bench - - make clean - - make unittests + - cmake . $CONF + - make + - ./SOM++ -cp ./Smalltalk ./TestSuite/TestHarness.som + - ./unittests -cp ./Smalltalk:./TestSuite/BasicInterpreterTests ./Examples/Hello.som + + - rebench --experiment="CI ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c rebench.conf all "e:${NAME}" tagging_gen: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "true", GC_TYPE: generational} + variables: {CONF: "-DUSE_TAGGING=true -DGC_TYPE=GENERATIONAL", NAME: tag-gen} tagging_marksweep: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "true", GC_TYPE: mark_sweep} + variables: {CONF: "-DUSE_TAGGING=true -DGC_TYPE=MARK_SWEEP", NAME: tag-mark} tagging_copying: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "true", GC_TYPE: copying} + variables: {CONF: "-DUSE_TAGGING=true -DGC_TYPE=COPYING", NAME: tag-copy} boxed_generational_cached: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "false", GC_TYPE: generational, CACHE_INTEGER: "true"} + variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=GENERATIONAL -DCACHE_INTEGER=true", NAME: gen-cache} boxed_mark_sweep_cached: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "false", GC_TYPE: mark_sweep, CACHE_INTEGER: "true"} + variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=MARK_SWEEP -DCACHE_INTEGER=true", NAME: mark-cache} boxed_copying_cached: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "false", GC_TYPE: copying, CACHE_INTEGER: "true"} + variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=COPYING -DCACHE_INTEGER=true", NAME: copy-cache} boxed_generational_uncached: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "false", GC_TYPE: generational, CACHE_INTEGER: "false"} + variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=GENERATIONAL -DCACHE_INTEGER=false", NAME: gen} boxed_mark_sweep_uncached: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "false", GC_TYPE: mark_sweep, CACHE_INTEGER: "false"} + variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=MARK_SWEEP -DCACHE_INTEGER=false", NAME: mark} boxed_copying_uncached: <<: *BUILD_AND_TEST - variables: {USE_TAGGING: "false", GC_TYPE: copying, CACHE_INTEGER: "false"} + variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=COPYING -DCACHE_INTEGER=false", NAME: copy} -benchmark_job: - stage: benchmark + +complete_benchmarking: + stage: benchmark-completion tags: [benchmarks, infinity] - allow_failure: false script: - - make - - rebench -d --without-nice -c --commit-id="$CI_BUILD_REF" --environment="Infinity Ubuntu" --project=SOMpp --branch="$CI_BUILD_REF_NAME" core-lib/codespeed.conf SOMpp + - rebench --experiment="CI ID $CI_PIPELINE_ID" --report-completion rebench.conf diff --git a/rebench.conf b/rebench.conf new file mode 100644 index 00000000..ddabb274 --- /dev/null +++ b/rebench.conf @@ -0,0 +1,84 @@ +# -*- mode: yaml -*- +# Config file for ReBench +default_experiment: all +default_data_file: 'rebench.data' + +reporting: + # Benchmark results will be reported to ReBenchDB + rebenchdb: + # this url needs to point to the API endpoint + db_url: https://rebench.stefan-marr.de/rebenchdb + repo_url: https://github.com/smarr/SOMpp + record_all: true # make sure everything is recorded + project_name: SOMpp + +runs: + max_invocation_time: 60 + +benchmark_suites: + macro: + gauge_adapter: RebenchLog + command: &MACRO_CMD "-cp Smalltalk:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/Json:Examples/Benchmarks/GraphSearch Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s %(iterations)s 0 " + iterations: 10 + benchmarks: + - Richards: {extra_args: 1} + - DeltaBlue: {extra_args: 50} + - NBody: {extra_args: 500} + - JsonSmall: {extra_args: 1} + - GraphSearch: {extra_args: 4} + - PageRank: {extra_args: 40} + + micro: + gauge_adapter: RebenchLog + command: "-cp Smalltalk:Examples/Benchmarks/LanguageFeatures Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s %(iterations)s 0 " + iterations: 10 + benchmarks: + - Fannkuch: {extra_args: 6} + - Fibonacci: {extra_args: 3} + - Dispatch: {extra_args: 2} + - Bounce: {extra_args: 2} + - Loop: {extra_args: 5} + - Permute: {extra_args: 3} + - Queens: {extra_args: 2} + - List: {extra_args: 2} + - Recurse: {extra_args: 3} + - Storage: {extra_args: 1} + - Sieve: {extra_args: 4} + - BubbleSort: {extra_args: 3} + - QuickSort: {extra_args: 1} + - Sum: {extra_args: 2} + - Towers: {extra_args: 2} + - TreeSort: {extra_args: 1} + - IntegerLoop: {extra_args: 2} + - FieldLoop: {extra_args: 1} + - WhileLoop: {extra_args: 10} + - Mandelbrot: {extra_args: 30} + +executors: + tag-gen: {path: ., executable: SOM++} + tag-mark: {path: ., executable: SOM++} + tag-copy: {path: ., executable: SOM++} + gen-cache: {path: ., executable: SOM++} + mark-cache: {path: ., executable: SOM++} + copy-cache: {path: ., executable: SOM++} + gen: {path: ., executable: SOM++} + mark: {path: ., executable: SOM++} + copy: {path: ., executable: SOM++} + +# define the benchmarks to be executed for a re-executable benchmark run +experiments: + SOM++: + description: All benchmarks on CSOM + suites: + - micro + - macro + executions: + - tag-gen + - tag-mark + - tag-copy + - gen-cache + - mark-cache + - copy-cache + - gen + - mark + - copy