Compare local and weekly benchmarks using Hatchet

.gitlab-ci.yml

@@ -12,25 +12,52 @@ variables:
   BUILD_ROOT: ${CI_PROJECT_DIR}
   FULL_BUILD_ROOT: ${CI_BUILDS_DIR}/serac/${CI_JOB_NAME}
   ALLOC_BANK: eng
+  SPOT_DIR: ${BUILD_ROOT}/califiles
 
 # Whether and how to update uberenv
 .run_update_uberenv: &run_update_uberenv |
   [[ -n "${UPDATE_UBERENV}" ]] && ./scripts/gitlab/update-uberenv.sh "${UPDATE_UBERENV}"
 
+stages:
+  - src
+  - full
+  - benchmarks
+  - comparison
+
 # Run src build each push
 .src_workflow:
+  stage: src
   rules:
-    - if: $SERAC_CI_WORKFLOW_TYPE != "full" && $SERAC_CI_WORKFLOW_TYPE != "benchmarks"
+    - if: $SERAC_CI_WORKFLOW_TYPE != "full" &&
+          $SERAC_CI_WORKFLOW_TYPE != "benchmarks" &&
+          $SERAC_CI_WORKFLOW_TYPE != "comparison"
 
 # Run full build as a nightly scheduled pipeline
 .full_workflow:
+  stage: full
   rules:
     - if: $SERAC_CI_WORKFLOW_TYPE == "full"
 
 # Run benchmarks build as a weekly scheduled pipeline
+# or, run benchmarks and save them in tmp location for comparison
 .benchmarks_workflow:
+  stage: benchmarks
   rules:
     - if: $SERAC_CI_WORKFLOW_TYPE == "benchmarks"
+      variables:
+        # Note: make sure this matches what's in common_build_functions.py!
+        SPOT_DIR: /usr/workspace/smithdev/califiles/serac
+    - if: $SERAC_CI_WORKFLOW_TYPE == "comparison"
+
+# Compare caliper files (generated from .benchmarks_workflow) against develop
+.comparison_workflow:
+  stage: comparison
+  rules:
+    - if: $SERAC_CI_WORKFLOW_TYPE == "comparison"
+  dependencies:
+    - blueos-clang_10_0_1-benchmarks
+    - toss4-gcc_10_3_1-benchmarks
+    - toss4-clang_14_0_6-benchmarks
 
 ####
 # Templates
@@ -73,21 +100,31 @@ variables:
     reports:
       junit: ${FULL_BUILD_ROOT}/${SYS_TYPE}/*/_serac_build_and_test_*/build-*/junit.xml
 
-
 .benchmarks_build_script:
   script:
     # Builds src, runs benchmarks, and stores Caliper files in shared location
     - echo -e "section_start:$(date +%s):benchmarks_build\r\e[0K
       Benchmarks Build ${CI_PROJECT_NAME}"
-    - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py
+    - mkdir ${SPOT_DIR}
+    - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py --spot-dir ${SPOT_DIR}
     - echo -e "section_end:$(date +%s):benchmarks_build\r\e[0K"
   artifacts:
     expire_in: 2 weeks
     when: always
     paths:
       - _serac_build_and_test_*/output.log*.txt
       - _serac_build_and_test_*/build-*/output.log*.txt
-      - _serac_build_and_test_*/build-*/*.cali
+      - ${SPOT_DIR}/*.cali
+
+.comparison_build_script:
+  script:
+    # Takes Caliper files from previous jobs and performs benchmark analysis
+    - echo -e "section_start:$(date +%s):comparison\r\e[0K
+      Comparison Build ${CI_PROJECT_NAME}"
+    - ls ${SPOT_DIR}
+    - ls ${SPOT_DIR} | wc -l
+    - ${ALLOC_COMMAND} python3 scripts/llnl/compare_benchmarks.py --current-cali-dir ${SPOT_DIR} --verbose
+    - echo -e "section_end:$(date +%s):comparison\r\e[0K"
 
 # This is where jobs are included for each system
 include:

.gitlab/build_blueos.yml

@@ -31,15 +31,12 @@
 # Template
 .src_build_on_blueos:
   extends: [.src_build_script, .on_blueos, .src_workflow]
-  needs: []
 
 .full_build_on_blueos:
   extends: [.full_build_script, .on_blueos, .full_workflow]
-  needs: []
 
 .benchmarks_build_on_blueos:
   extends: [.benchmarks_build_script, .on_blueos, .benchmarks_workflow]
-  needs: []
 
 ####
 # Build jobs

.gitlab/build_toss4.yml

@@ -19,19 +19,18 @@
 # Templates
 .src_build_on_toss4:
   extends: [.src_build_script, .on_toss4, .src_workflow]
-  needs: []
 
 .full_build_on_toss4:
   extends: [.full_build_script, .on_toss4, .full_workflow]
-  needs: []
   before_script:
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
 .benchmarks_build_on_toss4:
   extends: [.benchmarks_build_script, .on_toss4, .benchmarks_workflow]
-  needs: []
 
+.comparison_build_on_toss4:
+  extends: [.comparison_build_script, .on_toss4, .comparison_workflow]
 
 ####
 # Build jobs
@@ -113,3 +112,11 @@ toss4-gcc_10_3_1-benchmarks:
     ALLOC_TIME: "120"
     ALLOC_DEADLINE: "180"
   extends: .benchmarks_build_on_toss4
+
+# This comparison job simply runs Hatchet and compares against benchmarks generated across all configurations
+toss4-comparison:
+  variables:
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+    ALLOC_DEADLINE: "45"
+  extends: .comparison_build_on_toss4

.gitlab/build_toss4_cray.yml

@@ -16,19 +16,15 @@
 # Templates
 .src_build_on_toss4_cray:
   extends: [.src_build_script, .on_toss4_cray, .src_workflow]
-  needs: []
 
 .full_build_on_toss4_cray:
   extends: [.full_build_script, .on_toss4_cray, .full_workflow]
-  needs: []
   before_script:
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
 .benchmarks_build_on_toss4_cray:
   extends: [.benchmarks_build_script, .on_toss4_cray, .benchmarks_workflow]
-  needs: []
-
 
 ####
 # Build jobs

config-build.py

@@ -73,8 +73,7 @@ def parse_arguments():
                         "--buildtype",
                         type=str,
                         choices=["Release", "Debug", "RelWithDebInfo", "MinSizeRel"],
-                        default="Debug",
-                        help="build type.")
+                        help="build type. defaults to Debug")
 
     parser.add_argument("-e",
                         "--eclipse",
@@ -105,8 +104,6 @@ def parse_arguments():
                         action='store_true',
                         help="use ninja generator to build serac instead of make")
 
-
-
     args, unknown_args = parser.parse_known_args()
     if unknown_args:
         print("[config-build]: Passing the following arguments directly to cmake... %s" % unknown_args)
@@ -230,6 +227,7 @@ def create_cmake_command_line(args, unknown_args, buildpath, installpath, hostco
 
     # Add build type (opt or debug)
     cmakeline += " -DCMAKE_BUILD_TYPE=" + args.buildtype
+
     # Set install dir
     cmakeline += " -DCMAKE_INSTALL_PREFIX=%s" % installpath
 
@@ -277,7 +275,7 @@ def run_cmake(buildpath, cmakeline):
 # Main
 ############################
 def main():
-    repodir = os.path.abspath(os.path.dirname(__file__))     
+    repodir = os.path.abspath(os.path.dirname(__file__))
     assert os.path.abspath(os.getcwd())==repodir, "config-build must be run from %s" % repodir
 
     args, unknown_args = parse_arguments()
@@ -295,6 +293,15 @@ def main():
        else:
           return False
 
+    # CMake build type is Debug by default, but if CMAKE_BUILD_TYPE is an unknown argument (i.e. a CMake argument), then
+    # use that option instead.
+    if args.buildtype == None:
+        args.buildtype = "Debug"
+        for unknown_arg in unknown_args:
+            if "-DCMAKE_BUILD_TYPE" in unknown_arg:
+                args.buildtype = unknown_arg.split("=")[1]
+                break
+
     basehostconfigpath = find_host_config(args, repodir)
     platform_info = get_platform_info(basehostconfigpath)
     buildpath = setup_build_dir(args, platform_info)