Add script for repeated runs

changelog-entries/226.md

@@ -0,0 +1 @@
+- Added mapping-tester script to repeat the tests and collect the statics from all runs, which is useful for runtime measurements.

examples/mapping_tester_runtime/clean.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
-cd "$(basedir "$0")"
-rm -f test-statistics.csv
+cd "$(dirname "$0")"
+rm -f test-statistics*.csv
 rm -fr ./case/

examples/mapping_tester_runtime/reference-statistics.csv

@@ -1,3 +1,11 @@
-computeMappingTime,constraint,globalTime,initializeTime,mapDataTime,mapping,mesh A,mesh B,peakMemA,peakMemB,ranks A,ranks B
-969.0,consistent,117814.0,90445.0,2.0,nn,coarse_mesh,fine_mesh,117276.0,116088.0,2,2
-312.0,consistent,142285.0,114217.0,371.0,tps,coarse_mesh,fine_mesh,118044.0,129236.0,2,2
+computeMappingTime,constraint,globalTime,initializeTime,mapDataTime,mapping,mesh A,mesh B,peakMemA,peakMemB,ranks A,ranks B,run
+958.0,consistent,115315.0,87333.0,2.0,nn,coarse_mesh,fine_mesh,115668.0,116996.0,2,2,0
+324.0,consistent,146370.0,118212.0,298.0,tps,coarse_mesh,fine_mesh,118200.0,129740.0,2,2,0
+959.0,consistent,115990.0,88043.0,2.0,nn,coarse_mesh,fine_mesh,115064.0,117284.0,2,2,1
+311.0,consistent,140914.0,113093.0,323.0,tps,coarse_mesh,fine_mesh,116232.0,128676.0,2,2,1
+960.0,consistent,114335.0,86142.0,2.0,nn,coarse_mesh,fine_mesh,115476.0,117288.0,2,2,2
+315.0,consistent,139689.0,111733.0,379.0,tps,coarse_mesh,fine_mesh,115204.0,131552.0,2,2,2
+961.0,consistent,114121.0,86120.0,1.0,nn,coarse_mesh,fine_mesh,115164.0,115672.0,2,2,3
+318.0,consistent,142488.0,114608.0,310.0,tps,coarse_mesh,fine_mesh,116672.0,131804.0,2,2,3
+966.0,consistent,114178.0,86872.0,2.0,nn,coarse_mesh,fine_mesh,115724.0,117572.0,2,2,4
+303.0,consistent,146331.0,118273.0,400.0,tps,coarse_mesh,fine_mesh,116300.0,130928.0,2,2,4

examples/mapping_tester_runtime/run.sh

@@ -4,7 +4,7 @@ set -e -x
 cd "$(basedir "$0")"
 
 # The mapping-tester location
-MAPPING_TESTER=../../tools/mapping-tester
+export MAPPING_TESTER=../../tools/mapping-tester
 
 # Generate the run scripts
 python3 "${MAPPING_TESTER}/generate.py" --template "${MAPPING_TESTER}"/config-template.xml --exit
@@ -15,13 +15,7 @@ python3 "${MAPPING_TESTER}/preparemeshes.py" --force
 export ASTE_A_MPIARGS=""
 export ASTE_B_MPIARGS=""
 
-# Run the actual cases
-bash cases/runall.sh
 
-# Postprocess the test cases
-bash cases/postprocessall.sh
-
-# Gather the generated statistics
-python3 "${MAPPING_TESTER}/gatherstats.py" --file test-statistics.csv
+python3 "${MAPPING_TESTER}/repeat.py" 5 --file "test-statistics{}.csv"
 
 python3 "${MAPPING_TESTER}/compare.py" reference-statistics.csv test-statistics.csv

tools/mapping-tester/repeat.py

@@ -0,0 +1,87 @@
+#!python3
+
+import argparse
+import csv
+import pathlib
+import subprocess
+
+
+def parseArguments():
+    # assume both scripts to be in the same folder by default
+    gatherstatsdefault = pathlib.Path(__file__).parent / "gatherstats.py"
+
+    parser = argparse.ArgumentParser(
+        description="Repeats all configured cases N times, gathers stats and merges the results into a single stats file."
+    )
+    parser.add_argument(
+        "repetitions",
+        metavar="N",
+        type=int,
+        help="How often to repeat the test case",
+    )
+    parser.add_argument(
+        "-c",
+        "--cases",
+        default="cases",
+        type=pathlib.Path,
+        help="Directory of the cases.",
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        default="stats{}.csv",
+        help="The resulting CSV file containing all stats. The {} will be replaced with -N for the Nth run.",
+    )
+    parser.add_argument(
+        "-g",
+        "--gatherstats",
+        default=gatherstatsdefault,
+        type=pathlib.Path,
+        help="Path to the gatherstats script.",
+    )
+    return parser.parse_args()
+
+
+def merge(repetitions: int, filefmt):
+    destination = filefmt.format("")
+    print(f"Merging to {destination}")
+    merged = []
+    for n in range(repetitions):
+        with open(filefmt.format(f"-{n}")) as file:
+            for row in csv.DictReader(file):
+                row["run"] = n
+                merged.append(row)
+
+    with open(filefmt.format(""), "w") as file:
+        writer = csv.DictWriter(file, merged[0].keys())
+        writer.writeheader()
+        writer.writerows(merged)
+
+
+def run(repetitions: int, casesdir: pathlib.Path, filefmt, gatherstats: pathlib.Path):
+    for n in range(repetitions):
+        print(f"### Run {n+1} of {repetitions}", flush=True)
+
+        # Run the actual cases
+        subprocess.run(["bash", casesdir / "runall.sh"], check=True)
+
+        # Postprocess the test cases
+        subprocess.run(["bash", casesdir / "postprocessall.sh"], check=True)
+
+        # Gather the generated statistics
+        subprocess.run(
+            ["python3", gatherstats, "--file", filefmt.format(f"-{n}")], check=True
+        )
+
+    merge(repetitions, filefmt)
+
+
+def main():
+    args = parseArguments()
+    assert args.gatherstats.exists()
+    assert "{}" in args.file
+    run(args.repetitions, args.cases, args.file, args.gatherstats)
+
+
+if __name__ == "__main__":
+    main()