Anchor benchmark on commit (#1649)

Background: #1632

Fix coverage-based benchmarks:
1. Remove `--depth 1` when cloning the benchmark repo.
2. Add the commit hash to the benchmark's `benchmark.yaml`.
3. Add the commit date to the benchmark's `benchmark.yaml`.

---------

Co-authored-by: jonathanmetzman <[email protected]>

.github/workflows/benchmarks.yml

@@ -23,23 +23,23 @@ jobs:
           - curl_curl_fuzzer_http
           - ffmpeg_ffmpeg_demuxer_fuzzer
           - file_magic_fuzzer
-          - freetype2-2017
+          - freetype2_ftfuzzer
           - grok_grk_decompress_fuzzer
-          - harfbuzz-1.3.2
+          - harfbuzz_hb-shape-fuzzer
           - jsoncpp_jsoncpp_fuzzer
           - lcms-2017-03-21
-          - libjpeg-turbo-07-2017
+          - libjpeg-turbo_libjpeg_turbo_fuzzer
           - libpcap_fuzz_both
-          - libpng-1.6.38
-          - libxml2-v2.9.2
+          - libpng_libpng_read_fuzzer
+          - libxml2_xml
           - libxslt_xpath
           - mbedtls_fuzz_dtlsclient
           - mruby-2018-05-23
           - njs_njs_process_script_fuzzer
           - openh264_decoder_fuzzer
           - openssl_x509
-          - openthread-2019-12-23
-          - php_php-fuzz-parser-2020-07-25
+          - openthread_ip6-send-fuzzer
+          - php_php-fuzz-parser
           - proj4-2017-08-14
           - quickjs_eval-2020-01-05
           - re2-2014-12-09

analysis/test_data_utils.py

@@ -46,14 +46,14 @@ def create_experiment_data(experiment='test_experiment',
                            experiment_filestore='gs://fuzzbench-data'):
     """Utility function to create test experiment data."""
     return pd.concat([
-        create_trial_data(0, 'libpng-1.6.38', 'afl', 10, 100, experiment,
-                          experiment_filestore),
-        create_trial_data(1, 'libpng-1.6.38', 'afl', 10, 200, experiment,
-                          experiment_filestore),
-        create_trial_data(2, 'libpng-1.6.38', 'libfuzzer', 10, 200, experiment,
-                          experiment_filestore),
-        create_trial_data(3, 'libpng-1.6.38', 'libfuzzer', 10, 300, experiment,
-                          experiment_filestore),
+        create_trial_data(0, 'libpng_libpng_read_fuzzer', 'afl', 10, 100,
+                          experiment, experiment_filestore),
+        create_trial_data(1, 'libpng_libpng_read_fuzzer', 'afl', 10, 200,
+                          experiment, experiment_filestore),
+        create_trial_data(2, 'libpng_libpng_read_fuzzer', 'libfuzzer', 10, 200,
+                          experiment, experiment_filestore),
+        create_trial_data(3, 'libpng_libpng_read_fuzzer', 'libfuzzer', 10, 300,
+                          experiment, experiment_filestore),
         create_trial_data(4, 'libxml', 'afl', 6 if incomplete else 10, 1000,
                           experiment, experiment_filestore),
         create_trial_data(5, 'libxml', 'afl', 10, 1200, experiment,
@@ -94,18 +94,19 @@ def test_clobber_experiments_data():
     df.reset_index(inplace=True)
 
     to_drop = df[(df.experiment == 'experiment-2') &
-                 (df.benchmark == 'libpng-1.6.38') & (df.fuzzer == 'afl')].index
+                 (df.benchmark == 'libpng_libpng_read_fuzzer') &
+                 (df.fuzzer == 'afl')].index
     df.drop(to_drop, inplace=True)
 
     experiments = list(df['experiment'].drop_duplicates().values)
     df = data_utils.clobber_experiments_data(df, experiments)
 
     columns = ['experiment', 'benchmark', 'fuzzer']
     expected_result = pd.DataFrame([
-        ['experiment-2', 'libpng-1.6.38', 'libfuzzer'],
+        ['experiment-2', 'libpng_libpng_read_fuzzer', 'libfuzzer'],
         ['experiment-2', 'libxml', 'afl'],
         ['experiment-2', 'libxml', 'libfuzzer'],
-        ['experiment-1', 'libpng-1.6.38', 'afl'],
+        ['experiment-1', 'libpng_libpng_read_fuzzer', 'afl'],
     ],
                                    columns=columns)
     expected_result.sort_index(inplace=True)
@@ -123,7 +124,7 @@ def test_filter_fuzzers():
 
 def test_filter_benchmarks():
     experiment_df = create_experiment_data()
-    benchmarks_to_keep = ['libpng-1.6.38']
+    benchmarks_to_keep = ['libpng_libpng_read_fuzzer']
     filtered_df = data_utils.filter_benchmarks(experiment_df,
                                                benchmarks_to_keep)
 
@@ -245,7 +246,10 @@ def test_experiment_summary():
     summary = data_utils.experiment_summary(snapshots_df)
 
     expected_summary = pd.DataFrame({
-        'benchmark': ['libpng-1.6.38', 'libpng-1.6.38', 'libxml', 'libxml'],
+        'benchmark': [
+            'libpng_libpng_read_fuzzer', 'libpng_libpng_read_fuzzer', 'libxml',
+            'libxml'
+        ],
         'fuzzer': ['libfuzzer', 'afl', 'afl', 'libfuzzer'],
         'time': [9, 9, 9, 9],
         'count': [2, 2, 2, 2],
@@ -306,10 +310,26 @@ def test_experiment_pivot_table():
 
     # yapf: disable
     expected_data = pd.DataFrame([
-        {'benchmark': 'libpng-1.6.38', 'fuzzer': 'afl', 'median':  150},
-        {'benchmark': 'libpng-1.6.38', 'fuzzer': 'libfuzzer', 'median':  250},
-        {'benchmark': 'libxml', 'fuzzer': 'afl', 'median': 1100},
-        {'benchmark': 'libxml', 'fuzzer': 'libfuzzer', 'median':  700},
+        {
+            'benchmark': 'libpng_libpng_read_fuzzer',
+            'fuzzer': 'afl',
+            'median':  150
+        },
+        {
+            'benchmark': 'libpng_libpng_read_fuzzer',
+            'fuzzer': 'libfuzzer',
+            'median':  250
+        },
+        {
+            'benchmark': 'libxml',
+            'fuzzer': 'afl',
+            'median': 1100
+        },
+        {
+            'benchmark': 'libxml',
+            'fuzzer': 'libfuzzer',
+            'median':  700
+        },
     ])
     # yapf: enable
     expected_pivot_table = pd.pivot_table(expected_data,

benchmarks/bloaty_fuzz_target/Dockerfile

@@ -24,8 +24,6 @@ RUN apt-get update && \
         libz-dev
 
 RUN git clone \
-    --depth 1 \
-    --branch v1.1\
     https://github.com/google/bloaty.git
 
 WORKDIR bloaty

benchmarks/bloaty_fuzz_target/benchmark.yaml

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commit: 3cf5c3feca15ab88730d6526b2f06302597926b5
-commit_date: 2020-05-25 18:51:34+00:00
+commit: 52948c107c8f81045e7f9223ec02706b19cfa882
+commit_date: 2022-11-11T17:41:21+00:00
 fuzz_target: fuzz_target
 project: bloaty
 unsupported_fuzzers:

benchmarks/freetype2-2017/benchmark.yaml → benchmarks/freetype2_ftfuzzer/benchmark.yaml

@@ -12,5 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+commit: e97544a7390b53bdf899a29c7a739e8056d07b7a
+commit_date: 2023-01-28T16:04:38+00:00
 fuzz_target: ftfuzzer
 project: freetype2

benchmarks/harfbuzz-1.3.2/benchmark.yaml → benchmarks/harfbuzz_hb-shape-fuzzer/benchmark.yaml

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+commit: cb47dca74cbf6d147aac9cf3067f249555aa68b1
+commit_date: 2023-01-29T17:03:52+00:00
 fuzz_target: hb-shape-fuzzer
 project: harfbuzz
 unsupported_fuzzers:

benchmarks/jsoncpp_jsoncpp_fuzzer/Dockerfile

@@ -22,7 +22,7 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5
     chmod +x cmake-3.14.5-Linux-x86_64.sh && \
     ./cmake-3.14.5-Linux-x86_64.sh --skip-license --prefix="/usr/local"
 
-RUN git clone --depth 1 https://github.com/open-source-parsers/jsoncpp
+RUN git clone https://github.com/open-source-parsers/jsoncpp
 WORKDIR jsoncpp
 
 COPY build.sh $SRC/

benchmarks/jsoncpp_jsoncpp_fuzzer/benchmark.yaml

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commit: 3beb37ea14aec1bdce1a6d542dc464d00f4a6cec
-commit_date: 2020-02-13 21:25:08+00:00
+commit: 8190e061bc2d95da37479a638aa2c9e483e58ec6
+commit_date: 2022-07-14T21:57:37+00:00
 fuzz_target: jsoncpp_fuzzer
 project: jsoncpp
 unsupported_fuzzers:

benchmarks/libjpeg-turbo-07-2017/benchmark.yaml → benchmarks/libjpeg-turbo_libjpeg_turbo_fuzzer/benchmark.yaml

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+commit: 3b19db4e6e7493a748369974819b4c5fa84c7614
+commit_date: 2023-01-28T00:24:41+00:00
 fuzz_target: libjpeg_turbo_fuzzer
 project: libjpeg-turbo
 unsupported_fuzzers:

benchmarks/libpcap_fuzz_both/Dockerfile

@@ -14,23 +14,16 @@
 #
 ################################################################################
 
-FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd
 FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd
 RUN apt-get update && \
     apt-get install -y make cmake flex bison
 
-RUN git clone \
-        --depth 1 \
-        --branch libpcap-1.10.1 \
-        https://github.com/the-tcpdump-group/libpcap.git libpcap
+RUN git clone https://github.com/the-tcpdump-group/libpcap.git libpcap
 
 # For corpus as wireshark.
-RUN git clone \
-        --depth=1 \
-        --branch tcpdump-4.99.1 \
-        https://github.com/the-tcpdump-group/tcpdump.git tcpdump
+RUN git clone https://github.com/the-tcpdump-group/tcpdump.git tcpdump && \
+    git -C tcpdump checkout 032e4923e5202ea4d5a6d1cead83ed1927135874
 
 WORKDIR $SRC
 COPY build.sh $SRC/
-COPY patch.diff $SRC/
 

benchmarks/libpcap_fuzz_both/benchmark.yaml

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commit: d615abec7e0237299250c409dca23effb8dd36cc
-commit_date: 2020-02-12 06:46:17+00:00
+commit: 17ff63e88ea99112a905eefc6f862dac20de09e1
+commit_date: 2023-01-07T10:15:39+0000
 fuzz_target: fuzz_both
 project: libpcap
 unsupported_fuzzers:

benchmarks/libpcap_fuzz_both/build.sh

@@ -22,6 +22,7 @@ cd build
 cmake ..
 make
 
+# build fuzz targets
 $CC $CFLAGS -I.. -c ../testprogs/fuzz/fuzz_both.c -o fuzz_both.o
 $CXX $CXXFLAGS fuzz_both.o -o $OUT/fuzz_both libpcap.a $LIB_FUZZING_ENGINE
 

benchmarks/libpng-1.6.38/Dockerfile → benchmarks/libpng_libpng_read_fuzzer/Dockerfile

@@ -30,8 +30,6 @@ RUN git clone \
         https://github.com/madler/zlib.git
 
 RUN git clone \
-        --depth 1 \
-        --branch v1.6.38 \
         https://github.com/glennrp/libpng.git
 RUN cp libpng/contrib/oss-fuzz/build.sh $SRC
 

benchmarks/libpng-1.6.38/benchmark.yaml → benchmarks/libpng_libpng_read_fuzzer/benchmark.yaml

@@ -12,5 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+commit: cd0ea2a7f53b603d3d9b5b891c779c430047b39a
+commit_date: 2023-01-09T13:17:31+00:00
 fuzz_target: libpng_read_fuzzer
 project: libpng

benchmarks/libxml2-v2.9.2/benchmark.yaml → benchmarks/libxml2_xml/benchmark.yaml

@@ -12,5 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+commit: c7260a47f19e01f4f663b6a56fbdc2dafd8a6e7e
+commit_date: 2023-01-23T09:19:59+00:00
 fuzz_target: xml
 project: libxml2

benchmarks/libxslt_xpath/Dockerfile

@@ -57,14 +57,9 @@ RUN apt-get update && \
     )
 
 
-RUN git clone \
-        --branch v2.10.3 \
-        --depth 1 \
-        https://gitlab.gnome.org/GNOME/libxml2.git && \
-    git clone \
-        --branch v1.1.37 \
-        --depth 1 \
-        https://gitlab.gnome.org/GNOME/libxslt.git
+RUN git clone https://gitlab.gnome.org/GNOME/libxml2.git && \
+    git -C libxml2 checkout c7260a47f19e01f4f663b6a56fbdc2dafd8a6e7e && \
+    git clone https://gitlab.gnome.org/GNOME/libxslt.git
 
 ENV ACLOCAL_PATH='/usr/share/aclocal/'
 

benchmarks/libxslt_xpath/benchmark.yaml

@@ -1,5 +1,5 @@
-commit: 7bcc8dca27041e2b03855508fe54d435d816a78f
-commit_date: 2022-09-22 10:59:13+00:00
+commit: 180cdb804efedcba363016fcf6cd3dbd2adca607
+commit_date: 2023-01-18T15:21:36+00:00
 fuzz_target: xpath
 project: libxslt
 unsupported_fuzzers:

benchmarks/mbedtls_fuzz_dtlsclient/Dockerfile

@@ -16,9 +16,15 @@
 
 FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd
 RUN apt-get update && apt-get install -y make cmake
-RUN git clone --recursive --depth 1 https://github.com/ARMmbed/mbedtls.git mbedtls
+RUN apt-get update && apt-get install -y \
+    python-all-dev \
+    python3-all-dev \
+    python3-pip  && \
+    ln -s /usr/local/bin/pip3 /usr/local/bin/pip
+
+RUN git clone --recursive -b development https://github.com/Mbed-TLS/mbedtls.git mbedtls
+
 RUN git clone --depth 1 https://github.com/google/boringssl.git boringssl
 RUN git clone --depth 1 https://github.com/openssl/openssl.git openssl
-RUN git clone https://github.com/ARMmbed/mbed-crypto mbedtls/crypto
 WORKDIR mbedtls
 COPY build.sh $SRC/

benchmarks/mbedtls_fuzz_dtlsclient/benchmark.yaml

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-commit: 4c08dd4e716c0c6d70d78117d3155c2b832f694e
-commit_date: 2020-02-11 08:17:02+00:00
+commit: 169d9e6eb4096cb48aa25651f42b276089841087
+commit_date: 2023-01-27T09:05:00+00:00
 fuzz_target: fuzz_dtlsclient
 project: mbedtls
 unsupported_fuzzers:

benchmarks/mbedtls_fuzz_dtlsclient/build.sh

@@ -15,12 +15,13 @@
 #
 ################################################################################
 
+pip3 install -r $SRC/mbedtls/scripts/basic.requirements.txt
+
 # build project
 perl scripts/config.pl set MBEDTLS_PLATFORM_TIME_ALT
-git -C crypto checkout -f 819799cfc68e4c4381673a8a27af19802c8263f2
 mkdir build
 cd build
-cmake ..
+cmake -DENABLE_TESTING=OFF ..
 # build including fuzzers
 make -j$(nproc) all
 cp programs/fuzz/fuzz_* $OUT/

benchmarks/openh264_decoder_fuzzer/Dockerfile

@@ -23,8 +23,6 @@ RUN dpkg --add-architecture i386 && \
         libstdc++-9-dev libstdc++-9-dev:i386 nasm subversion
 
 RUN git clone \
-        --branch v2.3.1 \
-        --depth 1 \
         https://github.com/cisco/openh264.git
 
 WORKDIR openh264

benchmarks/openh264_decoder_fuzzer/benchmark.yaml

@@ -1,3 +1,5 @@
+commit: 045aeac1dd01df12dec7b1ef8191b3193cf4273c
+commit_date: 2023-01-04T08:01:08+00:00
 fuzz_target: decoder_fuzzer
 project: openh264
 unsupported_fuzzers:

benchmarks/openthread-2019-12-23/benchmark.yaml → benchmarks/openthread_ip6-send-fuzzer/benchmark.yaml

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+commit: 25506997f286fdbfa72725f4cee78c922c896255
+commit_date: 2023-01-26T18:02:16+00:00
 fuzz_target: ip6-send-fuzzer
 project: openthread
 unsupported_fuzzers:

benchmarks/php_php-fuzz-parser-2020-07-25/Dockerfile → benchmarks/php_php-fuzz-parser/Dockerfile

@@ -21,8 +21,6 @@ RUN apt-get update && \
         autoconf automake libtool bison re2c pkg-config
 
 RUN git clone \
-        --depth 1 \
-        --branch php-8.2.0RC6 \
         https://github.com/php/php-src.git
 
 WORKDIR php-src

benchmarks/php_php-fuzz-parser-2020-07-25/benchmark.yaml → benchmarks/php_php-fuzz-parser/benchmark.yaml

@@ -1,3 +1,5 @@
+commit: 908d954de05e592cb2612bdce6ce8661be9dead1
+commit_date: 2023-01-28T19:44:37+0000
 fuzz_target: php-fuzz-parser
 project: php
 unsupported_fuzzers:

benchmarks/stb_stbi_read_fuzzer/Dockerfile

@@ -22,8 +22,6 @@ RUN apt-get update && \
 
 # This project does not have any release/tag.
 RUN git clone \
-        --depth 1 \
-        --branch master \
         https://github.com/nothings/stb.git
 
 RUN mkdir $SRC/stbi # CIFuzz workaround

benchmarks/stb_stbi_read_fuzzer/benchmark.yaml

@@ -1,3 +1,5 @@
+commit: 5736b15f7ea0ffb08dd38af21067c314d6a3aae9
+commit_date: 2023-01-29T18:46:04+00:00
 fuzz_target: stbi_read_fuzzer
 project: stb
 unsupported_fuzzers:

experiment/measurer/test_measure_manager.py

@@ -379,7 +379,7 @@ def test_measure_snapshot_coverage(  # pylint: disable=too-many-locals
         os.environ['WORK'] = str(tmp_path)
         mocked_is_cycle_unchanged.return_value = False
         # Set up the coverage binary.
-        benchmark = 'freetype2-2017'
+        benchmark = 'freetype2_ftfuzzer'
         coverage_binary_src = get_test_data_path(
             'test_measure_snapshot_coverage', benchmark + '-coverage')
         benchmark_cov_binary_dir = os.path.join(