Add default end to end test config file for testing; prepare for conf…

…ig validation refactoring (#550)

The initial version for enqueuing and working image building jobs in the new arch.

.github/workflows/presubmit.yml

@@ -37,4 +37,7 @@ jobs:
     - name: Run presubmit checks
       run: |
         FUZZBENCH_TEST_INTEGRATION=1 make presubmit
-    # TODO(zhichengcai): Add back end to end test.
+
+    - name: Run end to end CI test
+      run: |
+        make run-end-to-end-test

common/config_utils.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Provides helper functions to obtain configurations."""
+
+
+def validate_and_expand(config):
+    """Validates |config| and returns the expanded configuration."""
+    # TODO: move the logic from experiment/run_experiment.py to here.
+    return config

compose/e2e-test.yaml

@@ -8,4 +8,8 @@ services:
       - queue-server
     environment:
       E2E_INTEGRATION_TEST: 1
-    command: python3 -m pytest -vv fuzzbench/test_e2e_run.py
+    command: python3 -m pytest -vv fuzzbench/test_e2e/test_e2e_run.py
+
+  run-experiment:
+    environment:
+      EXPERIMENT_CONFIG: fuzzbench/test_e2e/end-to-end-test-config.yaml

docker/fuzzbench/Dockerfile

@@ -31,6 +31,8 @@ COPY benchmarks benchmarks
 COPY common common
 COPY database database
 COPY docker docker
+COPY experiment/build experiment/build
+COPY experiment/*.py experiment/
 COPY fuzzbench fuzzbench
 COPY fuzzers fuzzers
 

docker/image_types.yaml

@@ -15,6 +15,15 @@
   tag: 'dispatcher-image'
   type: 'dispatcher'
 
+# TODO: It would be better to call this benchmark builder. But that would be
+# confusing because this doesn't involve benchmark-builder/Dockerfile. Rename
+# that and then rename this.
+'{benchmark}-project-builder':
+  dockerfile: 'benchmarks/{benchmark}/Dockerfile'
+  context: 'benchmarks/{benchmark}'
+  tag: 'builders/benchmark/{benchmark}'
+  type: 'builder'
+
 'coverage-{benchmark}-builder-intermediate':
   build_arg:
     - 'parent_image=gcr.io/fuzzbench/builders/benchmark/{benchmark}'
@@ -37,15 +46,6 @@
   tag: 'builders/coverage/{benchmark}'
   type: 'coverage'
 
-# TODO: It would be better to call this benchmark builder. But that would be
-# confusing because this doesn't involve benchmark-builder/Dockerfile. Rename
-# that and then rename this.
-'{benchmark}-project-builder':
-  dockerfile: 'benchmarks/{benchmark}/Dockerfile'
-  context: 'benchmarks/{benchmark}'
-  tag: 'builders/benchmark/{benchmark}'
-  type: 'builder'
-
 '{fuzzer}-{benchmark}-builder-intermediate':
   build_arg:
     - 'parent_image=gcr.io/fuzzbench/builders/benchmark/{benchmark}'

fuzzbench/jobs.py

@@ -19,14 +19,19 @@
 BASE_TAG = 'gcr.io/fuzzbench'
 
 
-def build_image(name: str):
+def build_image(image):
     """Builds a Docker image and returns whether it succeeds."""
-    image_tag = os.path.join(BASE_TAG, name)
+    image_tag = os.path.join(BASE_TAG, image['tag'])
     subprocess.run(['docker', 'pull', image_tag], check=True)
-    subprocess.run(
-        ['docker', 'build', '--tag', image_tag,
-         os.path.join('docker', name)],
-        check=True)
+    command = ['docker', 'build', '--tag', image_tag, image['context']]
+    cpu_options = ['--cpu-period', '100000', '--cpu-quota', '100000']
+    command.extend(cpu_options)
+    if 'dockerfile' in image:
+        command.extend(['--file', image['dockerfile']])
+    if 'build_arg' in image:
+        for arg in image['build_arg']:
+            command.extend(['--build-arg', arg])
+    subprocess.run(command, check=True)
     return True
 
 

fuzzbench/local-experiment-config.yaml

@@ -0,0 +1,6 @@
+benchmarks:
+    - freetype2-2017
+    - bloaty_fuzz_target
+fuzzers:
+    - afl
+    - libfuzzer

fuzzbench/run_experiment.py

@@ -18,20 +18,34 @@
 import redis
 import rq
 
+from common import config_utils, environment, yaml_utils
+from experiment.build import docker_images
 from fuzzbench import jobs
 
 
-def run_experiment():
+def run_experiment(config):
     """Main experiment logic."""
     print('Initializing the job queue.')
     # Create the queue for scheduling build jobs and run jobs.
     queue = rq.Queue('build_n_run_queue')
+
+    images_to_build = docker_images.get_images_to_build(config['fuzzers'],
+                                                        config['benchmarks'])
     jobs_list = []
-    jobs_list.append(
-        queue.enqueue(jobs.build_image,
-                      'base-image',
-                      job_timeout=600,
-                      job_id='base-image'))
+    # TODO(#643): topological sort before enqueuing jobs.
+    for name, image in images_to_build.items():
+        depends = image.get('depends_on', None)
+        if depends is not None:
+            assert len(depends) == 1, 'image %s has %d dependencies. Multiple '\
+            'dependencies are currently not supported.' % (name, len(depends))
+        jobs_list.append(
+            queue.enqueue(
+                jobs.build_image,
+                image=image,
+                job_timeout=30 * 60,
+                result_ttl=-1,
+                job_id=name,
+                depends_on=depends[0] if 'depends_on' in image else None))
 
     while True:
         print('Current status of jobs:')
@@ -52,8 +66,14 @@ def run_experiment():
 def main():
     """Set up Redis connection and start the experiment."""
     redis_connection = redis.Redis(host="queue-server")
+
+    config_path = environment.get('EXPERIMENT_CONFIG',
+                                  'fuzzbench/local-experiment-config.yaml')
+    config = yaml_utils.read(config_path)
+    config = config_utils.validate_and_expand(config)
+
     with rq.Connection(redis_connection):
-        return run_experiment()
+        return run_experiment(config)
 
 
 if __name__ == '__main__':

fuzzbench/test_e2e/end-to-end-test-config.yaml

@@ -0,0 +1,4 @@
+benchmarks:
+    - bloaty_fuzz_target
+fuzzers:
+    - libfuzzer

fuzzbench/test_e2e_run.py → fuzzbench/test_e2e/test_e2e_run.py

@@ -19,7 +19,17 @@
 
 import pytest
 import redis
-from rq.job import Job
+import rq
+
+from common import config_utils, yaml_utils
+from experiment.build import docker_images
+
+
+@pytest.fixture(scope='class')
+def experiment_config():
+    """Returns the default configuration for end-to-end testing."""
+    return config_utils.validate_and_expand(
+        yaml_utils.read('fuzzbench/test_e2e/end-to-end-test-config.yaml'))
 
 
 @pytest.fixture(scope='class')
@@ -31,17 +41,32 @@ def redis_connection():
 # pylint: disable=no-self-use
 @pytest.mark.skipif('E2E_INTEGRATION_TEST' not in os.environ,
                     reason='Not running end-to-end test.')
-@pytest.mark.usefixtures('redis_connection')
+@pytest.mark.usefixtures('redis_connection', 'experiment_config')
 class TestEndToEndRunResults:
     """Checks the result of a test experiment run."""
 
-    def test_jobs_dependency(self):  # pylint: disable=redefined-outer-name
+    def test_jobs_dependency(self, experiment_config, redis_connection):  # pylint: disable=redefined-outer-name
         """Tests that jobs dependency preserves during working."""
-        assert True
+        all_images = docker_images.get_images_to_build(
+            experiment_config['fuzzers'], experiment_config['benchmarks'])
+        jobs = {
+            name: rq.job.Job.fetch(name, connection=redis_connection)
+            for name in all_images
+        }
+        for name, image in all_images.items():
+            if 'depends_on' in image:
+                for dep in image['depends_on']:
+                    assert jobs[dep].ended_at <= jobs[name].started_at
 
-    def test_all_jobs_finished_successfully(self, redis_connection):  # pylint: disable=redefined-outer-name
+    def test_all_jobs_finished_successfully(
+            self,
+            experiment_config,  # pylint: disable=redefined-outer-name
+            redis_connection):  # pylint: disable=redefined-outer-name
         """Tests all jobs finished successully."""
-        jobs = Job.fetch_many(['base-image'], connection=redis_connection)
+        all_images = docker_images.get_images_to_build(
+            experiment_config['fuzzers'], experiment_config['benchmarks'])
+        jobs = rq.job.Job.fetch_many(all_images.keys(),
+                                     connection=redis_connection)
         for job in jobs:
             assert job.get_status() == 'finished'