Initial docker image creation workflows.

aarlt · aarlt · commit e4f1f3e44553 · 2020-07-14T18:33:35.000-05:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -33,26 +33,11 @@ defaults:
 
   - run_build: &run_build
       name: Build
-      command: |
-        set -ex
-        if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n > prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" > prerelease.txt; fi
-        echo -n "$CIRCLE_SHA1" > commit_hash.txt
-        mkdir -p build
-        cd build
-        [ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON"
-        cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS -G "Unix Makefiles"
-        make -j4
+      command: scripts/ci/build.sh
 
   - run_build_ossfuzz: &run_build_ossfuzz
       name: Build_ossfuzz
-      command: |
-        mkdir -p build
-        cd build
-        protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz
-        protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz
-        protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz
-        cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS
-        make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j4
+      command: scripts/ci/build_ossfuzz.sh
 
   - run_proofs: &run_proofs
       name: Correctness proofs for optimization rules
diff --git a/.github/workflows/buildpack-deps.yml b/.github/workflows/buildpack-deps.yml
@@ -0,0 +1,42 @@
+name: buildpack-deps
+
+on:
+  pull_request:
+    branches: [ develop ]
+    paths:
+      - 'scripts/docker/buildpack-deps/Dockerfile.emscripten'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1804'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004'
+
+jobs:
+  buildpack-deps:
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      DOCKER_REPOSITORY: docker.pkg.github.com/${{ github.repository }}
+      IMAGE_NAME: buildpack-deps
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image_variant: [emscripten, ubuntu1604.clang.ossfuzz, ubuntu1804, ubuntu2004.clang, ubuntu2004]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Upgrade ${{ env.IMAGE_NAME }}-${{ matrix.image_variant }}
+        run: |
+          echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_ACTOR}" --password-stdin
+          scripts/ci/docker_upgrade.sh ${{ env.IMAGE_NAME }} ${{ matrix.image_variant }} ${{ env.DOCKER_REPOSITORY }}
+          docker logout docker.pkg.github.com
+
+      - name: comment PR
+        if: "env.DOCKER_IMAGE"
+        uses: aarlt/comment-on-pr@v1.2.0
+        with:
+          msg: "`${{ env.DOCKER_IMAGE }} ${{ env.DOCKER_REPO_DIGEST }}`."
diff --git a/.gitignore b/.gitignore
@@ -32,7 +32,7 @@ prerelease.txt
 
 # Build directory
 build/
-build*/
+/build*/
 emscripten_build/
 docs/_build
 __pycache__
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -ex
+
+ROOTDIR="$(dirname "$0")/../.."
+cd "${ROOTDIR}"
+
+# shellcheck disable=SC2166
+if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n >prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" >prerelease.txt; fi
+if [ -n "$CIRCLE_SHA1" ]
+then
+  echo -n "$CIRCLE_SHA1" >commit_hash.txt
+fi
+
+mkdir -p build
+cd build
+
+# shellcheck disable=SC2166
+[ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON"
+
+# shellcheck disable=SC2086
+cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" $CMAKE_OPTIONS -G "Unix Makefiles"
+
+make -j 4
diff --git a/scripts/ci/build_ossfuzz.sh b/scripts/ci/build_ossfuzz.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -e
+
+ROOTDIR="$(dirname "$0")/../.."
+BUILDDIR="${ROOTDIR}/build"
+
+mkdir -p "${BUILDDIR}"
+cd "${BUILDDIR}"
+
+protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz
+protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz
+protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz
+cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/libfuzzer.cmake
+
+make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j 4
diff --git a/scripts/ci/buildpack-deps_test_emscripten.sh b/scripts/ci/buildpack-deps_test_emscripten.sh
@@ -0,0 +1 @@
+../../scripts/travis-emscripten/build_emscripten.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh b/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh
@@ -0,0 +1 @@
+build_ossfuzz.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu1804.sh b/scripts/ci/buildpack-deps_test_ubuntu1804.sh
@@ -0,0 +1 @@
+build.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh
@@ -0,0 +1 @@
+build.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.sh
@@ -0,0 +1 @@
+build.sh
diff --git a/scripts/ci/docker_upgrade.sh b/scripts/ci/docker_upgrade.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -e
+
+function error() {
+  echo >&2 "ERROR: ${1} Aborting." && false
+}
+
+function warning() {
+  echo >&2 "WARNING: ${1}"
+}
+
+[[ $# == 3 ]] || error "Expected exactly 3 parameters: '${0} <IMAGE_NAME> <IMAGE_VARIANT> <DOCKER_REPOSITORY>'."
+
+IMAGE_NAME="${1}"
+IMAGE_VARIANT="${2}"
+DOCKER_REPOSITORY="${3}"
+DOCKERFILE="scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}"
+
+echo "-- check_dockerfile_was_changed"
+
+# exit, if the dockerfile was not changed.
+if git diff --quiet origin/develop HEAD -- "${DOCKERFILE}"; then
+  echo "${DOCKERFILE} was not changed. Nothing to do."
+  exit 0
+fi
+
+echo "-- check_version"
+
+PREV_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*-LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }')
+NEXT_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*+LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }')
+
+[[ $NEXT_VERSION != "" ]] || error "No version label defined in Dockerfile. You may need to add 'LABEL version' in '${DOCKERFILE}'."
+
+[[ $PREV_VERSION != "" ]] || {
+  warning "no previous version found. Will set \$PREV_VERSION = 0."
+  PREV_VERSION=0
+}
+
+if [[ $((PREV_VERSION + 1)) != $((NEXT_VERSION)) ]]; then
+  error "Version label in Dockerfile was not incremented. You may need to change 'LABEL version' in '${DOCKERFILE}'."
+fi
+
+echo "-- build_docker"
+
+# This is a workaround: we run `docker build` twice to prevent the `layer does not exist` problem.
+# See https://github.com/moby/moby/issues/37965.
+docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}" ||
+  docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}"
+
+echo "-- test_docker @ '${PWD}'"
+
+docker run --rm --volume "${PWD}:/root/project" "${IMAGE_NAME}" "/root/project/scripts/ci/${IMAGE_NAME}_test_${IMAGE_VARIANT}.sh"
+
+echo "-- push_docker"
+
+VERSION=$(docker inspect --format='{{.Config.Labels.version}}' "${IMAGE_NAME}")
+DOCKER_IMAGE_ID="${DOCKER_REPOSITORY}/${IMAGE_NAME}-${IMAGE_VARIANT}"
+
+docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:${VERSION}"
+docker push "${DOCKER_IMAGE_ID}:${VERSION}"
+
+REPO_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "${DOCKER_IMAGE_ID}:${VERSION}")
+
+docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:latest"
+docker push "${DOCKER_IMAGE_ID}:latest"
+
+echo "::set-env name=DOCKER_IMAGE::${DOCKER_IMAGE_ID}:${VERSION}"
+echo "::set-env name=DOCKER_REPO_DIGEST::${REPO_DIGEST}"
diff --git a/scripts/docker/buildpack-deps/Dockerfile.emscripten b/scripts/docker/buildpack-deps/Dockerfile.emscripten
@@ -29,6 +29,7 @@
 #   make version=1.39.15 build
 #
 FROM emscripten/emsdk:1.39.15 AS base
+LABEL version="0"
 
 ADD emscripten.jam /usr/src
 RUN set -ex; \
@@ -63,3 +64,4 @@ RUN set -ex; \
 		cxxflags="-s DISABLE_EXCEPTION_CATCHING=0 -Wno-unused-local-typedef -Wno-variadic-macros -Wno-c99-extensions -Wno-all" \
 	       --prefix=/emsdk/emscripten/sdk/system install; \
 	rm -r /usr/src/boost_1_73_0
+
diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM gcr.io/oss-fuzz-base/base-clang as base
+LABEL version="0"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -99,3 +100,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu1804
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:bionic AS base
+LABEL version="0"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -91,3 +92,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:focal AS base
+LABEL version="0"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -60,3 +61,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:focal AS base
+LABEL version="0"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -62,3 +63,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/scripts/docker/buildpack-deps/README.md b/scripts/docker/buildpack-deps/README.md
@@ -0,0 +1,35 @@
+# buildpack-deps docker images
+
+The `buildpack-deps` docker images are used to compile and test solidity within our CI.
+
+## GitHub Workflow
+
+The creation of the images are triggered by a single workflow, defined in `.github/workflows/buildpack-deps.yml`.
+For each resulting `buildpack-deps` docker image a strategy is defined in the workflow file - the image variant.
+The workflow gets triggered, if any Dockerfile defined in `scripts/docker/buildpack-deps/Dockerfile.*` were changed
+within the PR.
+
+### Versioning
+
+The version of the docker images can be defined within the Dockerfile with `LABEL version`. A new docker image
+will only be created and pushed, if the new version is incremented by `1` compared with the version of the Dockerfile
+located in `develop`.
+
+### Build, Test & Push
+
+Note that the whole workflow - including all defined strategies (image variants) - will be triggered,
+even if only a single Dockerfile was change. The full workflow will only gets executed, if the corresponding
+Dockerfile was changed. The execution of workflows of unchanged Dockerfiles will not continue and just return success. 
+See `scripts/ci/docker_upgrade.sh`.
+
+If the version check was successful, the docker image will be built using the Dockerfile located in
+`scripts/docker/buildpack-deps/Dockerfile.*`.
+
+The resulting docker image will be tested by executing
+the corresponding `scripts/ci/buildpack-deps_test_*` scripts. These scripts are normally symlinked to `scripts/ci/build.sh`,
+except for the `buildpack-deps-ubuntu1604.clang.ossfuzz` docker image, that is symlinked to `scripts/ci/build_ossfuzz.sh`.
+These scripts `scripts/ci/build.sh` and `scripts/ci/build_ossfuzz.sh` are also used by CircleCI, see `.circleci/config.yml`.
+
+If the tests passed successfully, the docker image will get tagged by the version defined within the corresponding `Dockerfile`.
+Finally, a comment will be added to the PR that contains the full repository, version and repository digest
+of the freshly created docker image.
diff --git a/scripts/docker/buildpack-deps/emscripten.jam b/scripts/docker/buildpack-deps/emscripten.jam

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+../../scripts/travis-emscripten/build_emscripten.sh`