diff --git a/.github/workflows/dockerhub_ci.yml b/.github/workflows/dockerhub_ci.yml new file mode 100644 index 0000000..4434fe6 --- /dev/null +++ b/.github/workflows/dockerhub_ci.yml @@ -0,0 +1,115 @@ +name: Dockerhub CI + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + # Triggers every week on tuesday night + - cron: '0 0 * * 2' + +env: + # Specify which image to build + DOCKER_REPO: godatadriven/airflow + +jobs: + build: + name: build + runs-on: ubuntu-latest + strategy: + matrix: + # Specify which tags to build + DOCKER_TAG: ["2.1.0-python3.7,2.1.0-python3,2.1-python3,python3,latest","2.0.2-python3.7,2.0.2-python3,2.0-python3","1.10.14-python3.7,1.10.14-python3,1.10-python3", "1.10.14-python2.7,1.10.14-python2,1.10-python2,python2", "master-python3.7"] + + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Parse args + id: args + run: | + DOCKER_TAG="${{ matrix.DOCKER_TAG }}" + IMAGE_TAG="${DOCKER_TAG%%,*}" + + # Manipulate DOCKER_TAG to create build args + AIRFLOW_VERSION=${IMAGE_TAG%-*} + PYTHON_TAG=${IMAGE_TAG#*-} + PYTHON_VERSION=${PYTHON_TAG:6} + MINICONDA_VERSION=${PYTHON_VERSION:0:1} + + if [ ${MINICONDA_VERSION} = "3" ]; then + AIRFLOW_EXTRAS=async,celery,crypto,jdbc,hdfs,hive,azure,emr,password,postgres,slack,ssh,kubernetes + else + AIRFLOW_EXTRAS=async,celery,crypto,jdbc,hdfs,hive,azure,gcp_api,emr,password,postgres,slack,ssh,kubernetes + fi + + BUILD_ARGS="PYTHON_VERSION=$PYTHON_VERSION + AIRFLOW_VERSION=$AIRFLOW_VERSION + AIRFLOW_EXTRAS=$AIRFLOW_EXTRAS" + + # No modification needed beyond this point + BUILD_ARGS="${BUILD_ARGS//'%'/'%25'}" + BUILD_ARGS="${BUILD_ARGS//$'\n'/'%0A'}" + BUILD_ARGS="${BUILD_ARGS//$'\r'/'%0D'}" + + echo "::set-output name=build_args::$BUILD_ARGS" + + - name: Prepare + id: prep + run: | + DOCKER_TAG="${{ matrix.DOCKER_TAG }}" + DOCKER_IMAGE_NAME="$DOCKER_REPO:${DOCKER_TAG%%,*}" + echo ::set-output name=image_name::${DOCKER_IMAGE_NAME} + + TAGS="$DOCKER_REPO:${DOCKER_TAG//,/,$DOCKER_REPO:}" + echo ::set-output name=tags::${TAGS} + + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + + - name: Build image + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: false + tags: ${{ steps.prep.outputs.tags }} + build-args: | + ${{ steps.args.outputs.build_args }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Test image + env: + IMAGE_NAME: ${{ steps.prep.outputs.image_name }} + run: | + if [[ -f "docker-compose.test.yml" ]]; then + docker-compose --file docker-compose.test.yml build + docker-compose --file docker-compose.test.yml run sut + fi + + - name: Login to DockerHub + if: github.ref == 'refs/heads/master' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB }} + + - name: Push image + if: github.ref == 'refs/heads/master' + run: | + docker push --all-tags $DOCKER_REPO + + - name: Report Status + if: always() && github.ref == 'refs/heads/master' + uses: ravsamhq/notify-slack-action@master + with: + status: ${{ job.status }} + notify_when: 'failure' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GDD_GENERAL_WEBHOOK }} diff --git a/Dockerfile b/Dockerfile index f6989a5..c33fccc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,10 +14,7 @@ LABEL org.label-schema.name="Apache Airflow ${AIRFLOW_VERSION}" \ RUN set -x\ && apt-get update \ - && apt-get install -y gcc g++ netcat git ca-certificates libpq-dev curl procps --no-install-recommends \ - && if [ "$AIRFLOW_VERSION" = "1.8.2" ]; then\ - conda install -y pip==9;\ - fi\ + && apt-get install -y gcc g++ netcat git ca-certificates libpq-dev curl procps libsasl2-dev --no-install-recommends \ && if [ "$AIRFLOW_VERSION" = "master" ]; then\ pip install --no-cache-dir git+https://github.com/apache/airflow/#egg=apache-airflow[$AIRFLOW_EXTRAS];\ curl -sL https://deb.nodesource.com/setup_8.x | bash - ;\ @@ -27,8 +24,6 @@ RUN set -x\ npm run prod;\ cd /;\ apt-get remove -y --purge nodejs ;\ - elif [ "$AIRFLOW_VERSION" = "1.9.0" ]; then\ - pip install --no-cache-dir apache-airflow[$AIRFLOW_EXTRAS]==$AIRFLOW_VERSION "werkzeug<1.0.0";\ elif [ -n "$AIRFLOW_VERSION" ]; then\ pip install --no-cache-dir apache-airflow[$AIRFLOW_EXTRAS]==$AIRFLOW_VERSION;\ else\ diff --git a/hooks/build b/hooks/build deleted file mode 100644 index ad23ab2..0000000 --- a/hooks/build +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# $IMAGE_NAME var is injected into the build so the tag is correct. -IMAGE_REPO=${IMAGE_NAME%:*} -IMAGE_TAG=${IMAGE_NAME#*:} - -AIRFLOW_VERSION=${IMAGE_TAG%-*} -PYTHON_TAG=${IMAGE_TAG#*-} -PYTHON_VERSION=${PYTHON_TAG:6} -MINICONDA_VERSION=${PYTHON_VERSION:0:1} - -if [ ${MINICONDA_VERSION} = "3" ]; then - AIRFLOW_EXTRAS=async,celery,crypto,jdbc,hdfs,hive,azure,emr,password,postgres,slack,ssh,kubernetes -else - AIRFLOW_EXTRAS=async,celery,crypto,jdbc,hdfs,hive,azure,gcp_api,emr,password,postgres,slack,ssh,kubernetes -fi - -# Trigger build -docker build --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ - --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg AIRFLOW_VERSION=$AIRFLOW_VERSION \ - --build-arg AIRFLOW_EXTRAS=$AIRFLOW_EXTRAS \ - -t $DOCKER_REPO:${DOCKER_TAG//,/ -t $DOCKER_REPO:} .