QGOV changes

.ahoy.yml

@@ -0,0 +1,190 @@
+---
+ahoyapi: v2
+
+commands:
+
+  # Docker commands.
+  build:
+    usage: Build or rebuild project.
+    cmd: |
+      ahoy title "Building project"
+      ahoy pre-flight
+      ahoy clean
+      ahoy build-network
+      ahoy up -- --build --force-recreate
+      ahoy install-site
+      ahoy title "Build complete"
+      ahoy doctor
+      ahoy info 1
+
+  build-network:
+    usage: Ensure that the amazeeio network exists.
+    cmd: |
+      docker network prune -f > /dev/null
+      docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
+  info:
+    usage: Print information about this project.
+    cmd: |
+      ahoy line "Project                  : " ${PROJECT}
+      ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
+      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
+
+  up:
+    usage: Build and start Docker containers.
+    cmd: |
+      docker-compose up -d "$@"
+      sleep 10
+      docker-compose logs
+      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m"
+      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
+      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
+      export DOCTOR_CHECK_CLI=0
+
+  down:
+    usage: Stop Docker containers and remove container, images, volumes and networks.
+    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+
+  start:
+    usage: Start existing Docker containers.
+    cmd: docker-compose start "$@"
+
+  stop:
+    usage: Stop running Docker containers.
+    cmd: docker-compose stop "$@"
+
+  restart:
+    usage: Restart all stopped and running Docker containers.
+    cmd: docker-compose restart "$@"
+
+  logs:
+    usage: Show Docker logs.
+    cmd: docker-compose logs "$@"
+
+  pull:
+    usage: Pull latest docker images.
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+
+  cli:
+    usage: Start a shell inside CLI container or run a command.
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+
+  doctor:
+    usage: Find problems with current project setup.
+    cmd: .docker/scripts/doctor.sh "$@"
+
+  install-site:
+    usage: Install a site.
+    cmd: |
+      ahoy title "Installing a fresh site"
+      ahoy cli "./scripts/init.sh"
+
+  clean:
+    usage: Remove containers and all build files.
+    cmd: |
+      ahoy down
+      # Remove other directories.
+      # @todo: Add destinations below.
+      rm -rf \
+        ./ckan
+
+  reset:
+    usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files."
+    cmd: |
+      ahoy clean
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
+      find . -type d -not -path "./.git/*" -empty -delete
+
+  flush-redis:
+    usage: Flush Redis cache.
+    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+
+  lint:
+    usage: Lint code.
+    cmd: |
+      ahoy cli "flake8 ${@:-ckanext}" || \
+      [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
+
+  test-unit:
+    usage: Run unit tests.
+    cmd: |
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI}' || \
+      [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
+
+  test-bdd:
+    usage: Run BDD tests.
+    cmd: |
+      ahoy start-ckan-job-worker &
+      ahoy start-mailmock &
+      sleep 5 &&
+      ahoy cli "behave -k ${*:-test/features}" --tags @smoke && \
+      ahoy cli "behave -k ${*:-test/features}" || \
+      [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      ahoy stop-mailmock
+      ahoy stop-ckan-job-worker
+
+  start-mailmock:
+    usage: Starts email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Starting mailmock'
+      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails --no-stdout' # for debugging mailmock email output remove --no-stdout
+
+  stop-mailmock:
+    usage: Stops email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Stopping mailmock'
+      ahoy cli "killall -2 mailmock"
+
+  start-ckan-job-worker:
+    usage: Starts default CKAN background job worker
+    cmd: |
+      ahoy title 'Starting default CKAN background job worker'
+      ahoy cli "ckan_cli jobs clear && \
+                ckan_cli jobs worker"
+
+  stop-ckan-job-worker:
+    usage: Stops CKAN background job worker
+    cmd: |
+      ahoy title 'Stopping CKAN background job worker'
+      ahoy cli "pkill -f 'jobs worker'"
+
+  # Utilities.
+  title:
+    cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n"
+    hide: true
+
+  line:
+    cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n"
+    hide: true
+
+  getvar:
+    cmd: eval echo "${@}"
+    hide: true
+
+  pre-flight:
+    cmd: |
+      export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1}
+      export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1}
+      export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0}
+      export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1}
+      export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0}
+      export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0}
+      export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0}
+      export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0}
+      ahoy doctor
+    hide: true
+
+entrypoint:
+  - bash
+  - "-c"
+  - "-e"
+  - |
+    export LAGOON_LOCALDEV_URL=http://$PROJECT.docker.amazee.io
+    [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
+    bash -e -c "$0" "$@"
+  - "{{cmd}}"
+  - "{{name}}"

.circleci/build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+##
+# Build site in CI.
+#
+set -e
+
+# Process Docker Compose configuration. This is used to avoid multiple
+# docker-compose.yml files.
+# Remove lines containing '###'.
+sed -i -e "/###/d" docker-compose.yml
+# Uncomment lines containing '##'.
+sed -i -e "s/##//" docker-compose.yml
+
+# Pull the latest images.
+ahoy pull
+
+# Disable checks used on host machine.
+export DOCTOR_CHECK_PYGMY=0
+export DOCTOR_CHECK_PORT=0
+export DOCTOR_CHECK_SSH=0
+export DOCTOR_CHECK_WEBSERVER=0
+export DOCTOR_CHECK_BOOTSTRAP=0
+
+ahoy build || (ahoy logs; exit 1)

.circleci/config.yml

@@ -0,0 +1,43 @@
+version: 2
+aliases:
+
+  # Shared configuration applied to each job.
+  - &container_config
+    working_directory: /app
+    docker:
+      #; Using "runner" container where each job will be executed. This container
+      #; has all necessary tools to run dockerized environment.
+      #; @see https://github.com/integratedexperts/ci-builder
+      - image: integratedexperts/ci-builder
+
+  # Step to setup remote docker.
+  - &step_setup_remote_docker
+      setup_remote_docker
+
+jobs:
+  build:
+    <<: *container_config
+    parallelism: 1
+    steps:
+      - attach_workspace:
+          at: /workspace
+      - checkout
+      - *step_setup_remote_docker
+      - run:
+          command: .circleci/build.sh
+          environment:
+            CKAN_VERSION: 2.9.5
+      - run: .circleci/test.sh
+      - run:
+          name: Process artifacts
+          command: .circleci/process-artifacts.sh
+          when: always
+      - store_artifacts:
+          path: /tmp/artifacts
+          when: always
+
+workflows:
+  version: 2
+  main:
+    jobs:
+      - build

.circleci/process-artifacts.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+##
+# Process test artifacts.
+#
+set -e
+
+# Create screenshots directory in case it was not created before. This is to
+# avoid this script to fail when copying artifacts.
+ahoy cli "mkdir -p test/screenshots"
+
+# Copy from the app container to the build host for storage.
+mkdir -p /tmp/artifacts/behave
+docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/

.circleci/test.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+##
+# Run tests in CI.
+#
+set -e
+
+echo "==> Lint code"
+ahoy lint
+
+echo "==> Run Unit tests"
+ahoy test-unit
+
+echo "==> Run BDD tests"
+ahoy test-bdd || (ahoy logs; exit 1)

.docker/Dockerfile.ckan

@@ -0,0 +1,47 @@
+FROM amazeeio/python:2.7-ckan-21.8.0
+
+ARG SITE_URL=http://ckan:3000/
+ARG CKAN_REPO=ckan/ckan
+ARG CKAN_VERSION=2.8.8
+ENV SITE_URL="${SITE_URL}"
+ENV VENV_DIR=/app/ckan/default
+ENV APP_DIR=/app
+ENV CKAN_INI=/app/ckan/default/production.ini
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache curl build-base \
+    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
+
+# Install CKAN.
+
+RUN . ${VENV_DIR}/bin/activate \
+    && pip install setuptools==36.1 \
+    && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \
+    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
+    && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \
+        pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \
+        pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \
+    && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
+    && deactivate \
+    && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
+    && fix-permissions ${APP_DIR}/ckan
+
+COPY .docker/test.ini $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY .docker/scripts ${APP_DIR}/scripts
+
+COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh ${VENV_DIR}/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/scripts/init-ext.sh
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/app/scripts/serve.sh"]