Merge branch 'develop' of https://github.com/akvo/akvo-rsr into develop

martinchristov · martinchristov · commit a42d7cb3e3bb · 2019-09-23T13:23:21.000+03:00
diff --git a/ci/k8s/deployment.yml b/ci/k8s/deployment.yml
@@ -23,12 +23,12 @@ spec:
         ports:
         - containerPort: 80
           protocol: TCP
-        - containerPort: 8080
+        - containerPort: 8081
           protocol: TCP
         livenessProbe:
           httpGet:
             path: /
-            port: 8080
+            port: 8081
           initialDelaySeconds: 5
           timeoutSeconds: 1
         readinessProbe:
diff --git a/ci/training-envs/delete-env.sh b/ci/training-envs/delete-env.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eu
+set -u
 
 function log {
    echo "$(date +"%T") - INFO - $*"
@@ -11,6 +11,10 @@ disk_name=rsr-${release_name}-1
 
 gcloud container clusters get-credentials test --zone europe-west1-d --project akvo-lumen
 
+log "Maybe deleting dangling snapshot"
+snapshot_name=rsr-${release_name}-1
+gcloud compute snapshots delete ${snapshot_name} --quiet
+
 log "Deleting helm chart..."
 helm delete --purge ${release_name}
 log "Deleting Postgress persistent volume claim..."
diff --git a/ci/training-envs/new-env.sh b/ci/training-envs/new-env.sh
@@ -20,14 +20,19 @@ function read_version () {
     VERSION=$(kubectl get deployments rsr -o jsonpath="{@.spec.template.metadata.labels['rsr-version']}")
 }
 
+function read_report_server_version () {
+    VERSION=$(kubectl get deployments reportserver -o jsonpath="{@.spec.template.metadata.labels['akvo-report-server-version']}")
+}
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 release_name=$1
 restore_from=$2
 rsr_version=${3:-prod}
+report_server_version=${4:-prod}
 HELM_EXTRA_OPTS=""
 
-if [[ ! $release_name =~ ^rsr[1-4]$ ]]; then
+if [[ ! ${release_name} =~ ^rsr[1-4]$ ]]; then
     echo "Release must be one of rsr1,rsr2,rsr3 or rsr4"
     exit 1
 fi
@@ -57,12 +62,28 @@ if [[ ${rsr_version} == "prod" ]]; then
     read_version
     rsr_version=${VERSION}
     switch_cluster "test"
+elif [[ ${rsr_version} == "test" ]]; then
+    read_version
+    rsr_version=${VERSION}
+fi
+
+if [[ ${report_server_version} == "prod" ]]; then
+    switch_cluster "production"
+    read_report_server_version
+    report_server_version=${VERSION}
+    switch_cluster "test"
+elif [[ ${report_server_version} == "test" ]]; then
+    read_report_server_version
+    report_server_version=${VERSION}
 fi
 
 kubectl get pods > /dev/null
 
 log Running helm ...
-helm install . --dep-up --namespace rsr-demo --name ${release_name} --set restoreFrom="${restore_from}" --set rsrVersion="${rsr_version}" ${HELM_EXTRA_OPTS}
+helm install . --dep-up --namespace rsr-demo --name ${release_name} --set restoreFrom="${restore_from}" \
+    --set rsrVersion="${rsr_version}" \
+    --set reportServerVersion="${report_server_version}" \
+    ${HELM_EXTRA_OPTS}
 
 log Waiting for new environment to be ready
 ${DIR}/helpers/wait-for-k8s-deployment-to-be-ready.sh ${release_name}
diff --git a/ci/training-envs/templates/deployment.yaml b/ci/training-envs/templates/deployment.yaml
@@ -23,12 +23,12 @@ spec:
         ports:
         - containerPort: 80
           protocol: TCP
-        - containerPort: 8080
+        - containerPort: 8081
           protocol: TCP
         livenessProbe:
           httpGet:
             path: /
-            port: 8080
+            port: 8081
           initialDelaySeconds: 5
           timeoutSeconds: 1
         readinessProbe:
@@ -48,11 +48,14 @@ spec:
               mountPath: "/var/akvo/rsr/mediaroot"
         env:
         - name: REPORT_SERVER_API_KEY
-          value: "none useful"
+          valueFrom:
+            secretKeyRef:
+              name: "rsr-common"
+              key: report-server-api-key
         - name: ENVIRONMENT
           value: "test"
         - name: REPORT_SERVER_URL
-          value: "http://localhost"
+          value: "http://localhost:8080"
       - name: rsr-backend
         image: "eu.gcr.io/akvo-lumen/rsr-backend:{{ .Values.rsrVersion }}"
         imagePullPolicy: Always
@@ -123,6 +126,55 @@ spec:
             - "echo stats | nc 127.0.0.1 11211 | grep version"
           initialDelaySeconds: 10
           periodSeconds: 5
+      - name: reportserver
+        image: akvo/akvo-reportserver:{{ .Values.reportServerVersion }}
+        ports:
+        - containerPort: 8080
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          initialDelaySeconds: 360
+        env:
+        - name: RS_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_db_user
+        - name: RS_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_db_password
+        - name: RS_DB_NAME
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_db_name
+        - name: RS_DB_HOST
+          value: "{{ include "rsrchart.dbname" . }}"
+        - name: DISABLE_SSL_DB_CONNECTION
+          value: "yes"
+        - name: RS_PBE_SALT
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_pbe_salt
+        - name: RS_PBE_PASSPHRASE
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_pbe_passphrase
+        - name: RS_HMAC_PASSPHRASE
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_hmac_passphrase
+        - name: RS_EXTRA_FONT_URL
+          valueFrom:
+            secretKeyRef:
+              name: rsr-reportserver-training
+              key: rs_extra_font_url
       initContainers:
       - name: copy-assets-to-backend-container
         image: "eu.gcr.io/akvo-lumen/rsr-nginx:{{ .Values.rsrVersion }}"
diff --git a/ci/training-envs/templates/seedDatabase.yaml b/ci/training-envs/templates/seedDatabase.yaml
@@ -37,8 +37,8 @@ data:
       log Assign perms to RSR user in RSR DB
       psql "${psql_settings[@]}" --dbname="${RSR_DB_NAME}" --command="ALTER SCHEMA public OWNER TO ${RSR_DB_USER};"
 
+      DUMP_FILE=/tmp/backup.lzo
       if [[ "{{.Values.restoreFrom}}" == "prod" ]]; then
-        DUMP_FILE=/tmp/backup.lzo
 
         log Downloading latest production RSR backup
         curl -u :${ELEPHANTSQL_API_KEY}  https://api.elephantsql.com/api/backup?db=rsr_db | jq .[0].url | xargs curl --output "${DUMP_FILE}"
@@ -67,7 +67,55 @@ data:
         log Deleting tmp files
         rm /tmp/cleaned.dump
         rm $DUMP_FILE
+
       fi
 
+      log Creating ReportServer read-only user
+      REPORTSERVER_DB_USER=${REPORTSERVER_DB_USER}
+      REPORTSERVER_USER_PASSWORD=${REPORTSERVER_USER_PASSWORD}
+      REPORTSERVER_DB_NAME=${REPORTSERVER_DB_NAME}
+
+      psql_settings=("--username=${SUPER_USER}" "--host=${DB_HOST}" "--set" "ON_ERROR_STOP=on")
+      psql "${psql_settings[@]}" --command="CREATE USER ${REPORTSERVER_DB_USER} WITH ENCRYPTED PASSWORD '${REPORTSERVER_USER_PASSWORD}';"
+      log Assinging read-only permissions to ReportServer user
+      psql_settings=("--username=${SUPER_USER}" "--host=${DB_HOST}" "--dbname=${RSR_DB_NAME}" "--set" "ON_ERROR_STOP=on")
+      psql "${psql_settings[@]}" --command="GRANT USAGE ON SCHEMA public TO ${REPORTSERVER_DB_USER};"
+      psql "${psql_settings[@]}" --command="GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${REPORTSERVER_DB_USER};"
+      psql "${psql_settings[@]}" --command="ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${REPORTSERVER_DB_USER};"
+
+      log Creating ReportServer DB
+      psql_settings=("--username=${SUPER_USER}" "--host=${DB_HOST}" "--set" "ON_ERROR_STOP=on")
+      psql "${psql_settings[@]}" --command="CREATE DATABASE ${REPORTSERVER_DB_NAME} OWNER ${REPORTSERVER_DB_USER};"
+      log Assign perms to ReportServer user in ReportServer DB
+      psql "${psql_settings[@]}" --dbname="${REPORTSERVER_DB_NAME}" --command="ALTER SCHEMA public OWNER TO ${REPORTSERVER_DB_USER};"
+
+      log Downloading latest production ReportServer backup
+      curl -u :${ELEPHANTSQL_API_KEY}  https://api.elephantsql.com/api/backup?db=reportserver | jq .[0].url | xargs curl --output "${DUMP_FILE}"
+      log Preparing ReportServer DB dump
+      dump_owner=reportserver
+      lzop -cd "${DUMP_FILE}" | \
+        sed -e '/COPY public.rs_audit_log_property/,/^--/d' | \
+        sed -e "s/${dump_owner};\$/${REPORTSERVER_DB_USER};/" | \
+        sed -e "/^GRANT/d" | sed -e "/ALTER DEFAULT PRIVILEGES/d" | \
+        sed -e '/spatial_ref_sys/,+1 d' | \
+        sed -e '/NOT EXISTS plv8/d' | sed -e '/ON EXTENSION plv8/d' | \
+        sed -e '/NOT EXISTS "uuid-ossp/d' | sed -e '/ON EXTENSION "uuid-ossp/d' | \
+        sed -e '/NOT EXISTS postgis/d' | sed -e '/ON EXTENSION postgis/d' \
+        > /tmp/cleaned.dump
+
+      log Seeding ReportServer DB
+      psql_settings=("--username=${SUPER_USER}" "--host=${DB_HOST}" "--dbname=${REPORTSERVER_DB_NAME}" "--set" "ON_ERROR_STOP=on")
+      cat /tmp/cleaned.dump | psql "${psql_settings[@]}"
+      log Deleting ReportServer tmp files
+      rm /tmp/cleaned.dump
+      rm $DUMP_FILE
+
+      log Changing credentials of ReportServer datasource
+      ## Removing the internal datasource so that it doesnt point to production
+      db_external_host=$(echo $HOSTNAME | cut -f1-2 -d\-)
+      psql "${psql_settings[@]}" --command="UPDATE rs_database_datasource SET username='none', password='none' WHERE username='$dump_owner'"
+      psql "${psql_settings[@]}" --command="UPDATE rs_database_datasource SET username='${REPORTSERVER_DB_USER}', password='${ENCRYPTED_RSR_PASSWORD}', url='jdbc:postgresql://${db_external_host}:5432/${RSR_DB_NAME}' WHERE username='reportserver_user'"
+
+
       log Done
       exit 0
diff --git a/ci/training-envs/values.yaml b/ci/training-envs/values.yaml
@@ -14,4 +14,24 @@ postgresql:
         secretKeyRef:
           name: "rsr-common"
           key: elephantsql-api-key
+    - name: REPORTSERVER_USER_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: rsr-reportserver-training
+          key: rs_db_password
+    - name: REPORTSERVER_DB_USER
+      valueFrom:
+        secretKeyRef:
+          name: rsr-reportserver-training
+          key: rs_db_user
+    - name: REPORTSERVER_DB_NAME
+      valueFrom:
+        secretKeyRef:
+          name: rsr-reportserver-training
+          key: rs_db_name
+    - name: ENCRYPTED_RSR_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: rsr-reportserver-training
+          key: rs_rsr_db_encrypted_password
   initdbScriptsConfigMap: "{{ include \"rsrchart.fullname\" . }}-seed-db"
diff --git a/nginx/default.conf b/nginx/default.conf
@@ -1,5 +1,5 @@
 server {
-  listen 8080;
+  listen 8081;
   server_name health;
   location / {
     stub_status on;
