Run ciux-based CI (#761)

* Run ciux-based CI - Hack to install go module in v3 - Increase wait time for topics retrieval in CI - Use ciux to install itest deps * Use latest finkctl in gha * Upgrade fink-broker libraries Downgrade kind Update sparks libs to 3.4.1 Check output topics are created on self-hosted CI Bump common-pool jar to 2-2.11.1 Manage spark_py image with ciux Bump ciux to rc11 Use ciux labels
astrolabsoftware · Dec 18, 2023 · 3e6c2b9 · 3e6c2b9
1 parent 9222222
commit 3e6c2b9
Show file tree

Hide file tree

Showing 11 changed files with 109 additions and 106 deletions.
diff --git a/.ciux b/.ciux
@@ -0,0 +1,24 @@
+apiVersion: v1alpha1
+registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink
+dependencies:
+  - url: https://github.com/astrolabsoftware/fink-alert-simulator
+    # If true repository will be locally cloned
+    clone: true
+    # If true, container image will be pulled
+    pull: true
+    labels:
+      itest: "true"
+      ci: "true"
+  - url: https://github.com/astrolabsoftware/finkctl
+    clone: true
+    labels:
+      itest: "true"
+      ci: "true"
+  - image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1
+    labels:
+      build: "true"
+  - package: github.com/k8s-school/[email protected]
+    labels:
+      itest: "optional"
+      ci: "true"
+
diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/itest-gha.yml
@@ -5,10 +5,11 @@ on:
     branches:
       - master
 env:
-  FINKCTL_VERSION: v1.1.0-rc2
+  CIUXCONFIG: /tmp/ciux.sh
+  CIUX_VERSION: v0.0.1-rc11
+  GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   NOSCIENCE: true
   MINIMAL: true
-
 jobs:
   build:
     name: Build image
@@ -18,24 +19,23 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Declare Version Variables
-        id: vars
-        shell: bash
-        run: |
-          DIR=$(pwd)
-          . ./conf.sh
-          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
-          echo "IMAGE=${IMAGE}"
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Ciux project ignition
+        run: ciux ignite $PWD
       - name: Build fink-broker image for k8s
         run: |
           ./build.sh
       - name: Export fink-broker image
         run: |
           docker images
-          echo "${{ env.IMAGE }}"
+          . ./conf.sh
           mkdir -p artifacts
-          docker save "${{ env.IMAGE }}" > artifacts/image.tar
-          echo "${{ env.IMAGE }}" > artifacts/image-tag
+          docker save "$IMAGE" > artifacts/image.tar
+          echo "$IMAGE" > artifacts/image-tag
       - uses: actions/upload-artifact@v2
         with:
           name: docker-artifact
@@ -49,21 +49,23 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Declare Version Variables
-        id: vars
-        shell: bash
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Ciux project ignition
         run: |
-          echo "FINK_ALERT_SIMULATOR_DIR=$(. ./conf.sh && echo $FINK_ALERT_SIMULATOR_DIR)" >> $GITHUB_ENV
-      - name: Clone fink-alert-simulator code
-        run: ./itest/clone-fink-alert-simulator.sh
+          ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD
       - name: Create k8s/kind cluster
         run: |
-          git clone --depth 1 -b "k8s-1.25.0" --single-branch https://github.com/k8s-school/kind-helper.git
-          ./kind-helper/k8s-create.sh -s
+          ktbx install kind
+          ktbx install kubectl
+          ktbx create -s
       - name: Load fink-alert-simulator image inside kind
         run: |
-          DIR="${{ env.FINK_ALERT_SIMULATOR_DIR }}"
-          .  "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/conf.sh
+          . "$CIUXCONFIG"
+          .  "$FINK_ALERT_SIMULATOR_DIR"/conf.sh
           if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then
             echo "::notice Succeeded to pull $IMAGE"
           else
@@ -81,7 +83,8 @@ jobs:
           docker exec -- kind-control-plane crictl image
       - name: Install fink-alert-simulator pre-requisites (argoCD)
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/prereq-install.sh
+          . "$CIUXCONFIG"
+          .  "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh
       - name: Install strimzi (kafka-operator)
         run: ./itest/strimzi-install.sh
       - name: Setup kafka
@@ -90,14 +93,10 @@ jobs:
         run: ./itest/prereq-install.sh
       - name: Install MinIO
         run: ./itest/minio-install.sh
-      - uses: actions/setup-go@v4
-        with:
-          go-version: '1.19.2'
-      - name: Install finkctl
-        run: go install github.com/astrolabsoftware/finkctl@"${{ env.FINKCTL_VERSION }}"
       - name: Run fink-alert-simulator
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/argo-submit.sh
+          . "$CIUXCONFIG"
+          "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh
           argo watch @latest
       # - name: Setup tmate session
       # uses: mxschmitt/action-tmate@v3

diff --git a/.github/workflows/itest-noscience.yml b/.github/workflows/itest-noscience.yml
@@ -1,5 +1,6 @@
 name: "Fink-broker e2e workflow (noscience, self-hosted)"
 on:
+  push:
   schedule:
     # At 03:00 UTC on every day-of-week from Monday through Friday.
     - cron:  '0 3 * * 1-5'

diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml
@@ -14,9 +14,10 @@ on:
         required: true
       registry_token:
         required: true
-
 env:
-  FINKCTL_VERSION: v1.1.0-rc2
+  CIUXCONFIG: /tmp/ciux.sh
+  CIUX_VERSION: v0.0.1-rc11
+  GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   MINIMAL: ${{ inputs.minimal }}
   NOSCIENCE: ${{ inputs.noscience }}
   # Override the self-hosted runner value
@@ -34,13 +35,20 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Ciux project ignition
+        run: ciux ignite --selector build $PWD
       - name: Build fink-broker image for k8s
         run: |
           ./build.sh
       - name: Push image to local registry
         id: push
         run: |
-          DIR=$(pwd)
+          . "$CIUXCONFIG"
           . ./conf.sh
           docker push $IMAGE
           echo "IMAGE=$IMAGE" >> "$GITHUB_OUTPUT"
@@ -54,19 +62,17 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Declare Version Variables
-        id: vars
-        shell: bash
-        run: |
-          DIR=$(pwd)
-          . ./conf.sh
-          echo "FINK_ALERT_SIMULATOR_DIR=$FINK_ALERT_SIMULATOR_DIR" >> $GITHUB_ENV
-          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
-      - name: Clone fink-alert-simulator code
-        run: ./itest/clone-fink-alert-simulator.sh
-      - name: Create kind-helper configuration
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Ciux project ignition
+        run: ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD
+      - name: Create ktbx configuration
         run: |
-          cat <<EOF > $HOME/.kind-helper
+          mkdir -p $HOME/.ktbx
+          cat <<EOF > $HOME/.ktbx/config
           kind:
             # Supported only for clusters with one node
             # Use host directory to share data between host and kind node
@@ -89,13 +95,16 @@ jobs:
           EOF
       - name: Create k8s/kind cluster
         run: |
-          VERSION="v1.0.2-rc1"
-          curl -sfL https://raw.githubusercontent.com/k8s-school/kind-helper/$VERSION/install.sh | bash
-          kind-helper create
+          ktbx install kind
+          ktbx install kubectl
+          kind version
+          kubectl version --client
+          ktbx version
+          ktbx create -s
       - name: Load fink-alert-simulator image inside kind
         run: |
-          DIR="${{ env.FINK_ALERT_SIMULATOR_DIR }}"
-          .  "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/conf.sh
+          . "$CIUXCONFIG"
+          .  "$FINK_ALERT_SIMULATOR_DIR"/conf.sh
           if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then
             echo "::notice Succeeded to pull $IMAGE"
           else
@@ -112,7 +121,8 @@ jobs:
           fi
       - name: Install fink-alert-simulator pre-requisites (argo-workflows)
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/prereq-install.sh
+          . "$CIUXCONFIG"
+          "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh
       - name: Install strimzi (kafka-operator)
         run: ./itest/strimzi-install.sh
       - name: Setup kafka
@@ -124,20 +134,19 @@ jobs:
           ./itest/prereq-install.sh
       - name: Install MinIO
         run: ./itest/minio-install.sh
-      - uses: actions/setup-go@v4
-        with:
-          go-version: '1.19.2'
-      - name: Install finkctl
-        run: go install github.com/astrolabsoftware/finkctl@"${{ env.FINKCTL_VERSION }}"
       - name: Run fink-alert-simulator
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/argo-submit.sh
+          . "$CIUXCONFIG"
+          "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh
           argo watch @latest
       # - name: Setup tmate session
       # uses: mxschmitt/action-tmate@v3
       - name: Run fink-broker
         run: |
           ./itest/fink-start.sh
+      - name: Check ouput topics are created
+        run: |
+          ./itest/check-results.sh
   image-analysis:
     name: Analyze image
     runs-on: [self-hosted, v3]

diff --git a/Dockerfile b/Dockerfile
@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG spark_image_tag
-FROM gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:${spark_image_tag} as noscience
+ARG spark_py_image
+FROM ${spark_py_image} as noscience
 
 ARG spark_uid=185
 ENV spark_uid ${spark_uid}
@@ -75,4 +75,4 @@ FROM noscience AS full
 ADD deps/requirements-science.txt $FINK_HOME/
 RUN pip install -r $FINK_HOME/requirements-science.txt
 ADD deps/requirements-science-no-deps.txt $FINK_HOME/
-RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps
+RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps
diff --git a/TODO.org b/TODO.org
@@ -1,3 +1,4 @@
+
 * 729
 ** DONE use "kubectl get kafkatopics.kafka.strimzi.io  -n kafka" to check success of integration tests, maybe in fnkctl?
 ** TODO DELAYED BECAUSE IT NOT BLOCKS BUT WARN create topic in distribute before sending alerts in order to avoid error below: https://fink-broker.slack.com/archives/D03KJ390F17/p1692008729660549

diff --git a/build.sh b/build.sh
@@ -22,8 +22,10 @@
 
 set -euxo pipefail
 
+
 DIR=$(cd "$(dirname "$0")"; pwd -P)
 
+ciux ignite refresh "$DIR"
 . $DIR/conf.sh
 
 if $NOSCIENCE
@@ -34,5 +36,5 @@ else
 fi
 
 # Build image
-docker image build --tag "$IMAGE" --build-arg spark_image_tag="$SPARK_IMAGE_TAG" "$DIR" --target $TARGET
+docker image build --tag "$IMAGE" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $TARGET
 
diff --git a/conf.sh b/conf.sh
@@ -1,6 +1,5 @@
-# Set variable below to empty string in order to
-# build and install current development version
-FINK_BROKER_RELEASE=''
+
+. "$CIUXCONFIG"
 
 # Do not launch science pipeline if true
 NOSCIENCE="${NOSCIENCE:-false}"
@@ -15,7 +14,7 @@ MINIMAL="${MINIMAL:-false}"
 REPO="gitlab-registry.in2p3.fr/astrolabsoftware/fink"
 CI_REPO="${CI_REPO:-$REPO}"
 # Tag to apply to the built image, or to identify the image to be pushed
-TAG=${FINK_BROKER_RELEASE:-$(git -C $DIR describe --dirty --always)}
+TAG="$FINK_BROKER_VERSION"
 # WARNING "spark-py" is hard-coded in spark build script
 
 # Disable science pipeline
@@ -31,9 +30,8 @@ fi
 # Spark parameters
 # ----------------
 
-# Spark image tag
-# Spark image is built here: https://github.com/astrolabsoftware/k8s-spark-py/
-SPARK_IMAGE_TAG="k8s-3.4.1"
+# TODO remove and manage with ciux
+SPARK_PY_IMAGE="gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1"
 
 # Spark version
 SPARK_VERSION="3.4.1"
@@ -59,5 +57,3 @@ KAFKA_CLUSTER="kafka-cluster"
 # Default values are the ones set in fink-alert-simulator CI environment
 KAFKA_SOCKET=${KAFKA_SOCKET:-"kafka-cluster-kafka-external-bootstrap.kafka:9094"}
 KAFKA_TOPIC=${KAFKA_TOPIC:-"ztf-stream-sim"}
-
-FINK_ALERT_SIMULATOR_DIR="/tmp/fink-alert-simulator"
diff --git a/deps/jars-urls.txt b/deps/jars-urls.txt
@@ -1,13 +1,13 @@
-https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10-assembly_2.12/3.2.3/spark-streaming-kafka-0-10-assembly_2.12-3.2.3.jar
-https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.2.3/spark-sql-kafka-0-10_2.12-3.2.3.jar
-https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.2.3/spark-avro_2.12-3.2.3.jar
-https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.2.3/spark-token-provider-kafka-0-10_2.12-3.2.3.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10-assembly_2.12/3.4.1/spark-streaming-kafka-0-10-assembly_2.12-3.4.1.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.4.1/spark-sql-kafka-0-10_2.12-3.4.1.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.4.1/spark-avro_2.12-3.4.1.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.4.1/spark-token-provider-kafka-0-10_2.12-3.4.1.jar
 https://repo1.maven.org/maven2/org/apache/hbase/hbase-shaded-mapreduce/2.2.7/hbase-shaded-mapreduce-2.2.7.jar
-https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.3/hadoop-aws-3.2.3.jar
+https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1/hadoop-aws-3.3.1.jar
 https://repo1.maven.org/maven2/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar
 https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/2.8.1/kafka-clients-2.8.1.jar
 https://repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar
-https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.6.2/commons-pool2-2.6.2.jar
+https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.11.1/commons-pool2-2.11.1.jar
 https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.1/hadoop-client-runtime-3.3.1.jar
 https://repo1.maven.org/maven2/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
 https://repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar

diff --git a/itest/check-results.sh b/itest/check-results.sh
@@ -26,5 +26,5 @@ DIR=$(cd "$(dirname "$0")"; pwd -P)
 
 . $DIR/../conf.sh
 
-finkctl wait topics --expected 10 --timeout 240s
+finkctl wait topics --expected 10 --timeout 600s -v1
 finkctl get topics
diff --git a/itest/clone-fink-alert-simulator.sh b/itest/clone-fink-alert-simulator.sh