diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..7547ed582 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,9 @@ +// QUIRK: For running the AMD64 containers on different platforms (eg. ARM64) do: docker pull --platform=linux/amd64 before starting the devcontainer. +// See: https://github.com/microsoft/vscode-remote-release/issues/8965 +{ + "name": "Julea Dev Container", + "image": "ghcr.io/parcio/julea-dev:latest", + "containerEnv": { + "JULEA_SPACK_DIR": "/app/dependencies" + } +} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6fa37f0f2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# Dependencies folder is not needed in the image (it will be rebuild in there) +dependencies + +# Git(-hub) specific +.git +.github + +# Ignore Docker related things +Dockerfile* +docker-* + +!docker-spack-entrypoint.sh \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ebc7de5c..d58e41e34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,65 +77,6 @@ jobs: JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }} run: | ./scripts/install-dependencies.sh - dependencies-containers: - name: Dependencies (Containers) - runs-on: ubuntu-24.04 - container: ${{ matrix.os.registry }}/${{ matrix.os.image }} - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - os: - - image: ubuntu:24.04 - registry: docker.io/library - compiler: gcc - compiler_version: 13.2.0 - - image: ubuntu:24.04 - registry: docker.io/library - compiler: clang - compiler_version: 18.1.3 - - image: ubuntu:22.04 - registry: docker.io/library - compiler: gcc - compiler_version: 11.4.0 - - image: ubuntu:22.04 - registry: docker.io/library - compiler: clang - compiler_version: 14.0.0 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - show-progress: false - - name: Install packages - if: ${{ matrix.os.image == 'ubuntu:24.04' || matrix.os.image == 'ubuntu:22.04' }} - run: | - apt update - apt --yes --no-install-recommends install build-essential zstd - apt --yes --no-install-recommends install ca-certificates git gnupg patchelf python3 unzip - if test "${{ matrix.os.compiler }}" = 'clang' - then - apt --yes --no-install-recommends install clang - - if test "${{ matrix.os.image }}" = 'ubuntu:24.04' - then - apt --yes --no-install-recommends install libclang-rt-dev - fi - fi - - name: Cache dependencies - id: cache - uses: actions/cache@v4 - with: - path: /dependencies - key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} - - name: Install dependencies - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - env: - JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }} - JULEA_SPACK_DIR: /dependencies - run: | - ./scripts/install-dependencies.sh build: name: Build needs: dependencies diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index f31e450c1..0debe4e05 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -16,33 +16,48 @@ name: Containers on: + workflow_dispatch: push: branches: - master - paths: - - '.github/workflows/containers.yml' - - 'containers/*' - pull_request: - branches: - - master - paths: - - '.github/workflows/containers.yml' - - 'containers/*' - schedule: - - cron: '0 0 * * 0' + # pull_request: + # branches: + # - master + workflow_call: +# schedule: +# - cron: '0 0 * * 0' + +env: + REGISTRY_BASE: ghcr.io/parcio + jobs: - ubuntu-dev: - name: Ubuntu Development Container - runs-on: ubuntu-24.04 + prepare: + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.generate.outputs.targets }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: List targets + id: generate + uses: docker/bake-action/subaction/list-targets@v4 + with: + target: ubuntu + + build-containesr: + name: Build Julea Containers + runs-on: ubuntu-latest + needs: + - prepare + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.prepare.outputs.targets) }} permissions: packages: write - timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v4 - with: - persist-credentials: false - show-progress: false - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} uses: docker/login-action@v3 @@ -50,10 +65,18 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and Push to GitHub Container Registry - uses: docker/build-push-action@v5 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and Push + uses: docker/bake-action@v5 + env: + COMMIT_SHA: ${{ github.sha }} + BASE_IMAGE_NAME: ${{ env.REGISTRY_BASE }}/julea with: - context: containers - file: containers/ubuntu-22.04-dev - push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/parcio/ubuntu-dev:22.04 + targets: ${{ matrix.target }} + push: true + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} diff --git a/Dockerfile.spack b/Dockerfile.spack new file mode 100644 index 000000000..f6f67ccc4 --- /dev/null +++ b/Dockerfile.spack @@ -0,0 +1,60 @@ +ARG UBUNTU_VERSION="24.04" + +# Stage 0 +FROM ubuntu:${UBUNTU_VERSION} AS base +SHELL [ "/bin/bash", "-c" ] +ARG JULEA_SPACK_COMPILER="gcc" +ARG CC="gcc" +ARG BUILDTYPE="release" +WORKDIR /app + +FROM base AS julea_runtime_dependencies + +RUN DEBIAN_FRONTEND=noninteractive apt-get -y update \ +&& apt-get -y install \ + python3 + +FROM julea_runtime_dependencies AS julea_dependencies + +RUN DEBIAN_FRONTEND=noninteractive apt-get -y update && apt-get -y upgrade && apt-get -y install \ + git \ + build-essential \ + zstd \ + ca-certificates \ + git \ + gnupg \ + patchelf \ + unzip \ + clang + +COPY ./scripts/ /app/scripts +RUN JULEA_SPACK_COMPILER=${JULEA_SPACK_COMPILER} JULEA_SPACK_DIR="/app/dependencies" /app/scripts/install-dependencies.sh + + +FROM julea_dependencies AS julea_build + +COPY ./ /app + +RUN . /app/scripts/environment.sh && \ + CC=${CC} meson setup --prefix="/app/julea-install" --buildtype=${BUILDTYPE} --werror "-Dgdbm_prefix=$(spack location --install-dir gdbm)" bld && \ + ninja -C bld && \ + DESTDIR="/" ninja -C bld install + +FROM julea_runtime_dependencies AS julea + +# Set Environment to support X86 emulators like QEMU (See: https://github.com/spack/spack/issues/41639) +ENV SPACK_ROOT="/app/dependencies/" + +COPY --from=julea_build /app/julea-install /app/julea-install +COPY --from=julea_build /app/dependencies /app/dependencies +COPY ./scripts/ /app/scripts/ +COPY ./docker-spack-entrypoint.sh /app/docker-spack-entrypoint.sh + +ENTRYPOINT [ "/app/docker-spack-entrypoint.sh" ] + + +FROM julea_dependencies AS julea_dev +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + gdb diff --git a/Dockerfile.system b/Dockerfile.system new file mode 100644 index 000000000..d772ddde3 --- /dev/null +++ b/Dockerfile.system @@ -0,0 +1,65 @@ +ARG UBUNTU_VERSION="24.04" + +FROM ubuntu:${UBUNTU_VERSION} AS base +SHELL [ "/bin/bash", "-c" ] +ARG UBUNTU_VERSION="24.04" +ARG CC="gcc" +ARG BUILDTYPE="release" +WORKDIR /app + + +FROM base AS julea_runtime_dependencies +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + libglib2.0-dev \ + libbson-dev \ + libfabric-dev \ + libgdbm-dev \ + liblmdb-dev \ + libsqlite3-dev \ + libleveldb-dev \ + libmongoc-dev \ + libmariadb-dev \ + librocksdb-dev \ + libfuse3-dev \ + libopen-trace-format-dev \ + librados-dev + + +FROM julea_runtime_dependencies AS julea_dependencies +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + build-essential \ + ninja-build \ + pkgconf \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + clang \ + # Install meson with pip if Ubuntu Version is older than 22.04. Otherwise use apt + && if awk "BEGIN {exit !($UBUNTU_VERSION >= 22.04)}" ; then echo "installing with apt..." && apt-get --yes install meson ; else pip install meson ; fi + + +FROM julea_dependencies AS julea_build +COPY . /app/ + +RUN CC=${CC} meson setup --buildtype=${BUILDTYPE} --werror --prefix="/usr/local" bld +RUN ninja -C bld +RUN DESTDIR=/app/julea-install/ ninja -C bld install + + +FROM julea_runtime_dependencies AS julea +COPY --from=julea_build /app/julea-install/ / +# Required for ld to detect the added libraries +RUN ldconfig + + +FROM julea_dependencies AS julea_dev + RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + gdb diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000..86c0bcce3 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,109 @@ +target "base" { platforms = ["linux/amd64"] } + +variable "BASE_IMAGE_NAME" { default = "ghcr.io/parcio/julea" } + +variable "COMMIT_SHA" { default = "UNKNOWN"} + +group "ubuntu" { targets = ["ubuntu-spack", "ubuntu-system", "ubuntu-latest", "ubuntu-spack-dev-container", "ubuntu-system-dev-container", "ubuntu-dev-latest"] } + +target "ubuntu-spack" { + name = "julea-ubuntu-${versions}-04-spack-${compilers}" + inherits = ["base"] + matrix = { + versions = ["24", "22", "20"] + compilers = ["gcc", "clang"] + } + args = { + UBUNTU_VERSION = "${versions}.04" + JULEA_SPACK_COMPILER = compilers + CC = compilers + } + + tags = ["${BASE_IMAGE_NAME}:${compilers}-spack-ubuntu-${versions}.04", "${BASE_IMAGE_NAME}:${compilers}-spack-ubuntu-${versions}.04-${COMMIT_SHA}"] + target = "julea" + dockerfile = "Dockerfile.spack" + cache-from = [ "type=gha,scope=julea-ubuntu-${versions}-04-spack-${compilers}" ] + cache-to = [ "type=gha,scope=julea-ubuntu-${versions}-04-spack-${compilers},mode=max" ] +} + +# Target build docker images with prebuild julea + dependencies using system method +target "ubuntu-system" { + name = "julea-ubuntu-${versions}-04-system-${compilers}" + inherits = ["base"] + matrix = { + versions = ["24", "22"] + compilers = ["gcc", "clang"] + } + args = { + UBUNTU_VERSION = "${versions}.04" + CC = compilers + } + tags = ["${BASE_IMAGE_NAME}:${compilers}-system-ubuntu-${versions}.04", "${BASE_IMAGE_NAME}:${compilers}-system-ubuntu-${versions}.04-${COMMIT_SHA}"] + target = "julea" + dockerfile = "Dockerfile.system" +} + +# Build latest image +target "ubuntu-latest" { + inherits = ["base"] + args = { + UBUNTU_VERSION = "24.04" + CC = "gcc" + } + tags = ["${BASE_IMAGE_NAME}:latest"] + target = "julea" + dockerfile = "Dockerfile.system" +} + +target "ubuntu-spack-dev-container" { + inherits = ["base"] + name = "julea-dev-ubuntu-${versions}-04-spack-${compilers}" + args = { + UBUNTU_VERSION = "${versions}.04" + CC = compilers + } + + matrix = { + versions = ["24", "22", "20"] + compilers = ["gcc", "clang"] + } + + tags = ["${BASE_IMAGE_NAME}-dev:${compilers}-spack-ubuntu-${versions}.04", "${BASE_IMAGE_NAME}-dev:${compilers}-spack-ubuntu-${versions}.04-${COMMIT_SHA}"] + target = "julea_dev" + dockerfile = "Dockerfile.spack" + + cache-from = [ "type=gha,scope=julea-ubuntu-${versions}-04-spack-${compilers}" ] + cache-to = [ "type=gha,scope=julea-ubuntu-${versions}-04-spack-${compilers},mode=max" ] +} + + +target "ubuntu-system-dev-container" { + inherits = ["base"] + name = "julea-dev-ubuntu-${versions}-04-system" + args = { + UBUNTU_VERSION = "${versions}.04" + } + + matrix = { + versions = ["24", "22", "20"] + } + + tags = ["${BASE_IMAGE_NAME}-dev:system-ubuntu-${versions}.04", "${BASE_IMAGE_NAME}-dev:system-ubuntu-${versions}.04-${COMMIT_SHA}"] + target = "julea_dev" + dockerfile = "Dockerfile.system" +} + +target "ubuntu-dev-latest" { + inherits = ["base"] + args = { + UBUNTU_VERSION = "24.04" + CC = "gcc" + } + + tags = ["${BASE_IMAGE_NAME}-dev:latest"] + target = "julea_dev" + dockerfile = "Dockerfile.spack" + + cache-from = [ "type=gha,scope=julea-ubuntu-24-04-spack-gcc" ] + cache-to = [ "type=gha,scope=julea-ubuntu-24-04-spack-gcc,mode=max" ] +} \ No newline at end of file diff --git a/docker-spack-entrypoint.sh b/docker-spack-entrypoint.sh new file mode 100755 index 000000000..afb6be1c5 --- /dev/null +++ b/docker-spack-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export JULEA_PREFIX="/app/julea-install" + +# shellcheck source=scripts/environment.sh +. /app/scripts/environment.sh + +# Run CMD +# shellcheck disable=SC2068 +$@ \ No newline at end of file