package_linux.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Package Linux

on:
  push:
    branches:
      - '**'
      - '!dependabot/**'
      - '!release-*'
    paths:
      - '.github/workflows/check_labels.yml'
      - '.github/workflows/package_linux.yml'
      - '.github/workflows/report_ci.yml'
      - 'cpp/**'
      - 'c_glib/**'
      - 'dev/archery/archery/**'
      - 'dev/release/binary-task.rb'
      - 'dev/release/verify-apt.sh'
      - 'dev/release/verify-yum.sh'
      - 'dev/tasks/linux-packages/**'
      - 'format/Flight.proto'
    tags:
      - "apache-arrow-*-rc*"
  pull_request:
    paths:
      - '.github/workflows/check_labels.yml'
      - '.github/workflows/package_linux.yml'
      - '.github/workflows/report_ci.yml'
      - 'cpp/**'
      - 'c_glib/**'
      - 'dev/archery/archery/**'
      - 'dev/release/binary-task.rb'
      - 'dev/release/verify-apt.sh'
      - 'dev/release/verify-yum.sh'
      - 'dev/tasks/linux-packages/**'
      - 'format/Flight.proto'
    types:
      - labeled
      - opened
      - reopened
      - synchronize
  schedule:
    - cron: "0 2 * * *"

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

permissions:
  actions: read
  contents: read
  pull-requests: read

jobs:
  check-labels:
    if: github.event_name != 'schedule' || github.repository == 'apache/arrow'
    uses: ./.github/workflows/check_labels.yml
    with:
      parent-workflow: package_linux

  package:
    permissions:
      # Upload artifacts to GitHub Release
      contents: write
      # Upload cached Docker images to GitHub Packages
      packages: write
    name: ${{ matrix.id }}
    runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
    needs: check-labels
    if: >-
      needs.check-labels.outputs.force == 'true' ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: Package: Linux')
    timeout-minutes: 160
    strategy:
      fail-fast: false
      matrix:
        id:
          - almalinux-8-amd64
          - almalinux-8-arm64
          - almalinux-9-amd64
          - almalinux-9-arm64
          - almalinux-10-amd64
          - almalinux-10-arm64
          - amazon-linux-2023-amd64
          - amazon-linux-2023-arm64
          - centos-9-stream-amd64
          - centos-9-stream-arm64
          - debian-trixie-amd64
          - debian-trixie-arm64
          - debian-forky-amd64
          - debian-forky-arm64
          - ubuntu-jammy-amd64
          - ubuntu-jammy-arm64
          - ubuntu-noble-amd64
          - ubuntu-noble-arm64
          - ubuntu-resolute-amd64
          - ubuntu-resolute-arm64
    env:
      BUILD_DIR: "${{ github.workspace }}/packages.build"
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v7
        with:
          persist-credentials: false
          fetch-depth: 0
          submodules: recursive
      - name: Free up disk space
        run: |
          ci/scripts/util_free_space.sh
      - name: Prepare environment variables
        env:
          ID: ${{ matrix.id }}
        run: |
          set -x

          case "${ID}" in
            centos-*)
              # Example: centos-9-stream-amd64 -> centos
              distribution="${ID%%-*}"
              ;;
            *)
              # Example: almalinux-8-amd64 -> almalinux
              # Example: amazon-linux-2023-amd64 -> amazon-linux
              distribution="${ID%-*-*}"
              ;;
          esac
          echo "DISTRIBUTION=${distribution}" >> "${GITHUB_ENV}"

          # Example: almalinux-8-amd64 -> amd64
          architecture="${ID##*-}"
          echo "ARCHITECTURE=${architecture}" >> "${GITHUB_ENV}"

          # Example: almalinux-8-amd64 -> almalinux-8
          target="${ID%-*}"
          case "${target}" in
            almalinux-*|amazon-linux-*|centos-*)
              echo "TASK_NAMESPACE=yum" >> "${GITHUB_ENV}"
              # Example: centos-9-stream-amd64 -> centos-9-stream
              # Example: amazon-linux-2023-amd64 -> amazon-linux-2023
              version="${ID%-*}"
              # Example: centos-9-stream -> 9-stream
              # Example: amazon-linux-2023 -> 2023
              version="${version##${distribution}-}"
              echo "DISTRIBUTION_VERSION=${version}" >> "${GITHUB_ENV}"
              if [ "${architecture}" = "arm64" ]; then
                # Example: almalinux-8 -> almalinux-8-aarch64
                target="${target}-aarch64"
              fi
              echo "YUM_TARGETS=${target}" >> "${GITHUB_ENV}"
              ;;
            *)
              echo "TASK_NAMESPACE=apt" >> "${GITHUB_ENV}"
              # Example: debian-bookworm-amd64 -> debian-bookworm
              code_name="${ID%-*}"
              # Example: debian-bookworm -> bookworm
              code_name="${code_name#*-}"
              echo "DISTRIBUTION_CODE_NAME=${code_name}" >> "${GITHUB_ENV}"
              if [ "${architecture}" = "arm64" ]; then
                # Example: ubuntu-noble -> ubuntu-noble-arm64
                target="${target}-arm64"
              fi
              echo "APT_TARGETS=${target}" >> "${GITHUB_ENV}"
              ;;
          esac
          echo "TARGET=${target}" >> "${GITHUB_ENV}"

          if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
            # Example: apache-arrow-21.0.0-rc0 -> 21.0.0-rc0
            version="${GITHUB_REF_NAME#apache-arrow-}"
            echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}"
          fi
      - name: Restore ccache
        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
        with:
          path: ${{ env.BUILD_DIR }}/${{ env.TARGET }}
          key: package-linux-${{ matrix.id }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y \
            rake \
            reprotest \
            ruby \
            ruby-dev
      - name: Allow unprivileged user namespaces
        if: env.TASK_NAMESPACE == 'apt'
        # Reprotest's domain_host variation uses `unshare -r --uts`,
        # which writes to /proc/self/uid_map. Ubuntu 24.04 restricts
        # unprivileged user namespaces via AppArmor by default, so
        # the call fails. Try to lift the restriction so reprotest
        # can run the full set of variations.
        run: |
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
      - name: Prepare apache-arrow-apt-source for arm64
        if: env.ARCHITECTURE == 'arm64'
        run: |
          pushd dev/tasks/linux-packages/apache-arrow-apt-source/apt
          for target in *-*; do
            cp -a ${target} ${target}-arm64
          done
          popd
      - name: Prepare apache-arrow-release for arm64
        if: env.ARCHITECTURE == 'arm64'
        run: |
          pushd dev/tasks/linux-packages/apache-arrow-release/yum
          for target in *-*; do
            cp -a ${target} ${target}-aarch64
          done
          popd
      - name: Update version
        if: github.ref_type != 'tag'
        run: |
          pushd dev/tasks/linux-packages
          rake version:update
          popd
      - name: Login to GitHub Container registry
        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Wait for creating GitHub Release
        if: github.ref_type == 'tag'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          dev/release/utils-watch-gh-workflow.sh \
            ${GITHUB_REF_NAME} \
            release_candidate.yml
      - name: Build
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          rake -C dev/tasks/linux-packages/apache-arrow docker:pull:${TARGET} || :
          rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build
      - name: Save ccache
        if: >-
          !cancelled()
        continue-on-error: true
        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
        with:
          path: ${{ env.BUILD_DIR }}/${{ env.TARGET }}
          key: package-linux-${{ matrix.id }}
          include-hidden-files: true
      - name: Docker Push
        continue-on-error: true
        if: >-
          success() &&
          github.event_name == 'push' &&
          github.ref_name == 'main'
        run: |
          rake -C dev/tasks/linux-packages/apache-arrow docker:push:${TARGET}
      - name: Build artifact tarball
        run: |
          mkdir -p "${DISTRIBUTION}"
          cp -a \
            dev/tasks/linux-packages/*/${TASK_NAMESPACE}/repositories/${DISTRIBUTION}/* \
            "${DISTRIBUTION}/"
          set -x
          # We use latest .deb/.rpm of
          # apache-arrow-apt-source/apache-arrow-release built for
          # amd64 because they are architecture independent.
          if [ "${ARCHITECTURE}" = "amd64" ]; then
            if [ "${TASK_NAMESPACE}" = "apt" ]; then
              # Create
              # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/apache-arrow-apt-source-latest-${DISTRIBUTION_CODE_NAME}.deb
              # for easy to install.
              cp -a \
                ${DISTRIBUTION}/pool/${DISTRIBUTION_CODE_NAME}/*/a/apache-arrow-apt-source/*.deb \
                ${DISTRIBUTION}/apache-arrow-apt-source-latest-${DISTRIBUTION_CODE_NAME}.deb
            else
              # Create
              # https://packages.apache.org/artifactory/arrow/${DISTRIBUTION}/${DISTRIBUTION_VERSION}/apache-arrow-release-latest.rpm
              # for easy to install.
              cp -a \
                ${DISTRIBUTION}/${DISTRIBUTION_VERSION}/x86_64/Packages/apache-arrow-release-*.rpm \
                ${DISTRIBUTION}/${DISTRIBUTION_VERSION}/apache-arrow-release-latest.rpm
            fi
          fi
          tar cvzf ${{ matrix.id }}.tar.gz ${DISTRIBUTION}
          dev/release/utils-generate-checksum.sh ${{ matrix.id }}.tar.gz
      - name: Upload the artifacts to the job
        uses: actions/upload-artifact@v7
        with:
          name: ${{ matrix.id }}
          path: ${{ matrix.id }}.tar.gz*
      - name: Upload the artifacts to GitHub Release
        if: github.ref_type == 'tag'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          gh release upload ${GITHUB_REF_NAME} \
            --clobber \
            ${{ matrix.id }}.tar.gz*
      - name: Set up test
        run: |
          sudo apt install -y \
            apt-utils \
            cpio \
            createrepo-c \
            devscripts \
            gpg \
            rpm \
            rsync
          gem install --user-install apt-dists-merge
          {
            echo "Key-Type: RSA"
            echo "Key-Length: 4096"
            echo "Name-Real: Test"
            echo "Name-Email: test@example.com"
            echo "%no-protection"
          } | gpg --full-generate-key --batch
          GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10)
          echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV}
          if [ "${TASK_NAMESPACE}" = "yum" ]; then
            repositories_dir=dev/tasks/linux-packages/apache-arrow-release/yum/repositories
            rpm2cpio ${repositories_dir}/*/*/*/Packages/apache-arrow-release-*.noarch.rpm |
              cpio -id
            mv etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow \
              dev/tasks/linux-packages/KEYS
          fi
          gpg --export --armor test@example.com >> dev/tasks/linux-packages/KEYS
      - name: Test
        run: |
          pushd dev/tasks/linux-packages
          rake --trace ${TASK_NAMESPACE}:test
          popd
      - name: Verify Reproducibility
        if: env.TASK_NAMESPACE == 'apt'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Validate reproducibility. Reprotest runs the build twice
          # inside its own tempdir and doesn't copy artifacts back,
          # so this is purely a verification step.
          reprotest \
            --vary=-fileordering \
            --build-command \
              "rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build" \
            "${PWD}" \
            "dev/tasks/linux-packages/*/apt/repositories/${DISTRIBUTION}/pool/${DISTRIBUTION_CODE_NAME}/*/*/*/*.deb"

  report-package-linux:
    if: github.event_name == 'schedule' && always()
    needs:
      - package
    uses: ./.github/workflows/report_ci.yml
    secrets:
      ARROW_SMTP_PASSWORD: ${{ secrets.ARROW_SMTP_PASSWORD }}
      ARROW_ZULIP_WEBHOOK: ${{ secrets.ARROW_ZULIP_WEBHOOK }}