From a2509c3f2aa4115cc99ee89b2991bde658dd7688 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 17 Mar 2025 15:03:26 +0100 Subject: [PATCH 01/19] wip --- druid/Dockerfile | 11 +++-------- rust/patchable/src/main.rs | 23 ++++++++++++++++++++--- stackable-base/Dockerfile | 21 +++++++++++++++++++++ zookeeper/Dockerfile | 9 ++++----- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index de5137591..be9aaca79 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -23,10 +23,8 @@ microdnf update # This requirement is documented in docs/development/build.md and version 5.1 or later is required. # UBI 9 ships with 5.4.x so that should be fine # -# patch: Required for the apply-patches.sh script microdnf install \ -python-pyyaml \ -patch +python-pyyaml microdnf clean all rm -rf /var/cache/yum @@ -35,8 +33,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:0 druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh -COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT} +COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base @@ -50,10 +47,8 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < { struct Opts { #[clap(subcommand)] cmd: Cmd, + + /// Specify a custom root directory for the images repository + #[clap(long)] + images_repo_root: Option, } #[derive(clap::Parser)] @@ -165,7 +169,7 @@ enum Cmd { pv: ProductVersion, }, - /// Shwos the images repository root + /// Shows the images repository root ImagesDir, } @@ -201,6 +205,8 @@ pub enum Error { FindImagesRepo { source: repo::Error }, #[snafu(display("images repository has no work directory"))] NoImagesRepoWorkdir, + #[snafu(display("images repository root at {path:?} is not a directory"))] + ImagesRepoRootDirCheck { path: PathBuf }, #[snafu(display("failed to fetch patch series' base commit"))] FetchBaseCommit { source: repo::Error }, @@ -263,8 +269,19 @@ fn main() -> Result<()> { .context(ConfigureGitLoggingSnafu)?; let opts = ::parse(); - let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?; - let images_repo_root = images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?; + let images_repo_root_pathbuf = match opts.images_repo_root { + Some(path) => { + if !path.is_dir() { + return ImagesRepoRootDirCheckSnafu { path }.fail(); + } + path + } + None => { + let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?; + images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?.to_owned() + } + }; + let images_repo_root = images_repo_root_pathbuf.as_path(); match opts.cmd { Cmd::Checkout { pv, base_only } => { let ctx = ProductVersionContext { diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 3109ef2b5..b0d485355 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -59,6 +59,23 @@ rustup toolchain install cargo auditable --quiet build --release && cargo cyclonedx --all --spec-version 1.5 --describe binaries EOF +FROM product-utils-builder AS patchable + +# Copy source code of patchable +COPY rust/patchable/ /patchable/rust/patchable +# Copy workspace files +COPY Cargo.* /patchable/ + +RUN < Date: Thu, 20 Mar 2025 13:33:54 +0100 Subject: [PATCH 02/19] Update druid/Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- druid/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index be9aaca79..734357f27 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -33,7 +33,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/druid/stackable/patches/${PRODUCT} +COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base From 21fbd889c7509d8946dec0309cf6f31187d5fd91 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 20 Mar 2025 13:35:01 +0100 Subject: [PATCH 03/19] fix: remove unnecessary check / shadow repo root var --- rust/patchable/src/main.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 59f0b414f..d32f9b4b2 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -205,8 +205,6 @@ pub enum Error { FindImagesRepo { source: repo::Error }, #[snafu(display("images repository has no work directory"))] NoImagesRepoWorkdir, - #[snafu(display("images repository root at {path:?} is not a directory"))] - ImagesRepoRootDirCheck { path: PathBuf }, #[snafu(display("failed to fetch patch series' base commit"))] FetchBaseCommit { source: repo::Error }, @@ -269,19 +267,14 @@ fn main() -> Result<()> { .context(ConfigureGitLoggingSnafu)?; let opts = ::parse(); - let images_repo_root_pathbuf = match opts.images_repo_root { - Some(path) => { - if !path.is_dir() { - return ImagesRepoRootDirCheckSnafu { path }.fail(); - } - path - } + let images_repo_root = match opts.images_repo_root { + Some(path) => path, None => { let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?; images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?.to_owned() } }; - let images_repo_root = images_repo_root_pathbuf.as_path(); + let images_repo_root = images_repo_root.as_path(); match opts.cmd { Cmd::Checkout { pv, base_only } => { let ctx = ProductVersionContext { From 5fa3620de331c563336906dcd1b13e45aba2532c Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 20 Mar 2025 13:47:39 +0100 Subject: [PATCH 04/19] fix: druid src path --- druid/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index 734357f27..8932944af 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -47,7 +47,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < Date: Thu, 20 Mar 2025 13:53:38 +0100 Subject: [PATCH 05/19] fix: druid src path --- druid/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index 8932944af..c3cfd2b17 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -47,8 +47,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < Date: Fri, 21 Mar 2025 13:27:10 +0100 Subject: [PATCH 06/19] feat: introduce stackable-devel image --- conf.py | 2 + java-devel/Dockerfile | 4 +- java-devel/versions.py | 12 +++--- stackable-base/Dockerfile | 65 +++------------------------- stackable-base/versions.py | 1 + stackable-devel/Dockerfile | 86 +++++++++++++++++++++++++++++++++++++ stackable-devel/versions.py | 5 +++ 7 files changed, 108 insertions(+), 67 deletions(-) create mode 100644 stackable-devel/Dockerfile create mode 100644 stackable-devel/versions.py diff --git a/conf.py b/conf.py index e863dd145..d580c6d57 100644 --- a/conf.py +++ b/conf.py @@ -26,6 +26,7 @@ opa = importlib.import_module("opa.versions") spark_k8s = importlib.import_module("spark-k8s.versions") stackable_base = importlib.import_module("stackable-base.versions") +stackable_devel = importlib.import_module("stackable-devel.versions") superset = importlib.import_module("superset.versions") trino_cli = importlib.import_module("trino-cli.versions") trino = importlib.import_module("trino.versions") @@ -54,6 +55,7 @@ {"name": "opa", "versions": opa.versions}, {"name": "spark-k8s", "versions": spark_k8s.versions}, {"name": "stackable-base", "versions": stackable_base.versions}, + {"name": "stackable-devel", "versions": stackable_devel.versions}, {"name": "superset", "versions": superset.versions}, {"name": "trino-cli", "versions": trino_cli.versions}, {"name": "trino", "versions": trino.versions}, diff --git a/java-devel/Dockerfile b/java-devel/Dockerfile index f135d4738..e8a2f1ac1 100644 --- a/java-devel/Dockerfile +++ b/java-devel/Dockerfile @@ -2,10 +2,10 @@ # check=error=true # -# Base imaege for builder stages +# Base image for builder stages in Java based products # -FROM stackable/image/stackable-base +FROM stackable/image/stackable-devel ARG PRODUCT diff --git a/java-devel/versions.py b/java-devel/versions.py index 9ba25519d..0ef5fa87d 100644 --- a/java-devel/versions.py +++ b/java-devel/versions.py @@ -1,26 +1,26 @@ versions = [ { "product": "8", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, { "product": "11", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, { "product": "17", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, { "product": "21", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, { "product": "22", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, { "product": "23", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, ] diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index b0d485355..319dddcbb 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -1,81 +1,32 @@ # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5 # check=error=true -# Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti -# IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures -# rather than just the "Image Digest" that references the image for the selected architecture. -FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c0e70387664f30cd9cf2795b547e4a9a51002c44a4a86aa9335ab030134bf392 AS product-utils-builder +FROM stackable/image/stackable-devel AS config-utils # Find the latest version here: https://github.com/stackabletech/config-utils/tags # renovate: datasource=github-tags packageName=stackabletech/config-utils ENV CONFIG_UTILS_VERSION=0.2.0 -# Find the latest version here: https://github.com/stackabletech/containerdebug/tags -# renovate: datasource=github-tags packageName=stackabletech/containerdebug -ENV CONTAINERDEBUG_VERSION=0.1.1 -# This SHOULD be kept in sync with operator-templating and other tools to reduce build times -# Find the latest version here: https://doc.rust-lang.org/stable/releases.html -# renovate: datasource=github-releases packageName=rust-lang/rust -ENV RUST_DEFAULT_TOOLCHAIN_VERSION=1.84.1 -# Find the latest version here: https://crates.io/crates/cargo-cyclonedx -# renovate: datasource=crate packageName=cargo-cyclonedx -ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7 -# Find the latest version here: https://crates.io/crates/cargo-auditable -# renovate: datasource=crate packageName=cargo-auditable -ENV CARGO_AUDITABLE_CRATE_VERSION=0.6.6 - -RUN < Date: Fri, 21 Mar 2025 13:41:31 +0100 Subject: [PATCH 07/19] fix: use PathBuf in ProductVersionContext --- rust/patchable/src/main.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index d32f9b4b2..c1adcb53d 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -5,11 +5,7 @@ mod repo; mod utils; use core::str; -use std::{ - fs::File, - io::Write, - path::{Path, PathBuf}, -}; +use std::{fs::File, io::Write, path::PathBuf}; use git2::{Oid, Repository}; use serde::{Deserialize, Serialize}; @@ -35,12 +31,12 @@ struct ProductVersionConfig { base: Oid, } -struct ProductVersionContext<'a> { +struct ProductVersionContext { pv: ProductVersion, - images_repo_root: &'a Path, + images_repo_root: PathBuf, } -impl ProductVersionContext<'_> { +impl ProductVersionContext { fn load_config(&self) -> Result { let path = &self.config_path(); tracing::info!( @@ -271,10 +267,12 @@ fn main() -> Result<()> { Some(path) => path, None => { let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?; - images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?.to_owned() + images_repo + .workdir() + .context(NoImagesRepoWorkdirSnafu)? + .to_owned() } }; - let images_repo_root = images_repo_root.as_path(); match opts.cmd { Cmd::Checkout { pv, base_only } => { let ctx = ProductVersionContext { From 029372804bd8a18badf40c39e846495c14854f0e Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 21 Mar 2025 13:49:24 +0100 Subject: [PATCH 08/19] chore: align zookeeper patch directory structure --- druid/Dockerfile | 2 +- zookeeper/Dockerfile | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index c3cfd2b17..6c297c64a 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -33,7 +33,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base diff --git a/zookeeper/Dockerfile b/zookeeper/Dockerfile index 7b4520882..1e8c79bbd 100644 --- a/zookeeper/Dockerfile +++ b/zookeeper/Dockerfile @@ -10,15 +10,16 @@ ARG PRODUCT ARG JMX_EXPORTER ARG STACKABLE_USER_UID -# Copy patches and JMX config into the builder -COPY --chown=${STACKABLE_USER_UID}:0 zookeeper/stackable /stackable +# Copy patches into the builder +COPY --chown=${STACKABLE_USER_UID}:0 zookeeper/stackable/patches/${PRODUCT} /stackable/src/zookeeper/stackable/patches/${PRODUCT} +# Copy JMX config into the builder +COPY --chown=${STACKABLE_USER_UID}:0 zookeeper/stackable/jmx /stackable/jmx USER ${STACKABLE_USER_UID} WORKDIR /stackable # Download ZooKeeper sources from our own repo -RUN mkdir -p zookeeper/stackable && mv patches zookeeper/stackable/ && \ - BUILD_SRC_DIR="$(/stackable/patchable --images-repo-root=. checkout zookeeper ${PRODUCT})" && \ +RUN BUILD_SRC_DIR="$(/stackable/patchable --images-repo-root=src checkout zookeeper ${PRODUCT})" && \ cd "${BUILD_SRC_DIR}" && \ # Exclude the `zookeeper-client` submodule, this is not needed and has c parts # that created all kinds of issues for the build container From 455e2faead2664071b45abebd52b3477776e685c Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 21 Mar 2025 13:51:46 +0100 Subject: [PATCH 09/19] fix: stackable-devel dnf and shell config --- stackable-devel/Dockerfile | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index 8a5ddbeaa..fef7e36bb 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -16,6 +16,24 @@ ARG STACKABLE_USER_UID ARG STACKABLE_USER_GID ARG STACKABLE_USER_NAME +# Sets the default shell to Bash with strict error handling and robust pipeline processing. +# "-e": Exits immediately if a command exits with a non-zero status +# "-u": Treats unset variables as an error, preventing unexpected behavior from undefined variables. +# "-o pipefail": Causes a pipeline to return the exit status of the last command in the pipe that failed, ensuring errors in any part of a pipeline are not ignored. +# "-c": Allows the execution of commands passed as a string +# This is automatically inherited in all other Dockerfiles that use this unless it is overwritten +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] + +# We configure microdnf to not install weak dependencies in this file +# Not doing this caused the content of images to become unpredictable because +# based on which packages get updated by `microdnf update` new weak dependencies +# might be installed that were not present earlier (the ubi base image doesn't +# seem to install weak dependencies) +# This also affects the packages that are installed in our Dockerfiles (java as prime +# example). +# https://github.com/stackabletech/docker-images/pull/533 +COPY stackable-base/stackable/dnf.conf /etc/dnf/dnf.conf + # This SHOULD be kept in sync with operator-templating and other tools to reduce build times # Find the latest version here: https://doc.rust-lang.org/stable/releases.html # renovate: datasource=github-releases packageName=rust-lang/rust @@ -28,11 +46,11 @@ ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7 ENV CARGO_AUDITABLE_CRATE_VERSION=0.6.6 RUN < Date: Mon, 24 Mar 2025 18:42:58 +0100 Subject: [PATCH 10/19] chore: switch patch process in other products --- druid/stackable/patches/apply_patches.sh | 44 ----------- hadoop/Dockerfile | 6 +- hadoop/stackable/patches/apply_patches.sh | 43 ----------- hbase/Dockerfile | 50 +++++-------- hbase/stackable/patches/apply_patches.sh | 49 ------------- hive/Dockerfile | 22 +++--- hive/stackable/patches/apply_patches.sh | 50 ------------- kafka/Dockerfile | 18 +---- kafka/stackable/patches/apply_patches.sh | 44 ----------- nifi/Dockerfile | 23 +----- nifi/stackable/patches/apply_patches.sh | 43 ----------- omid/Dockerfile | 7 +- omid/stackable/patches/apply_patches.sh | 44 ----------- spark-k8s/Dockerfile | 73 +++---------------- spark-k8s/hbase-connectors/apply_patches.sh | 44 ----------- spark-k8s/stackable/patches/apply_patches.sh | 44 ----------- stackable-devel/Dockerfile | 8 ++ superset/Dockerfile | 4 - superset/stackable/patches/4.0.2/.gitkeep | 0 .../stackable/patches/4.0.2/patchable.toml | 2 - superset/stackable/patches/4.1.1/.gitkeep | 0 .../stackable/patches/4.1.1/patchable.toml | 2 - superset/stackable/patches/apply_patches.sh | 54 -------------- trino-storage-connector/Dockerfile | 18 +---- .../stackable/patches/apply_patches.sh | 44 ----------- trino/Dockerfile | 31 +++----- trino/stackable/patches/apply_patches.sh | 44 ----------- zookeeper/Dockerfile | 8 +- zookeeper/stackable/patches/apply_patches.sh | 50 ------------- 29 files changed, 74 insertions(+), 795 deletions(-) delete mode 100755 druid/stackable/patches/apply_patches.sh delete mode 100755 hadoop/stackable/patches/apply_patches.sh delete mode 100644 hbase/stackable/patches/apply_patches.sh delete mode 100755 hive/stackable/patches/apply_patches.sh delete mode 100755 kafka/stackable/patches/apply_patches.sh delete mode 100644 nifi/stackable/patches/apply_patches.sh delete mode 100755 omid/stackable/patches/apply_patches.sh delete mode 100755 spark-k8s/hbase-connectors/apply_patches.sh delete mode 100755 spark-k8s/stackable/patches/apply_patches.sh delete mode 100644 superset/stackable/patches/4.0.2/.gitkeep delete mode 100644 superset/stackable/patches/4.0.2/patchable.toml delete mode 100644 superset/stackable/patches/4.1.1/.gitkeep delete mode 100644 superset/stackable/patches/4.1.1/patchable.toml delete mode 100755 superset/stackable/patches/apply_patches.sh delete mode 100755 trino-storage-connector/stackable/patches/apply_patches.sh delete mode 100755 trino/stackable/patches/apply_patches.sh delete mode 100755 zookeeper/stackable/patches/apply_patches.sh diff --git a/druid/stackable/patches/apply_patches.sh b/druid/stackable/patches/apply_patches.sh deleted file mode 100755 index 833b3e9c7..000000000 --- a/druid/stackable/patches/apply_patches.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - patch --directory "." --strip=1 < "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index acb4e0947..f3f4bcd50 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -54,16 +54,14 @@ RUN microdnf update && \ WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/${PRODUCT} /stackable/src/hadoop/stackable/patches/${PRODUCT} # Hadoop Pipes requires libtirpc to build, whose headers are not packaged in RedHat UBI, so skip building this module # Build from source to enable FUSE module, and to apply custom patches. # Also skip building the yarn, mapreduce and minicluster modules: this will result in the modules being excluded but not all # jar files will be stripped if they are needed elsewhere e.g. share/hadoop/yarn will not be part of the build, but yarn jars # will still exist in share/hadoop/tools as they would be needed by the resource estimator tool. Such jars are removed in a later step. -RUN curl "https://repo.stackable.tech/repository/packages/hadoop/hadoop-${PRODUCT}-src.tar.gz" | tar -xzC . && \ - patches/apply_patches.sh ${PRODUCT} && \ - cd hadoop-${PRODUCT}-src && \ +RUN cd "$(/stackable/patchable --images-repo-root=src checkout hadoop ${PRODUCT})" && \ mvn --batch-mode --no-transfer-progress clean package -Pdist,native -pl '!hadoop-tools/hadoop-pipes,!hadoop-yarn-project,!hadoop-mapreduce-project,!hadoop-minicluster' -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true && \ cp -r hadoop-dist/target/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT} && \ mv hadoop-dist/target/bom.json /stackable/hadoop-${PRODUCT}/hadoop-${PRODUCT}.cdx.json && \ diff --git a/hadoop/stackable/patches/apply_patches.sh b/hadoop/stackable/patches/apply_patches.sh deleted file mode 100755 index 14f0f257c..000000000 --- a/hadoop/stackable/patches/apply_patches.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - git apply --directory "hadoop-${VERSION}-src" "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/hbase/Dockerfile b/hbase/Dockerfile index ca80123c9..10273c8a6 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -26,7 +26,7 @@ COPY hbase/licenses /licenses USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx # Cache mounts are owned by root by default @@ -44,13 +44,7 @@ RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=${STACKABLE_USER_UID},targe ### ### HBase ### -curl "https://repo.stackable.tech/repository/packages/hbase/hbase-${PRODUCT}-src.tar.gz" | tar -xzC . -mv hbase-${PRODUCT} hbase-${PRODUCT}-src - -chmod +x patches/apply_patches.sh -patches/apply_patches.sh ${PRODUCT} - -cd /stackable/hbase-${PRODUCT}-src/ +cd "$(/stackable/patchable --images-repo-root=src checkout hbase ${PRODUCT})" && \ # The release scripts of HBase also run the build twice (three times in fact, once again to build the site which we skip here). # I chose to replicate that exact behavior for consistency so please don't merge the two mvn runs into one unless you really know what you're doing! @@ -139,8 +133,7 @@ ARG DELETE_CACHES="true" # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile # does not work, so please ignore the according warning (SC2016). COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/ -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches /stackable/patches -COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/apply_patches.sh /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} /stackable/src/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} COPY --chown=${STACKABLE_USER_UID}:0 --chmod=755 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/ USER ${STACKABLE_USER_UID} @@ -149,11 +142,14 @@ WORKDIR /stackable # Cache mounts are owned by root by default # We need to explicitly give the uid to use RUN --mount=type=cache,id=maven-hbase-operator-tools-${HBASE_OPERATOR_TOOLS},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository < /stackable/bin/hbck2 chmod +x /stackable/bin/hbck2 @@ -233,20 +229,13 @@ ARG STACKABLE_USER_UID # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" -COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches /stackable/patches -COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/apply_patches.sh /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} USER ${STACKABLE_USER_UID} WORKDIR /stackable RUN --mount=type=cache,id=maven-phoenix-${PHOENIX},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <>> Build spark diff --git a/spark-k8s/hbase-connectors/apply_patches.sh b/spark-k8s/hbase-connectors/apply_patches.sh deleted file mode 100755 index 833b3e9c7..000000000 --- a/spark-k8s/hbase-connectors/apply_patches.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - patch --directory "." --strip=1 < "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/spark-k8s/stackable/patches/apply_patches.sh b/spark-k8s/stackable/patches/apply_patches.sh deleted file mode 100755 index 833b3e9c7..000000000 --- a/spark-k8s/stackable/patches/apply_patches.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - patch --directory "." --strip=1 < "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index fef7e36bb..dfb0e06bb 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -34,6 +34,9 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"] # https://github.com/stackabletech/docker-images/pull/533 COPY stackable-base/stackable/dnf.conf /etc/dnf/dnf.conf +# Default curl configuration to avoid forgetting settings and to declutter the Dockerfiles +COPY stackable-base/stackable/curlrc /root/.curlrc + # This SHOULD be kept in sync with operator-templating and other tools to reduce build times # Find the latest version here: https://doc.rust-lang.org/stable/releases.html # renovate: datasource=github-releases packageName=rust-lang/rust @@ -50,8 +53,10 @@ microdnf update # git: Needed to fetch source # gcc: Needed for compilation +# findutils: Needed for xargs, used (at least) by `gradlew` in the Kafka build process microdnf install \ gcc \ + findutils \ git ### @@ -80,6 +85,9 @@ microdnf remove shadow-utils microdnf clean all rm -rf /var/cache/yum +cp /root/.curlrc /stackable/.curlrc +chown ${STACKABLE_USER_UID}:0 /stackable/.curlrc + # WARNING (@NickLarsenNZ): We should pin the rustup version curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_DEFAULT_TOOLCHAIN_VERSION" . "$HOME/.cargo/env" && cargo --quiet install cargo-cyclonedx@"$CARGO_CYCLONEDX_CRATE_VERSION" cargo-auditable@"$CARGO_AUDITABLE_CRATE_VERSION" && rustup toolchain install diff --git a/superset/Dockerfile b/superset/Dockerfile index 5c9b6a03c..a6038061c 100644 --- a/superset/Dockerfile +++ b/superset/Dockerfile @@ -59,7 +59,6 @@ RUN microdnf update \ libffi-devel \ openldap-devel \ openssl-devel \ - patch \ python${PYTHON} \ python${PYTHON}-devel \ python${PYTHON}-pip \ @@ -114,9 +113,6 @@ RUN python3 -m venv /stackable/app \ && if [ -n "$AUTHLIB" ]; then pip install Authlib==${AUTHLIB}; fi && \ pip install --no-cache-dir /tmp/opa_authorizer-0.1.0-py3-none-any.whl -COPY superset/stackable/patches /patches -RUN /patches/apply_patches.sh ${PRODUCT} - WORKDIR /stackable RUN source /stackable/app/bin/activate && cyclonedx-py environment --schema-version 1.5 --outfile app/superset-${PRODUCT}.cdx.json diff --git a/superset/stackable/patches/4.0.2/.gitkeep b/superset/stackable/patches/4.0.2/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/superset/stackable/patches/4.0.2/patchable.toml b/superset/stackable/patches/4.0.2/patchable.toml deleted file mode 100644 index 739bf4b8b..000000000 --- a/superset/stackable/patches/4.0.2/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -upstream = "https://github.com/apache/superset.git" -base = "f11fa091e261a35f4d39d8567a859fad07547d84" diff --git a/superset/stackable/patches/4.1.1/.gitkeep b/superset/stackable/patches/4.1.1/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/superset/stackable/patches/4.1.1/patchable.toml b/superset/stackable/patches/4.1.1/patchable.toml deleted file mode 100644 index 60c0eb205..000000000 --- a/superset/stackable/patches/4.1.1/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -upstream = "https://github.com/apache/superset.git" -base = "6264ff516532f0359d914bd72356f2007925109b" diff --git a/superset/stackable/patches/apply_patches.sh b/superset/stackable/patches/apply_patches.sh deleted file mode 100755 index e54546b55..000000000 --- a/superset/stackable/patches/apply_patches.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Dynamically find the site-packages directory under the /stackable/app/lib/ directory -SITE_PACKAGES_DIR=$(find /stackable/app/lib -type d -name "site-packages" | head -n 1) - -if [ -z "$SITE_PACKAGES_DIR" ]; then - echo "site-packages directory could not be found." - exit 1 -fi - -echo "Using site-packages directory: $SITE_PACKAGES_DIR" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - cat "$patch_file" | patch --directory "$SITE_PACKAGES_DIR" --strip=1 || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/trino-storage-connector/Dockerfile b/trino-storage-connector/Dockerfile index 5b92551b6..0058c81d1 100644 --- a/trino-storage-connector/Dockerfile +++ b/trino-storage-connector/Dockerfile @@ -6,28 +6,14 @@ FROM stackable/image/java-devel AS storage-connector-builder ARG PRODUCT ARG STACKABLE_USER_UID -RUN < Date: Thu, 27 Mar 2025 16:19:55 +0100 Subject: [PATCH 11/19] fix: hive build --- hive/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hive/Dockerfile b/hive/Dockerfile index c66f6e60d..dac58ca65 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -23,6 +23,8 @@ ARG DELETE_CACHES="true" COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT} # Copy JMX config into the builder COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/jmx /stackable/jmx +# Copy the start script into the builder +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin /stackable/bin USER ${STACKABLE_USER_UID} WORKDIR /stackable From de74bde2b4b17eb3df9117909efe4416f7069a3e Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 28 Mar 2025 14:27:26 +0100 Subject: [PATCH 12/19] fix: trino build --- trino/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trino/Dockerfile b/trino/Dockerfile index e6eeede0c..9c5562571 100644 --- a/trino/Dockerfile +++ b/trino/Dockerfile @@ -44,7 +44,7 @@ chown --recursive ${STACKABLE_USER_UID}:0 /stackable/trino-server-${PRODUCT} # Delete all intermediate build products to free some more space cd .. -rm -r ./${HBASE_OPERATOR_TOOLS} +rm -r ./${PRODUCT} EOF COPY --from=trino-storage-connector-image /stackable/src/trino-storage-connector/patchable-work/worktree/${PRODUCT}/target/trino-storage-${PRODUCT} /stackable/trino-server-${PRODUCT}/plugin/trino-storage-${PRODUCT} From e17b727ba093e9862e13785fbe5d792fe3a5b0a3 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 28 Mar 2025 15:48:15 +0100 Subject: [PATCH 13/19] fix: spark build --- spark-k8s/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index b733eb8c0..14f7dadde 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -45,7 +45,7 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/${HBASE_CONNECTOR} /stackable/src/spark-k8s/hbase-connectors/stackable/patches/${HBASE_CONNECTOR} RUN < Date: Mon, 31 Mar 2025 12:42:57 +0200 Subject: [PATCH 14/19] chore: make hadolint happy --- hive/Dockerfile | 36 +++++++++++++++++++----------------- trino/Dockerfile | 7 ++++--- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/hive/Dockerfile b/hive/Dockerfile index dac58ca65..c2fcefcd8 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -32,32 +32,34 @@ WORKDIR /stackable # Cache mounts are owned by root by default # We need to explicitly give the uid to use RUN --mount=type=cache,id=maven-hive-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository < Date: Fri, 4 Apr 2025 12:21:25 +0200 Subject: [PATCH 15/19] fix: remove hbase intermediate sources / remove unnecessary and operator --- hbase/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hbase/Dockerfile b/hbase/Dockerfile index 10273c8a6..f1524fd4b 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -44,7 +44,7 @@ RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=${STACKABLE_USER_UID},targe ### ### HBase ### -cd "$(/stackable/patchable --images-repo-root=src checkout hbase ${PRODUCT})" && \ +cd "$(/stackable/patchable --images-repo-root=src checkout hbase ${PRODUCT})" # The release scripts of HBase also run the build twice (three times in fact, once again to build the site which we skip here). # I chose to replicate that exact behavior for consistency so please don't merge the two mvn runs into one unless you really know what you're doing! @@ -52,7 +52,8 @@ mvn --batch-mode --no-transfer-progress -Dhadoop.profile=3.0 -Dhadoop-three.vers mvn --batch-mode --no-transfer-progress -Dhadoop.profile=3.0 -Dhadoop-three.version=${HADOOP} install assembly:single -DskipTests -Dcheckstyle.skip=true -Prelease tar -xzf hbase-assembly/target/hbase-${PRODUCT}-bin.tar.gz -C /stackable/ mv hbase-assembly/target/bom.json /stackable/hbase-${PRODUCT}/hbase-${PRODUCT}.cdx.json -rm -rf /stackable/hbase-${PRODUCT}-src +cd .. +rm -r ${PRODUCT} ln -s "/stackable/hbase-${PRODUCT}" /stackable/hbase ### From eb83abac58e76433eaba19479f83f37f615ffdfb Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 4 Apr 2025 13:27:30 +0200 Subject: [PATCH 16/19] fix: permissions in patchable build process --- stackable-devel/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index dfb0e06bb..c34f7e2e4 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -109,4 +109,6 @@ cd /patchable cargo auditable --quiet build --release && cargo cyclonedx --all --spec-version 1.5 --describe binaries mv /patchable/target/release/patchable /stackable/patchable microdnf clean all +chown ${STACKABLE_USER_UID}:0 /stackable/patchable +rm -rf /patchable EOF From d11d061e6561460f169eaa7f813a41fbed41b6e1 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 14 Apr 2025 13:58:53 +0200 Subject: [PATCH 17/19] chore: remove unnecessary curl command in build process --- druid/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index e424accc4..a45d6b02a 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -46,8 +46,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < Date: Mon, 14 Apr 2025 17:10:26 +0200 Subject: [PATCH 18/19] chore: move adding of JMX config and start-metastore script from builder stage to final stage --- hive/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hive/Dockerfile b/hive/Dockerfile index 943efaf51..65c7209b5 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -24,10 +24,6 @@ ARG DELETE_CACHES="true" # Copy patches into the builder COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT} -# Copy JMX config into the builder -COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/jmx /stackable/jmx -# Copy the start script into the builder -COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin /stackable/bin # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? @@ -66,8 +62,7 @@ fi cd /stackable rm -rf "$BUILD_SRC_DIR" -cp bin/start-metastore apache-hive-metastore-${PRODUCT}-bin/bin - +mkdir /stackable/jmx curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar @@ -133,6 +128,8 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hadoop-${HADOOP} /stackable/hadoop-${HADOOP} COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/jmx /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin/start-metastore /stackable/apache-hive-metastore-${PRODUCT}-bin/bin COPY hive/licenses /licenses @@ -144,6 +141,8 @@ chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum +chmod g=u /stackable/apache-hive-metastore-${PRODUCT}-bin/bin/start-metastore + ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore chown -h ${STACKABLE_USER_UID}:0 /stackable/hive-metastore chmod g=u /stackable/hive-metastore From 92b43bfa4caf082b2934f33040c8ed98f6a02a11 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 14 Apr 2025 17:11:38 +0200 Subject: [PATCH 19/19] chore: remove git repo in trino and hbase-operator-tools to avoid maven commit plugin bug --- hbase/Dockerfile | 11 ++++------- trino/Dockerfile | 3 ++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/hbase/Dockerfile b/hbase/Dockerfile index e3a4928b2..ae10f8054 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -152,13 +152,10 @@ WORKDIR /stackable # We need to explicitly give the uid to use RUN --mount=type=cache,id=maven-hbase-operator-tools-${HBASE_OPERATOR_TOOLS},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <