[pull] master from apache:master

.github/actions/setup-default-test-properties/test-properties.json

@@ -14,7 +14,7 @@
   },
   "JavaTestProperties": {
     "SUPPORTED_VERSIONS": ["8", "11", "17", "21", "25"],
-    "FLINK_VERSIONS": ["1.17", "1.18", "1.19", "1.20", "2.0"],
+    "FLINK_VERSIONS": ["1.17", "1.18", "1.19", "1.20", "2.0", "2.1"],
     "SPARK_VERSIONS": ["3"]
   },
   "GoTestProperties": {

.github/trigger_files/beam_PostCommit_Go.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 2
+    "modification": 3
 }

.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Dataflow.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 4
+  "modification": 5
 }

.github/workflows/run_rc_validation_java_quickstart.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Run QuickStart Java Flink Runner
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :runners:flink:2.0:runQuickstartJavaFlinkLocal
+          gradle-command: :runners:flink:2.1:runQuickstartJavaFlinkLocal
           arguments: |
             -Prepourl=${{ env.APACHE_REPO_URL }} \
             -Pver=${{ env.RELEASE_VERSION }}

.test-infra/validate-runner/build.gradle

@@ -31,6 +31,19 @@ repositories {
     }
 }
 
+// Flink 2.1+ uses at.yawk.lz4:lz4-java while Spark uses org.lz4:lz4-java
+// Resolve capability conflict by preferring Flink's version when both are present
+configurations.all {
+  resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java') {
+    def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4') }
+    if (candidate != null) {
+      select(candidate)
+    } else {
+      selectHighestVersion()
+    }
+  }
+}
+
 dependencies {
     implementation 'com.offbytwo.jenkins:jenkins-client:0.3.8'
     implementation library.java.jackson_databind

examples/java/common.gradle

@@ -35,6 +35,16 @@ configurations.sparkRunnerPreCommit {
   exclude group: "org.slf4j", module: "jul-to-slf4j"
   exclude group: "org.slf4j", module: "slf4j-jdk14"
 }
+configurations.flinkRunnerPreCommit {
+  resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") {
+    def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4') }
+    if (candidate != null) {
+      select(candidate)
+    } else {
+      selectHighestVersion()
+    }
+  }
+}
 
 dependencies {
   directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")

gradle.properties

@@ -39,7 +39,7 @@ docker_image_default_repo_root=apache
 docker_image_default_repo_prefix=beam_
 
 # supported flink versions
-flink_versions=1.17,1.18,1.19,1.20,2.0
+flink_versions=1.17,1.18,1.19,1.20,2.0,2.1
 # supported spark versions
 spark_versions=3,4
 # supported python versions

runners/flink/2.1/build.gradle

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+project.ext {
+  flink_major = '2.1'
+  flink_version = '2.1.3'
+  excluded_files = [
+    'main': [
+        // Used by DataSet API only
+        "org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapter.java",
+        "org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java",
+        "org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java",
+        "org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java",
+        "org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java",
+        // Moved to org.apache.flink.runtime.state.StateBackendFactory
+        "org/apache/beam/runners/flink/FlinkStateBackendFactory.java",
+    ],
+    'test': [
+        // Used by DataSet API only
+        "org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapterTest.java",
+        "org/apache/beam/runners/flink/batch/NonMergingGroupByKeyTest.java",
+        "org/apache/beam/runners/flink/batch/ReshuffleTest.java",
+    ]
+  ]
+}
+
+// Load the main build script which contains all build logic.
+apply from: "../flink_runner.gradle"
+
+// Flink 2.1 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
+// Explicitly prefer Flink's at.yawk.lz4 version to resolve capability conflict
+configurations.all {
+  resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java') {
+    def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4') }
+    if (candidate != null) {
+      select(candidate)
+    } else {
+      selectHighestVersion()
+    }
+  }
+}

runners/flink/2.1/job-server-container/build.gradle

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def basePath = '../../job-server-container'
+
+project.ext {
+  resource_path = basePath
+}
+
+// Load the main build script which contains all build logic.
+apply from: "$basePath/flink_job_server_container.gradle"

runners/flink/2.1/job-server/build.gradle

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def basePath = '../../job-server'
+
+project.ext {
+  // Look for the source code in the parent module
+  main_source_dirs = ["$basePath/src/main/java"]
+  test_source_dirs = ["$basePath/src/test/java"]
+  main_resources_dirs = ["$basePath/src/main/resources"]
+  test_resources_dirs = ["$basePath/src/test/resources"]
+  archives_base_name = 'beam-runners-flink-2.1-job-server'
+}
+
+// Load the main build script which contains all build logic.
+apply from: "$basePath/flink_job_server.gradle"
+
+// Flink 2.1 uses at.yawk.lz4:lz4-java instead of org.lz4:lz4-java
+// Explicitly prefer Flink's at.yawk.lz4 version to resolve capability conflict
+configurations.all {
+  resolutionStrategy.capabilitiesResolution.withCapability('org.lz4:lz4-java') {
+    def candidate = candidates.find { it.id.toString().contains('at.yawk.lz4') }
+    if (candidate != null) {
+      select(candidate)
+    } else {
+      selectHighestVersion()
+    }
+  }
+}

scripts/beam-sql.sh

@@ -36,6 +36,8 @@ MAVEN_DISTRIBUTION_URL="https://repo.maven.apache.org/maven2/org/apache/maven/ap
 
 # Maven Plugin Configuration
 MAVEN_SHADE_PLUGIN_VERSION="3.5.1"
+ICEBERG_VERSION="${ICEBERG_VERSION:-1.10.0}"
+APACHE_SNAPSHOT_REPOSITORY_URL="https://repository.apache.org/content/repositories/snapshots/"
 mkdir -p "${CACHE_DIR}"
 
 # Create a temporary directory for our Maven project.
@@ -84,13 +86,46 @@ function setup_maven_wrapper() {
   MAVEN_CMD="${mvnw_script}"
 }
 
+function add_beam_dependency() {
+  local artifact_id="$1"
+  cat >> "${POM_FILE}" << EOL
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>${artifact_id}</artifactId>
+            <version>\${beam.version}</version>
+        </dependency>
+EOL
+}
+
+function add_dependency() {
+  local group_id="$1"
+  local artifact_id="$2"
+  local version="$3"
+  cat >> "${POM_FILE}" << EOL
+        <dependency>
+            <groupId>${group_id}</groupId>
+            <artifactId>${artifact_id}</artifactId>
+            <version>${version}</version>
+        </dependency>
+EOL
+}
+
+function normalize_beam_version() {
+  case "${BEAM_VERSION}" in
+    *-[sS][nN][aA][pP][sS][hH][oO][tT])
+      BEAM_VERSION="${BEAM_VERSION%-*}-SNAPSHOT"
+      ;;
+  esac
+}
+
 function usage() {
   echo "Usage: $0 [--version <beam_version>] [--runner <runner_name>] [--io <io_connector>] [--list-versions] [--list-ios] [--list-runners] [--debug] [-h|--help]"
   echo ""
   echo "A self-contained launcher for the Apache Beam SQL Shell."
   echo ""
   echo "Options:"
   echo "  --version   Specify the Apache Beam version (default: ${DEFAULT_BEAM_VERSION})."
+  echo "              SNAPSHOT versions are resolved from Apache's Maven snapshot repository."
   echo "  --runner    Specify the Beam runner to use (default: direct)."
   echo "              Supported runners:"
   echo "                direct    - DirectRunner (runs locally, good for development)"
@@ -208,7 +243,7 @@ function list_runners() {
 
   echo "✅ Available runners for Beam ${BEAM_VERSION}:"
   echo ""
-  
+
   # Process each runner and provide descriptions
   while IFS= read -r runner; do
     case "$runner" in
@@ -217,7 +252,7 @@ function list_runners() {
         echo "                     Runs locally on your machine. Good for development and testing."
         ;;
       "google-cloud-dataflow-java")
-        echo "  dataflow         - DataflowRunner" 
+        echo "  dataflow         - DataflowRunner"
         echo "                     Runs on Google Cloud Dataflow for production workloads."
         ;;
       flink-*)
@@ -273,7 +308,7 @@ function list_runners() {
         ;;
     esac
   done <<< "$runners"
-  
+
   echo ""
   echo "💡 Usage: ./beam-sql.sh --runner <runner_name>"
   echo "   Default: direct"
@@ -291,7 +326,7 @@ DEBUG_MODE=false
 
 while [[ "$#" -gt 0 ]]; do
   case $1 in
-    --version) BEAM_VERSION="$2"; shift ;;
+    --version) BEAM_VERSION="$2"; normalize_beam_version; shift ;;
     --runner) BEAM_RUNNER=$(echo "$2" | tr '[:upper:]' '[:lower:]'); shift ;;
     --io) IO_CONNECTORS+=("$2"); shift ;;
     --list-versions) list_versions; exit 0 ;;
@@ -363,7 +398,15 @@ else
 EOL
 # Add IO and Runner dependencies
   for io in "${IO_CONNECTORS[@]}"; do
-  echo "        <dependency><groupId>org.apache.beam</groupId><artifactId>beam-sdks-java-io-${io}</artifactId><version>\${beam.version}</version></dependency>" >> "${POM_FILE}"
+    add_beam_dependency "beam-sdks-java-io-${io}"
+    case "${io}" in
+      iceberg)
+        add_beam_dependency "beam-sdks-java-extensions-sql-iceberg"
+        add_dependency "org.apache.iceberg" "iceberg-aws" "${ICEBERG_VERSION}"
+        add_dependency "org.apache.iceberg" "iceberg-aws-bundle" "${ICEBERG_VERSION}"
+        add_dependency "org.apache.iceberg" "iceberg-gcp" "${ICEBERG_VERSION}"
+        ;;
+    esac
   done
   RUNNER_ARTIFACT=""
   case "${BEAM_RUNNER}" in
@@ -372,12 +415,25 @@ EOL
     *) echo "❌ Error: Unsupported runner '${BEAM_RUNNER}'." >&2; exit 1 ;;
   esac
   if [ -n "${RUNNER_ARTIFACT}" ]; then
-  echo "        <dependency><groupId>org.apache.beam</groupId><artifactId>${RUNNER_ARTIFACT}</artifactId><version>\${beam.version}</version></dependency>" >> "${POM_FILE}"
+    add_beam_dependency "${RUNNER_ARTIFACT}"
   fi
 
 # Complete the POM with the build section for the maven-shade-plugin
 cat >> "${POM_FILE}" << EOL
     </dependencies>
+    <repositories>
+        <repository>
+            <id>apache.snapshots</id>
+            <url>${APACHE_SNAPSHOT_REPOSITORY_URL}</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+                <updatePolicy>always</updatePolicy>
+            </snapshots>
+        </repository>
+    </repositories>
     <properties>
       <beam.version>${BEAM_VERSION}</beam.version>
     </properties>
@@ -395,6 +451,11 @@ cat >> "${POM_FILE}" << EOL
                         </goals>
                         <configuration>
                             <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <manifestEntries>
+                                        <Multi-Release>true</Multi-Release>
+                                    </manifestEntries>
+                                </transformer>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                             </transformers>
                             <filters>

sdks/go/examples/wasm/README.md

@@ -68,13 +68,13 @@ cd $BEAM_HOME
 Expected output should include the following, from which you acquire the latest flink runner version.
 
 ```shell
-'flink_versions: 1.17,1.18,1.19,1.20'
+'flink_versions: 1.17,1.18,1.19,1.20,2.0,2.1'
 ```
 
-#### 2. Set to the latest flink runner version i.e. 1.16
+#### 2. Set to the latest flink runner version i.e. 2.1
 
 ```shell
-FLINK_VERSION=1.16
+FLINK_VERSION=2.1
 ```
 
 #### 3. In a separate terminal, start the flink runner (It should take a few minutes on the first execution)

sdks/go/test/build.gradle

@@ -36,8 +36,10 @@ task dataflowValidatesRunner() {
   dependsOn ":sdks:go:test:goBuild"
 
   doLast {
+    def pipelineOptions = ["--no_use_public_ips"]
     def options = [
         "--runner dataflow",
+        "--pipeline_opts \"${pipelineOptions.join(' ')}\"",
     ]
     exec {
       if (fork_java_home) {
@@ -59,6 +61,7 @@ task dataflowValidatesRunnerARM64() {
   doLast {
     def pipelineOptions = [  // Pipeline options piped directly to Go SDK flags.
         "--machine_type=t2a-standard-1",
+        "--no_use_public_ips",
     ]
     def options = [
         "--runner dataflow",
@@ -89,7 +92,7 @@ task flinkValidatesRunner {
   doFirst {
     // Copy Flink conf file
     copy {
-      from "${project.rootDir}/runners/flink/2.0/src/test/resources/flink-test-config.yaml"
+      from "${project.rootDir}/runners/flink/${flinkVersion}/src/test/resources/flink-test-config.yaml"
       into "${project.buildDir}/flink-conf"
 
       // Rename the file during the copy process