infra: upgrade spark to 4.0.1 (#2566)

<!--
Thanks for opening a pull request!
-->

<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->

# Rationale for this change
This PR changes the integration test to use Spark 4.0.1 with scala 2.13
and hadoop 3.4.1

Hadoop 3.4.x started to use `software.amazon.awssdk.bundle` instead of
`com.amazonaws.aws-java-sdk-bundle`

https://hadoop.apache.org/docs/r3.4.1/hadoop-aws/dependency-analysis.html

## Are these changes tested?

## Are there any user-facing changes?

<!-- In the case of user-facing changes, please add the changelog label.
-->

Author: kevinjqliu

dev/spark/Dockerfile

@@ -13,17 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BASE_IMAGE_SPARK_VERSION=3.5.6
+ARG BASE_IMAGE_SPARK_VERSION=4.0.1
 
 FROM apache/spark:${BASE_IMAGE_SPARK_VERSION}
 
 # Dependency versions - keep these compatible
 ARG ICEBERG_VERSION=1.10.0
-ARG ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
-ARG SPARK_VERSION=3.5.6
-ARG SCALA_VERSION=2.12
-ARG HADOOP_VERSION=3.3.4
-ARG AWS_SDK_VERSION=1.12.753
+ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13
+ARG SPARK_VERSION=4.0.1
+ARG HADOOP_VERSION=3.4.1
+ARG SCALA_VERSION=2.13
+ARG AWS_SDK_VERSION=2.24.6
 ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2
 
 USER root
@@ -47,7 +47,7 @@ ENV JARS_TO_DOWNLOAD="\
     org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
     org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
     org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
-    com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar"
+    software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"
 
 # Download JARs with retry logic
 RUN set -e && \

dev/spark/spark-defaults.conf

@@ -48,3 +48,5 @@ spark.sql.defaultCatalog               rest
 spark.ui.enabled                       true
 spark.eventLog.enabled                 true
 spark.eventLog.dir                     /home/iceberg/spark-events
+
+spark.sql.ansi.enabled                 false