diff --git a/docs/src/site/markdown/install.md b/docs/src/site/markdown/install.md index 44ca862a82..fddc76d160 100644 --- a/docs/src/site/markdown/install.md +++ b/docs/src/site/markdown/install.md @@ -19,29 +19,28 @@ Install/Deploy Instructions for Tez --------------------------------------------------------------------------- -Replace x.y.z with the tez release number that you are using. E.g. 0.5.0. For Tez -versions 0.8.3 and higher, Tez needs Apache Hadoop to be of version 2.6.0 or higher. -For Tez version 0.9.0 and higher, Tez needs Apache Hadoop to be version 2.7.0 -or higher. -1. Deploy Apache Hadoop using version of 2.7.0 or higher. - - You need to change the value of the hadoop.version property in the +Replace x.y.z with the tez release number that you are using. E.g. 0.5.0. Tez +needs Apache Hadoop to be of version 3.x. + +1. Deploy Apache Hadoop using version of 3.x. + - You need to change the value of the hadoop.version property in the top-level pom.xml to match the version of the hadoop branch being used. ``` - $ hadoop version + hadoop version ``` -2. Build tez using `mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true` - - This assumes that you have already installed JDK8 or later and Maven 3 or later. - - Tez also requires Protocol Buffers 3.19.4, including the protoc-compiler. - * This can be downloaded from https://github.com/google/protobuf/tags/. - * On Mac OS X with the homebrew package manager `brew install protobuf250` - * For rpm-based linux systems, the yum repos may not have the 3.19.4 version. +2. Build tez using `mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true` + - This assumes that you have already installed JDK8 or later and Maven 3 or later. + - Tez also requires Protocol Buffers 3.19.4, including the protoc-compiler. + - This can be downloaded from . + - On Mac OS X with the homebrew package manager `brew install protobuf250` + - For rpm-based linux systems, the yum repos may not have the 3.19.4 version. `rpm.pbone.net` has the protobuf-3.19.4 and protobuf-compiler-3.19.4 packages. - - If you prefer to run the unit tests, remove skipTests from the + - If you prefer to run the unit tests, remove skipTests from the command above. - - If you use Eclipse IDE, you can import the projects using + - If you use Eclipse IDE, you can import the projects using "Import/Maven/Existing Maven Projects". Eclipse does not automatically generate Java sources or include the generated sources into the projects. Please build using maven as described @@ -50,10 +49,10 @@ or higher. "Java Build Path" for these projects: tez-api, tez-mapreduce, tez-runtime-internals and tez-runtime-library. This needs to be done just once after importing the project. -3. Copy the relevant tez tarball into HDFS, and configure tez-site.xml - - A tez tarball containing tez and hadoop libraries will be found +3. Copy the relevant tez tarball into HDFS, and configure tez-site.xml + - A tez tarball containing tez and hadoop libraries will be found at tez-dist/target/tez-x.y.z-SNAPSHOT.tar.gz - - Assuming that the tez jars are put in /apps/ on HDFS, the + - Assuming that the tez jars are put in /apps/ on HDFS, the command would be ``` @@ -61,22 +60,22 @@ or higher. hadoop fs -copyFromLocal tez-dist/target/tez-x.y.z-SNAPSHOT.tar.gz /apps/tez-x.y.z-SNAPSHOT/ ``` - - tez-site.xml configuration. - - Set tez.lib.uris to point to the tar.gz uploaded to HDFS. + - tez-site.xml configuration. + - Set tez.lib.uris to point to the tar.gz uploaded to HDFS. Assuming the steps mentioned so far were followed, set tez.lib.uris to `${fs.defaultFS}/apps/tez-x.y.z-SNAPSHOT/tez-x.y.z-SNAPSHOT.tar.gz` - - Ensure tez.use.cluster.hadoop-libs is not set in tez-site.xml, + - Ensure tez.use.cluster.hadoop-libs is not set in tez-site.xml, or if it is set, the value should be false - - Please note that the tarball version should match the version of + - Please note that the tarball version should match the version of the client jars used when submitting Tez jobs to the cluster. Please refer to the [Version Compatibility Guide](https://cwiki.apache.org/confluence/display/TEZ/Version+Compatibility) for more details on version compatibility and detecting mismatches. -4. Optional: If running existing MapReduce jobs on Tez. Modify +4. Optional: If running existing MapReduce jobs on Tez. Modify mapred-site.xml to change "mapreduce.framework.name" property from its default value of "yarn" to "yarn-tez" -5. Configure the client node to include the tez-libraries in the hadoop +5. Configure the client node to include the tez-libraries in the hadoop classpath - - Extract the tez minimal tarball created in step 2 to a local directory + - Extract the tez minimal tarball created in step 2 to a local directory (assuming TEZ_JARS is where the files will be decompressed for the next steps) @@ -84,18 +83,18 @@ or higher. tar -xvzf tez-dist/target/tez-x.y.z-minimal.tar.gz -C $TEZ_JARS ``` - - set TEZ_CONF_DIR to the location of tez-site.xml - - Add $TEZ_CONF_DIR, ${TEZ_JARS}/* and ${TEZ_JARS}/lib/* to the application classpath. + - set TEZ_CONF_DIR to the location of tez-site.xml + - Add $TEZ_CONF_DIR, ${TEZ_JARS}/*and ${TEZ_JARS}/lib/* to the application classpath. For example, doing it via the standard Hadoop tool chain would use the following command - to set up the application classpath: + to set up the application classpath: ``` export HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/* ``` - - Please note the "*" which is an important requirement when + - Please note the "*" which is an important requirement when setting up classpaths for directories containing jar files. -6. There is a basic example of using an MRR job in the tez-examples.jar. +6. There is a basic example of using an MRR job in the tez-examples.jar. Refer to OrderedWordCount.java in the source code. To run this example: @@ -124,7 +123,8 @@ or higher. ``` $HADOOP_PREFIX/bin/hadoop jar tez-tests.jar testorderedwordcount -DUSE_TEZ_SESSION=true ``` -7. Submit a MR job as you normally would using something like: + +7. Submit a MR job as you normally would using something like: ``` $HADOOP_PREFIX/bin/hadoop jar hadoop-mapreduce-client-jobclient-3.0.0-SNAPSHOT-tests.jar sleep -mt 1 -rt 1 -m 1 -r 1 @@ -137,11 +137,12 @@ or higher. Various ways to configure tez.lib.uris --------------------------------------- -The `tez.lib.uris` configuration property supports a comma-separated list of values. The +The `tez.lib.uris` configuration property supports a comma-separated list of values. The types of values supported are: - - Path to simple file - - Path to a directory - - Path to a compressed archive ( tarball, zip, etc). + +- Path to simple file +- Path to a directory +- Path to a compressed archive ( tarball, zip, etc). For simple files and directories, Tez will add all these files and first-level entries in the directories (recursive traversal of dirs is not supported) into the working directory of the @@ -161,11 +162,11 @@ recommended method for installation. A full tarball with all dependencies is a b that existing jobs continue to run during a cluster's rolling upgrade. Although the `tez.lib.uris` configuration options enable a wide variety of usage patterns, there -are 2 main alternative modes that are supported by the framework: +are 2 main alternative modes that are supported by the framework: 1. Mode A: Using a tez tarball on HDFS along with Hadoop libraries available on the cluster. 2. Mode B: Using a tez tarball along with the Hadoop tarball. - + Both these modes will require a tez build without Hadoop dependencies and that is available at tez-dist/target/tez-x.y.z-minimal.tar.gz. @@ -177,7 +178,7 @@ to ensure that the tez version being used is compatible with the version of Hado Step 3 above changes as follows. Also subsequent steps should use tez-dist/target/tez-x.y.z-minimal.tar.gz instead of tez-dist/target/tez-x.y.z.tar.gz - - A tez build without Hadoop dependencies will be available at tez-dist/target/tez-x.y.z-minimal.tar.gz +- A tez build without Hadoop dependencies will be available at tez-dist/target/tez-x.y.z-minimal.tar.gz Assuming that the tez jars are put in /apps/ on HDFS, the command would be ``` @@ -185,10 +186,10 @@ instead of tez-dist/target/tez-x.y.z.tar.gz "hadoop fs -copyFromLocal tez-dist/target/tez-x.y.z-minimal.tar.gz /apps/tez-x.y.z" ``` - - tez-site.xml configuration - - Set tez.lib.uris to point to the paths in HDFS containing the tez jars. Assuming the steps mentioned so far were followed, +- tez-site.xml configuration + - Set tez.lib.uris to point to the paths in HDFS containing the tez jars. Assuming the steps mentioned so far were followed, set tez.lib.uris to `${fs.defaultFS}/apps/tez-x.y.z/tez-x.y.z-minimal.tar.gz` - - Set tez.use.cluster.hadoop-libs to true + - Set tez.use.cluster.hadoop-libs to true For Mode B: Tez tarball with Hadoop tarball -------------------------------------------- @@ -198,7 +199,7 @@ versions of Tez and Hadoop being used are compatible. To do this configuration, we need to change Step 3 of the default instructions in the following ways. - - Assuming that the tez archives/jars are put in /apps/ on HDFS, the command to put this +- Assuming that the tez archives/jars are put in /apps/ on HDFS, the command to put this minimal Tez archive into HDFS would be: ``` @@ -206,7 +207,7 @@ minimal Tez archive into HDFS would be: "hadoop fs -copyFromLocal tez-dist/target/tez-x.y.z-minimal.tar.gz /apps/tez-x.y.z" ``` - - Alternatively, you can put the minimal directory directly into HDFS and +- Alternatively, you can put the minimal directory directly into HDFS and reference the jars, instead of using an archive. The command to put the minimal directory into HDFS would be: @@ -214,9 +215,9 @@ minimal Tez archive into HDFS would be: "hadoop fs -copyFromLocal tez-dist/target/tez-x.y.z-minimal/* /apps/tez-x.y.z" ``` - - After building hadoop, the hadoop tarball will be available at +- After building hadoop, the hadoop tarball will be available at hadoop/hadoop-dist/target/hadoop-x.y.z-SNAPSHOT.tar.gz - - Assuming that the hadoop jars are put in /apps/ on HDFS, the command to put this +- Assuming that the hadoop jars are put in /apps/ on HDFS, the command to put this Hadoop archive into HDFS would be: ``` @@ -224,25 +225,26 @@ minimal Tez archive into HDFS would be: "hadoop fs -copyFromLocal hadoop-dist/target/hadoop-x.y.z-SNAPSHOT.tar.gz /apps/hadoop-x.y.z" ``` - - tez-site.xml configuration - - Set tez.lib.uris to point to the the archives and jars that are needed for Tez/Hadoop. +- tez-site.xml configuration + - Set tez.lib.uris to point to the the archives and jars that are needed for Tez/Hadoop. - - Example: When using both Tez and Hadoop archives, set tez.lib.uris to + - Example: When using both Tez and Hadoop archives, set tez.lib.uris to `${fs.defaultFS}/apps/tez-x.y.z/tez-x.y.z-minimal.tar.gz#tez,${fs.defaultFS}/apps/hadoop-x.y.z/hadoop-x.y.z-SNAPSHOT.tar.gz#hadoop-mapreduce` - - Example: When using Tez jars with a Hadoop archive, set tez.lib.uris to: + - Example: When using Tez jars with a Hadoop archive, set tez.lib.uris to: `${fs.defaultFS}/apps/tez-x.y.z,${fs.defaultFS}/apps/tez-x.y.z/lib,${fs.defaultFS}/apps/hadoop-x.y.z/hadoop-x.y.z-SNAPSHOT.tar.gz#hadoop-mapreduce` - - In tez.lib.uris, the text immediately following the '#' symbol is the fragment that + - In tez.lib.uris, the text immediately following the '#' symbol is the fragment that refers to the symlink that will be created for the archive. If no fragment is given, the symlink will be set to the name of the archive. Fragments should not be given to directories or jars. - - If any archives are specified in tez.lib.uris, then tez.lib.uris.classpath must be set - to define the classpath for these archives as the archive structure is not known. - - Example: Classpath when using both Tez and Hadoop archives, set tez.lib.uris.classpath to: + - If any archives are specified in tez.lib.uris, then tez.lib.uris.classpath must be set + to define the classpath for these archives as the archive structure is not known. + - Example: Classpath when using both Tez and Hadoop archives, set tez.lib.uris.classpath to: ``` + ./tez/*:./tez/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/common/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/common/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/hdfs/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/hdfs/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/yarn/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/yarn/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/mapreduce/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/mapreduce/lib/* ``` @@ -252,8 +254,5 @@ minimal Tez archive into HDFS would be: ./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/common/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/common/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/hdfs/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/hdfs/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/yarn/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/yarn/lib/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/mapreduce/*:./hadoop-mapreduce/hadoop-x.y.z-SNAPSHOT/share/hadoop/mapreduce/lib/* ``` - [Install instructions for older versions of Tez (pre 0.5.0)](./install_pre_0_5_0.html) ----------------------------------------------------------------------------------- - - diff --git a/hadoop-shim-impls/findbugs-exclude.xml b/hadoop-shim-impls/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/hadoop-shim-impls/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.7/pom.xml b/hadoop-shim-impls/hadoop-shim-2.7/pom.xml deleted file mode 100644 index 441c253310..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - hadoop-shim-impls - org.apache.tez - 1.0.0-SNAPSHOT - - 4.0.0 - hadoop-shim-2.7 - - - - org.slf4j - slf4j-api - - - junit - junit - - - org.apache.tez - hadoop-shim - - - org.apache.tez - hadoop-shim - test - test-jar - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-common - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java deleted file mode 100644 index bd1eabfa32..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.classification.InterfaceAudience.Private; - -@Private -public class HadoopShim25_26_27Provider extends HadoopShimProvider { - - @Override - public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { - if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) { - return new HadoopShim27(); - } - return null; - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java deleted file mode 100644 index c5163036a1..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; - -@Private -public class HadoopShim27 extends HadoopShim { - - @Override - public void setHadoopCallerContext(String context) { - // Not supported - } - - @Override - public void clearHadoopCallerContext() { - // Not supported - } - - @Override - public Set getSupportedResourceTypes(RegisterApplicationMasterResponse response) { - EnumSet supportedResourceTypes = response.getSchedulerResourceTypes(); - Set supportedTypes = new HashSet(); - for (SchedulerResourceTypes resourceType : supportedResourceTypes) { - supportedTypes.add(resourceType.name()); - } - return supportedTypes; - } - -} diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider deleted file mode 100644 index 4226f1861e..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java deleted file mode 100644 index c9937f8b94..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoop25_26_27ShimProvider { - - @Test - public void testShimProvider() { - HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider(); - Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 3)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 8)); - Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7)); - - Assert.assertEquals(HadoopShim27.class, - provider.createHadoopShim("foo", 2, 7).getClass()); - } - - @Test - public void testLoaderOverride() { - Configuration conf = new Configuration(false); - // Set shim and version to ensure that hadoop version from the build does not create - // a mismatch - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim25_26_27Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(HadoopShim27.class, shim.getClass()); - } - - @Test - public void testInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that shim in this module does not match - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - - @Test - public void testLoaderOverrideInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that override shim does not return a valid shim - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim25_26_27Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml deleted file mode 100644 index 58d4c1ea5e..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - hadoop-shim-impls - org.apache.tez - 1.0.0-SNAPSHOT - - 4.0.0 - hadoop-shim-2.8 - - - - org.slf4j - slf4j-api - - - junit - junit - - - org.apache.tez - hadoop-shim - - - org.apache.tez - hadoop-shim - test - test-jar - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-common - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java deleted file mode 100644 index c0d5577a4b..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.ipc.CallerContext; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; - -public class HadoopShim28 extends HadoopShim { - - final static CallerContext nullCallerContext = new CallerContext.Builder("").build(); - - @Override - public void setHadoopCallerContext(String context) { - CallerContext.setCurrent(new CallerContext.Builder(context).build()); - } - - @Override - public void clearHadoopCallerContext() { - CallerContext.setCurrent(nullCallerContext); - } - - @Override - public Set getSupportedResourceTypes(RegisterApplicationMasterResponse response) { - EnumSet supportedResourceTypes = response.getSchedulerResourceTypes(); - Set supportedTypes = new HashSet(); - for (SchedulerResourceTypes resourceType : supportedResourceTypes) { - supportedTypes.add(resourceType.name()); - } - return supportedTypes; - } - - @Override - public FinalApplicationStatus applyFinalApplicationStatusCorrection(FinalApplicationStatus orig, - boolean isSessionMode, boolean isError) { - switch (orig) { - case FAILED: - // App is failed if dag failed in non-session mode or there was an error. - return (!isSessionMode || isError) ? - FinalApplicationStatus.FAILED : FinalApplicationStatus.ENDED; - case SUCCEEDED: - return isSessionMode ? FinalApplicationStatus.ENDED : FinalApplicationStatus.SUCCEEDED; - default: - return orig; - } - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java deleted file mode 100644 index 318462d7e3..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -public class HadoopShim28Provider extends HadoopShimProvider { - - @Override - public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { - if (majorVersion == 2 && minorVersion >= 8) { - return new HadoopShim28(); - } - return null; - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider deleted file mode 100644 index 24fe49d1f5..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.tez.hadoop.shim.HadoopShim28Provider diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28.java deleted file mode 100644 index f3117c89dd..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; - -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoopShim28 { - - @Test - public void testApplyFinalApplicationStatusCorrection() { - HadoopShim shim = new HadoopShim28(); - // Session mode success/failure, change to ended - Assert.assertEquals(FinalApplicationStatus.ENDED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.SUCCEEDED, true, false)); - Assert.assertEquals(FinalApplicationStatus.ENDED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, true, false)); - - // Non-session mode success/failure, retain success/failure - Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.SUCCEEDED, false, false)); - Assert.assertEquals(FinalApplicationStatus.FAILED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, false, false)); - - // Session and non-session mode error, retain failed. - Assert.assertEquals(FinalApplicationStatus.FAILED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, true, true)); - Assert.assertEquals(FinalApplicationStatus.FAILED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, false, true)); - - // Session and non-session mode killed is killed. - Assert.assertEquals(FinalApplicationStatus.KILLED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.KILLED, true, false)); - Assert.assertEquals(FinalApplicationStatus.KILLED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.KILLED, false, false)); - - // Session and non-session mode undefined is undefined. - Assert.assertEquals(FinalApplicationStatus.UNDEFINED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.UNDEFINED, true, false)); - Assert.assertEquals(FinalApplicationStatus.UNDEFINED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.UNDEFINED, false, false)); - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java deleted file mode 100644 index f2f5e5ff71..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.conf.Configuration; - -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoopShim28Provider { - - @Test - public void testShimProvider() { - HadoopShim28Provider provider = new HadoopShim28Provider(); - Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 7)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 5)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 6)); - Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111)); - - Assert.assertEquals(HadoopShim28.class, - provider.createHadoopShim("foo", 2, 9).getClass()); - } - - @Test - public void testLoaderOverride() { - Configuration conf = new Configuration(false); - // Set shim and version to ensure that hadoop version from the build does not create - // a mismatch - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim28Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.8.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(HadoopShim28.class, shim.getClass()); - } - - @Test - public void testInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that shim in this module does not match - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.2.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - - @Test - public void testLoaderOverrideInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that override shim does not return a valid shim - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim28Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - -} diff --git a/hadoop-shim-impls/pom.xml b/hadoop-shim-impls/pom.xml deleted file mode 100644 index fc3b2dcf8d..0000000000 --- a/hadoop-shim-impls/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - 4.0.0 - - tez - org.apache.tez - 1.0.0-SNAPSHOT - - hadoop-shim-impls - pom - - - - hadoop27 - - false - - - hadoop-shim-2.7 - - - - hadoop28 - - true - - - hadoop-shim-2.8 - - - - - diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java index e7c7163e58..0a1ce83402 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java @@ -18,11 +18,40 @@ */ package org.apache.tez.hadoop.shim; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; /** - * Default Hadoop Shim. For now, it mimics HadoopShim. + * Default Hadoop Shim. Provides Hadoop 3.x baseline capabilities. */ @Private public class DefaultHadoopShim extends HadoopShim { + + private static final CallerContext nullCallerContext = new CallerContext.Builder("").build(); + + @Override + public void setHadoopCallerContext(String context) { + CallerContext.setCurrent(new CallerContext.Builder(context).build()); + } + + @Override + public void clearHadoopCallerContext() { + CallerContext.setCurrent(nullCallerContext); + } + + @Override + public Set getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + EnumSet supportedResourceTypes = response.getSchedulerResourceTypes(); + Set supportedTypes = new HashSet(); + for (SchedulerResourceTypes resourceType : supportedResourceTypes) { + supportedTypes.add(resourceType.name()); + } + return supportedTypes; + } } diff --git a/pom.xml b/pom.xml index ba08baeea5..0216f1ed52 100644 --- a/pom.xml +++ b/pom.xml @@ -685,6 +685,11 @@ + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + org.apache.hadoop hadoop-hdfs @@ -837,7 +842,6 @@ tez-ui tez-plugins tez-tools - hadoop-shim-impls tez-dist docs diff --git a/tez-api/pom.xml b/tez-api/pom.xml index 58da12ae21..62ce1bf85c 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -101,6 +101,15 @@ com.sun.jersey jersey-json + + org.apache.hadoop + hadoop-hdfs-client + + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs @@ -221,38 +230,4 @@ - - - hadoop27 - - false - - - - org.apache.hadoop - hadoop-hdfs - runtime - - - - - hadoop28 - - true - - - - org.apache.hadoop - hadoop-hdfs-client - ${hadoop.version} - - - org.apache.hadoop - hadoop-hdfs - test - ${hadoop.version} - - - - diff --git a/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java b/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java index 50af86039c..0fa20b64b1 100644 --- a/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java +++ b/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java @@ -18,8 +18,6 @@ */ package org.apache.tez.common; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.concurrent.Executor; import com.google.common.util.concurrent.MoreExecutors; @@ -29,27 +27,10 @@ */ public final class GuavaShim { - static { - try { - executorMethod = MoreExecutors.class.getDeclaredMethod("directExecutor"); - } catch (NoSuchMethodException nsme) { - try { - executorMethod = MoreExecutors.class.getDeclaredMethod("sameThreadExecutor"); - } catch (NoSuchMethodException nsmeSame) { - } - } - } - private GuavaShim() { } - private static Method executorMethod; - public static Executor directExecutor() { - try { - return (Executor) executorMethod.invoke(null); - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new RuntimeException(e); - } + return MoreExecutors.directExecutor(); } } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java index 8c8d2abb49..405cbb5f2b 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java @@ -34,15 +34,15 @@ public class TestTimelineReaderFactory { - // ensure on hadoop 2.6+ TimelineReaderTokenAuthenticatedStrategy is used. + // ensure TimelineReaderTokenAuthenticatedStrategy is used. @Test(timeout = 5000) - public void testShouldUseTokenDelegationAuthStrategyForHadoop26() throws TezException { + public void testShouldUseTokenDelegationAuthStrategy() throws TezException { String returnedClassName = TimelineReaderFactory.getTimelineReaderStrategy(mock(Configuration.class), false, 0) .getClass() .getCanonicalName(); - Assert.assertEquals("should use pseudo auth on hadoop2.4", + Assert.assertEquals("should use token delegation auth", "org.apache.tez.dag.api.client.TimelineReaderFactory.TimelineReaderTokenAuthenticatedStrategy", returnedClassName); } diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml index 08a8077cca..5c7b0a3d12 100644 --- a/tez-dist/pom.xml +++ b/tez-dist/pom.xml @@ -58,6 +58,31 @@ ${project.version} provided + + org.apache.tez + tez-yarn-timeline-history-with-acls + ${project.version} + + + org.apache.tez + tez-yarn-timeline-history-with-fs + ${project.version} + + + org.apache.tez + tez-yarn-timeline-cache-plugin + ${project.version} + + + org.apache.tez + tez-protobuf-history-plugin + ${project.version} + + + org.apache.tez + hadoop-shim + ${project.version} + @@ -65,62 +90,6 @@ - - hadoop27 - - false - - - - org.apache.tez - tez-yarn-timeline-history-with-acls - ${project.version} - - - org.apache.tez - tez-protobuf-history-plugin - ${project.version} - - - org.apache.tez - hadoop-shim-2.7 - ${project.version} - - - - - hadoop28 - - true - - - - org.apache.tez - tez-yarn-timeline-history-with-acls - ${project.version} - - - org.apache.tez - tez-yarn-timeline-history-with-fs - ${project.version} - - - org.apache.tez - tez-yarn-timeline-cache-plugin - ${project.version} - - - org.apache.tez - tez-protobuf-history-plugin - ${project.version} - - - org.apache.tez - hadoop-shim-2.8 - ${project.version} - - - docker diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml index 7001858ddb..88317f810e 100644 --- a/tez-plugins/pom.xml +++ b/tez-plugins/pom.xml @@ -29,37 +29,15 @@ tez-plugins pom - - - hadoop27 - - false - - - tez-protobuf-history-plugin - tez-yarn-timeline-history - tez-yarn-timeline-history-with-acls - tez-history-parser - tez-aux-services - - - - hadoop28 - - true - - - tez-protobuf-history-plugin - tez-yarn-timeline-history - tez-yarn-timeline-history-with-acls - tez-yarn-timeline-cache-plugin - tez-yarn-timeline-history-with-fs - tez-history-parser - tez-aux-services - - - - + + tez-protobuf-history-plugin + tez-yarn-timeline-history + tez-yarn-timeline-history-with-acls + tez-yarn-timeline-cache-plugin + tez-yarn-timeline-history-with-fs + tez-history-parser + tez-aux-services + diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml index 8406500110..692226ed8b 100644 --- a/tez-plugins/tez-yarn-timeline-history/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history/pom.xml @@ -149,17 +149,4 @@ - - - hadoop27 - - - org.apache.tez - tez-yarn-timeline-history - ${project.version} - - - - - diff --git a/tez-tools/analyzers/pom.xml b/tez-tools/analyzers/pom.xml index aeda3a4631..cb971f398f 100644 --- a/tez-tools/analyzers/pom.xml +++ b/tez-tools/analyzers/pom.xml @@ -29,25 +29,8 @@ tez-perf-analyzer pom - - - hadoop27 - - false - - - job-analyzer - - - - hadoop28 - - true - - - job-analyzer - - - + + job-analyzer +