Skip to content

Commit afe46ba

Browse files
mateizmateiz
committed
Merge pull request #892 from jey/fix-yarn-assembly
YARN build fixes
2 parents 2eebeff + 30a32c8 commit afe46ba

File tree

7 files changed

+145
-205
lines changed

7 files changed

+145
-205
lines changed

‎docs/running-on-yarn.md

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ This would be used to connect to the cluster, write to the dfs and submit jobs t
4242

4343
The command to launch the YARN Client is as follows:
4444

45-
SPARK_JAR=<SPARK_YARN_JAR_FILE> ./spark-class org.apache.spark.deploy.yarn.Client \
45+
SPARK_JAR=<SPARK_ASSEMBLY_JAR_FILE> ./spark-class org.apache.spark.deploy.yarn.Client \
4646
--jar <YOUR_APP_JAR_FILE> \
4747
--class <APP_MAIN_CLASS> \
4848
--args <APP_MAIN_ARGUMENTS> \
@@ -54,14 +54,27 @@ The command to launch the YARN Client is as follows:
5454

5555
For example:
5656

57-
SPARK_JAR=./yarn/target/spark-yarn-assembly-{{site.SPARK_VERSION}}.jar ./spark-class org.apache.spark.deploy.yarn.Client \
58-
--jar examples/target/scala-{{site.SCALA_VERSION}}/spark-examples_{{site.SCALA_VERSION}}-{{site.SPARK_VERSION}}.jar \
59-
--class org.apache.spark.examples.SparkPi \
60-
--args yarn-standalone \
61-
--num-workers 3 \
62-
--master-memory 4g \
63-
--worker-memory 2g \
64-
--worker-cores 1
57+
# Build the Spark assembly JAR and the Spark examples JAR
58+
$ SPARK_HADOOP_VERSION=2.0.5-alpha SPARK_YARN=true ./sbt/sbt assembly
59+
60+
# Configure logging
61+
$ cp conf/log4j.properties.template conf/log4j.properties
62+
63+
# Submit Spark's ApplicationMaster to YARN's ResourceManager, and instruct Spark to run the SparkPi example
64+
$ SPARK_JAR=./assembly/target/scala-{{site.SCALA_VERSION}}/spark-assembly-{{site.SPARK_VERSION}}-hadoop2.0.5-alpha.jar \
65+
./spark-class org.apache.spark.deploy.yarn.Client \
66+
--jar examples/target/scala-{{site.SCALA_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar \
67+
--class org.apache.spark.examples.SparkPi \
68+
--args yarn-standalone \
69+
--num-workers 3 \
70+
--master-memory 4g \
71+
--worker-memory 2g \
72+
--worker-cores 1
73+
74+
# Examine the output (replace $YARN_APP_ID in the following with the "application identifier" output by the previous command)
75+
# (Note: YARN_APP_LOGS_DIR is usually /tmp/logs or $HADOOP_HOME/logs/userlogs depending on the Hadoop version.)
76+
$ cat $YARN_APP_LOGS_DIR/$YARN_APP_ID/container*_000001/stdout
77+
Pi is roughly 3.13794
6578

6679
The above starts a YARN Client programs which periodically polls the Application Master for status updates and displays them in the console. The client will exit once your application has finished running.
6780

‎examples/pom.xml

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -127,20 +127,6 @@
127127
</dependency>
128128
</dependencies>
129129

130-
<profiles>
131-
<profile>
132-
<id>hadoop2-yarn</id>
133-
<dependencies>
134-
<dependency>
135-
<groupId>org.apache.spark</groupId>
136-
<artifactId>spark-yarn</artifactId>
137-
<version>${project.version}</version>
138-
<scope>provided</scope>
139-
</dependency>
140-
</dependencies>
141-
</profile>
142-
</profiles>
143-
144130
<build>
145131
<outputDirectory>target/scala-${scala.version}/classes</outputDirectory>
146132
<testOutputDirectory>target/scala-${scala.version}/test-classes</testOutputDirectory>

‎pom.xml

Lines changed: 93 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,99 @@
368368
</exclusion>
369369
</exclusions>
370370
</dependency>
371+
<dependency>
372+
<groupId>org.apache.hadoop</groupId>
373+
<artifactId>hadoop-yarn-api</artifactId>
374+
<version>${hadoop.version}</version>
375+
<exclusions>
376+
<exclusion>
377+
<groupId>asm</groupId>
378+
<artifactId>asm</artifactId>
379+
</exclusion>
380+
<exclusion>
381+
<groupId>org.jboss.netty</groupId>
382+
<artifactId>netty</artifactId>
383+
</exclusion>
384+
<exclusion>
385+
<groupId>org.codehaus.jackson</groupId>
386+
<artifactId>jackson-core-asl</artifactId>
387+
</exclusion>
388+
<exclusion>
389+
<groupId>org.codehaus.jackson</groupId>
390+
<artifactId>jackson-mapper-asl</artifactId>
391+
</exclusion>
392+
<exclusion>
393+
<groupId>org.codehaus.jackson</groupId>
394+
<artifactId>jackson-jaxrs</artifactId>
395+
</exclusion>
396+
<exclusion>
397+
<groupId>org.codehaus.jackson</groupId>
398+
<artifactId>jackson-xc</artifactId>
399+
</exclusion>
400+
</exclusions>
401+
</dependency>
402+
<dependency>
403+
<groupId>org.apache.hadoop</groupId>
404+
<artifactId>hadoop-yarn-common</artifactId>
405+
<version>${hadoop.version}</version>
406+
<exclusions>
407+
<exclusion>
408+
<groupId>asm</groupId>
409+
<artifactId>asm</artifactId>
410+
</exclusion>
411+
<exclusion>
412+
<groupId>org.jboss.netty</groupId>
413+
<artifactId>netty</artifactId>
414+
</exclusion>
415+
<exclusion>
416+
<groupId>org.codehaus.jackson</groupId>
417+
<artifactId>jackson-core-asl</artifactId>
418+
</exclusion>
419+
<exclusion>
420+
<groupId>org.codehaus.jackson</groupId>
421+
<artifactId>jackson-mapper-asl</artifactId>
422+
</exclusion>
423+
<exclusion>
424+
<groupId>org.codehaus.jackson</groupId>
425+
<artifactId>jackson-jaxrs</artifactId>
426+
</exclusion>
427+
<exclusion>
428+
<groupId>org.codehaus.jackson</groupId>
429+
<artifactId>jackson-xc</artifactId>
430+
</exclusion>
431+
</exclusions>
432+
</dependency>
433+
<dependency>
434+
<groupId>org.apache.hadoop</groupId>
435+
<artifactId>hadoop-yarn-client</artifactId>
436+
<version>${hadoop.version}</version>
437+
<exclusions>
438+
<exclusion>
439+
<groupId>asm</groupId>
440+
<artifactId>asm</artifactId>
441+
</exclusion>
442+
<exclusion>
443+
<groupId>org.jboss.netty</groupId>
444+
<artifactId>netty</artifactId>
445+
</exclusion>
446+
<exclusion>
447+
<groupId>org.codehaus.jackson</groupId>
448+
<artifactId>jackson-core-asl</artifactId>
449+
</exclusion>
450+
<exclusion>
451+
<groupId>org.codehaus.jackson</groupId>
452+
<artifactId>jackson-mapper-asl</artifactId>
453+
</exclusion>
454+
<exclusion>
455+
<groupId>org.codehaus.jackson</groupId>
456+
<artifactId>jackson-jaxrs</artifactId>
457+
</exclusion>
458+
<exclusion>
459+
<groupId>org.codehaus.jackson</groupId>
460+
<artifactId>jackson-xc</artifactId>
461+
</exclusion>
462+
</exclusions>
463+
</dependency>
371464
<!-- Specify Avro version because Kafka also has it as a dependency -->
372465
<dependency>
373466
<groupId>org.apache.avro</groupId>
@@ -620,131 +713,6 @@
620713

621714
<dependencyManagement>
622715
<dependencies>
623-
<!-- TODO: check versions, bringover from yarn branch ! -->
624-
<dependency>
625-
<groupId>org.apache.hadoop</groupId>
626-
<artifactId>hadoop-client</artifactId>
627-
<version>${hadoop.version}</version>
628-
<exclusions>
629-
<exclusion>
630-
<groupId>asm</groupId>
631-
<artifactId>asm</artifactId>
632-
</exclusion>
633-
<exclusion>
634-
<groupId>org.jboss.netty</groupId>
635-
<artifactId>netty</artifactId>
636-
</exclusion>
637-
<exclusion>
638-
<groupId>org.codehaus.jackson</groupId>
639-
<artifactId>jackson-core-asl</artifactId>
640-
</exclusion>
641-
<exclusion>
642-
<groupId>org.codehaus.jackson</groupId>
643-
<artifactId>jackson-mapper-asl</artifactId>
644-
</exclusion>
645-
<exclusion>
646-
<groupId>org.codehaus.jackson</groupId>
647-
<artifactId>jackson-jaxrs</artifactId>
648-
</exclusion>
649-
<exclusion>
650-
<groupId>org.codehaus.jackson</groupId>
651-
<artifactId>jackson-xc</artifactId>
652-
</exclusion>
653-
</exclusions>
654-
</dependency>
655-
<dependency>
656-
<groupId>org.apache.hadoop</groupId>
657-
<artifactId>hadoop-yarn-api</artifactId>
658-
<version>${hadoop.version}</version>
659-
<exclusions>
660-
<exclusion>
661-
<groupId>asm</groupId>
662-
<artifactId>asm</artifactId>
663-
</exclusion>
664-
<exclusion>
665-
<groupId>org.jboss.netty</groupId>
666-
<artifactId>netty</artifactId>
667-
</exclusion>
668-
<exclusion>
669-
<groupId>org.codehaus.jackson</groupId>
670-
<artifactId>jackson-core-asl</artifactId>
671-
</exclusion>
672-
<exclusion>
673-
<groupId>org.codehaus.jackson</groupId>
674-
<artifactId>jackson-mapper-asl</artifactId>
675-
</exclusion>
676-
<exclusion>
677-
<groupId>org.codehaus.jackson</groupId>
678-
<artifactId>jackson-jaxrs</artifactId>
679-
</exclusion>
680-
<exclusion>
681-
<groupId>org.codehaus.jackson</groupId>
682-
<artifactId>jackson-xc</artifactId>
683-
</exclusion>
684-
</exclusions>
685-
</dependency>
686-
<dependency>
687-
<groupId>org.apache.hadoop</groupId>
688-
<artifactId>hadoop-yarn-common</artifactId>
689-
<version>${hadoop.version}</version>
690-
<exclusions>
691-
<exclusion>
692-
<groupId>asm</groupId>
693-
<artifactId>asm</artifactId>
694-
</exclusion>
695-
<exclusion>
696-
<groupId>org.jboss.netty</groupId>
697-
<artifactId>netty</artifactId>
698-
</exclusion>
699-
<exclusion>
700-
<groupId>org.codehaus.jackson</groupId>
701-
<artifactId>jackson-core-asl</artifactId>
702-
</exclusion>
703-
<exclusion>
704-
<groupId>org.codehaus.jackson</groupId>
705-
<artifactId>jackson-mapper-asl</artifactId>
706-
</exclusion>
707-
<exclusion>
708-
<groupId>org.codehaus.jackson</groupId>
709-
<artifactId>jackson-jaxrs</artifactId>
710-
</exclusion>
711-
<exclusion>
712-
<groupId>org.codehaus.jackson</groupId>
713-
<artifactId>jackson-xc</artifactId>
714-
</exclusion>
715-
</exclusions>
716-
</dependency>
717-
<dependency>
718-
<groupId>org.apache.hadoop</groupId>
719-
<artifactId>hadoop-yarn-client</artifactId>
720-
<version>${hadoop.version}</version>
721-
<exclusions>
722-
<exclusion>
723-
<groupId>asm</groupId>
724-
<artifactId>asm</artifactId>
725-
</exclusion>
726-
<exclusion>
727-
<groupId>org.jboss.netty</groupId>
728-
<artifactId>netty</artifactId>
729-
</exclusion>
730-
<exclusion>
731-
<groupId>org.codehaus.jackson</groupId>
732-
<artifactId>jackson-core-asl</artifactId>
733-
</exclusion>
734-
<exclusion>
735-
<groupId>org.codehaus.jackson</groupId>
736-
<artifactId>jackson-mapper-asl</artifactId>
737-
</exclusion>
738-
<exclusion>
739-
<groupId>org.codehaus.jackson</groupId>
740-
<artifactId>jackson-jaxrs</artifactId>
741-
</exclusion>
742-
<exclusion>
743-
<groupId>org.codehaus.jackson</groupId>
744-
<artifactId>jackson-xc</artifactId>
745-
</exclusion>
746-
</exclusions>
747-
</dependency>
748716
</dependencies>
749717
</dependencyManagement>
750718
</profile>

‎project/SparkBuild.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@ object SparkBuild extends Build {
3838
lazy val core = Project("core", file("core"), settings = coreSettings)
3939

4040
lazy val repl = Project("repl", file("repl"), settings = replSettings)
41-
.dependsOn(core, bagel, mllib) dependsOn(maybeYarn: _*)
41+
.dependsOn(core, bagel, mllib)
4242

4343
lazy val examples = Project("examples", file("examples"), settings = examplesSettings)
44-
.dependsOn(core, mllib, bagel, streaming) dependsOn(maybeYarn: _*)
44+
.dependsOn(core, mllib, bagel, streaming)
4545

4646
lazy val tools = Project("tools", file("tools"), settings = toolsSettings) dependsOn(core) dependsOn(streaming)
4747

‎repl-bin/pom.xml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -105,16 +105,6 @@
105105
</build>
106106

107107
<profiles>
108-
<profile>
109-
<id>hadoop2-yarn</id>
110-
<dependencies>
111-
<dependency>
112-
<groupId>org.apache.spark</groupId>
113-
<artifactId>spark-yarn</artifactId>
114-
<version>${project.version}</version>
115-
</dependency>
116-
</dependencies>
117-
</profile>
118108
<profile>
119109
<id>deb</id>
120110
<build>

‎repl/pom.xml

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -131,16 +131,4 @@
131131
</plugin>
132132
</plugins>
133133
</build>
134-
<profiles>
135-
<profile>
136-
<id>hadoop2-yarn</id>
137-
<dependencies>
138-
<dependency>
139-
<groupId>org.apache.spark</groupId>
140-
<artifactId>spark-yarn</artifactId>
141-
<version>${project.version}</version>
142-
</dependency>
143-
</dependencies>
144-
</profile>
145-
</profiles>
146134
</project>

0 commit comments

Comments
 (0)