Skip to content

Commit fcd81cf

Browse files
qqndrewqqndrew
committed
Hadoop-common is not provided by default in non-spark envs
1 parent 337cbdd commit fcd81cf

2 files changed

Lines changed: 101 additions & 2 deletions

File tree

‎Core/pom.xml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,36 @@
9898
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
9999
<version>${beam.version}</version>
100100
</dependency>
101+
<!-- Note: this dependency is needed for every non-spark compile (where it is provided in runtime env)
102+
to prevent class not found errors -->
103+
<dependency>
104+
<groupId>org.apache.hadoop</groupId>
105+
<artifactId>hadoop-common</artifactId>
106+
<version>2.10.1</version>
107+
<scope>compile</scope>
108+
<exclusions>
109+
<exclusion>
110+
<groupId>org.checkerframework</groupId>
111+
<artifactId>jdk8</artifactId>
112+
</exclusion>
113+
<exclusion>
114+
<groupId>com.google.guava</groupId>
115+
<artifactId>guava-jdk5</artifactId>
116+
</exclusion>
117+
<exclusion>
118+
<groupId>jdk.tools</groupId>
119+
<artifactId>jdk.tools</artifactId>
120+
</exclusion>
121+
<exclusion>
122+
<groupId>com.google.protobuf</groupId>
123+
<artifactId>protobuf-lite</artifactId>
124+
</exclusion>
125+
<exclusion>
126+
<groupId>org.hamcrest</groupId>
127+
<artifactId>hamcrest-all</artifactId>
128+
</exclusion>
129+
</exclusions>
130+
</dependency>
101131
</dependencies>
102132
</profile>
103133
<!-- Spark -->
@@ -137,6 +167,36 @@
137167
<artifactId>spark-streaming_2.11</artifactId>
138168
<version>${spark.version}</version>
139169
</dependency>
170+
<!-- Note: this dependency is needed for every non-spark compile (where it is provided in runtime env)
171+
to prevent class not found errors -->
172+
<dependency>
173+
<groupId>org.apache.hadoop</groupId>
174+
<artifactId>hadoop-common</artifactId>
175+
<version>2.10.1</version>
176+
<scope>compile</scope>
177+
<exclusions>
178+
<exclusion>
179+
<groupId>org.checkerframework</groupId>
180+
<artifactId>jdk8</artifactId>
181+
</exclusion>
182+
<exclusion>
183+
<groupId>com.google.guava</groupId>
184+
<artifactId>guava-jdk5</artifactId>
185+
</exclusion>
186+
<exclusion>
187+
<groupId>jdk.tools</groupId>
188+
<artifactId>jdk.tools</artifactId>
189+
</exclusion>
190+
<exclusion>
191+
<groupId>com.google.protobuf</groupId>
192+
<artifactId>protobuf-lite</artifactId>
193+
</exclusion>
194+
<exclusion>
195+
<groupId>org.hamcrest</groupId>
196+
<artifactId>hamcrest-all</artifactId>
197+
</exclusion>
198+
</exclusions>
199+
</dependency>
140200
</dependencies>
141201
</profile>
142202
<!-- Apache Flink -->
@@ -151,6 +211,36 @@
151211
<artifactId>beam-runners-flink-1.13</artifactId>
152212
<version>${beam.version}</version>
153213
</dependency>
214+
<!-- Note: this dependency is needed for every non-spark compile (where it is provided in runtime env)
215+
to prevent class not found errors -->
216+
<dependency>
217+
<groupId>org.apache.hadoop</groupId>
218+
<artifactId>hadoop-common</artifactId>
219+
<version>2.10.1</version>
220+
<scope>compile</scope>
221+
<exclusions>
222+
<exclusion>
223+
<groupId>org.checkerframework</groupId>
224+
<artifactId>jdk8</artifactId>
225+
</exclusion>
226+
<exclusion>
227+
<groupId>com.google.guava</groupId>
228+
<artifactId>guava-jdk5</artifactId>
229+
</exclusion>
230+
<exclusion>
231+
<groupId>jdk.tools</groupId>
232+
<artifactId>jdk.tools</artifactId>
233+
</exclusion>
234+
<exclusion>
235+
<groupId>com.google.protobuf</groupId>
236+
<artifactId>protobuf-lite</artifactId>
237+
</exclusion>
238+
<exclusion>
239+
<groupId>org.hamcrest</groupId>
240+
<artifactId>hamcrest-all</artifactId>
241+
</exclusion>
242+
</exclusions>
243+
</dependency>
154244
</dependencies>
155245
</profile>
156246
</profiles>

‎IO/pom.xml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,21 @@
8787
<version>${beam.version}</version>
8888
</dependency>
8989

90-
<!-- HCatalog IO -->
90+
<!-- HDFS and HCatalog IO -->
91+
<dependency>
92+
<groupId>org.apache.beam</groupId>
93+
<artifactId>beam-sdks-java-io-hadoop-file-system</artifactId>
94+
<version>${beam.version}</version>
95+
</dependency>
96+
<dependency>
97+
<groupId>org.apache.beam</groupId>
98+
<artifactId>beam-sdks-java-extensions-sql-hcatalog</artifactId>
99+
<version>${beam.version}</version>
100+
</dependency>
91101
<dependency>
92102
<groupId>org.apache.beam</groupId>
93103
<artifactId>beam-sdks-java-io-hcatalog</artifactId>
94104
<version>${beam.version}</version>
95105
</dependency>
96106
</dependencies>
97-
98107
</project>

0 commit comments

Comments
 (0)