Skip to content

Commit 5199d12

Browse files
carolyndubycdubystas-panasiuk
authored
[CYB-207] Support CSA 1.12.0 CDH 7.1.9 (#74)
* Datahub 1.10.0 pom changes * [CYB-172] kafka topic created from tableapi kafka connector need ssb properties * Datahub 1.10.0 pom changes * [CYB-190] user can specify names of CDP cloud artifacts in example create_datahub_config.sh setup script * [CYB-190] user can specify names of CDP cloud artifacts in example create_datahub_config.sh setup script * [CYB-191] schemas produced for topics can't be consumed by SMM because of watermark properties * [CYB-206] dev log removed * [CYB-207] cybersec toolkit supports CSA 1.12.0 with CDP 7.1.9 * [CYB-207] fix indexing job serialization error * [CYB-207] add missing jupiter dependency. use jupiter for all tests * [CYB-207] remove deprecated forkMode for rest enrichments module * [CYB-207] add junit dependency * [CYB-207] use https for mock server for all tests. * [CYB-207] temporary change to run just problem module * [CYB-207] update mockserver dependency * Revert "[CYB-207] temporary change to run just problem module" This reverts commit ba0a0a9. * Revert "[CYB-207] use https for mock server for all tests." This reverts commit 1fe5223. * [CYB-207] Update mockserver dependency to remove conflicts * typo fix --------- Co-authored-by: cduby <[email protected]> Co-authored-by: Stas Panasiuk <[email protected]> Co-authored-by: stas-panasiuk <[email protected]>
1 parent 0512dcd commit 5199d12

File tree

37 files changed

+459
-179
lines changed

37 files changed

+459
-179
lines changed

‎flink-cyber/caracal-generator/pom.xml

+7
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@
3333
<artifactId>flink-connector-kafka</artifactId>
3434
</dependency>
3535

36+
<dependency>
37+
<groupId>org.apache.flink</groupId>
38+
<artifactId>flink-clients</artifactId>
39+
<scope>provided</scope>
40+
<version>${flink.version}</version>
41+
</dependency>
42+
3643
<dependency>
3744
<groupId>com.hortonworks.smm</groupId>
3845
<artifactId>monitoring-interceptors</artifactId>

‎flink-cyber/caracal-parser/pom.xml

+11-2
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,12 @@
126126

127127
<dependency>
128128
<groupId>org.apache.flink</groupId>
129-
<artifactId>flink-cloudera-registry</artifactId>
130-
<version>${flink.version}</version>
129+
<artifactId>flink-connector-cloudera-registry</artifactId>
130+
</dependency>
131+
132+
<dependency>
133+
<groupId>com.hortonworks.registries</groupId>
134+
<artifactId>schema-registry-serdes</artifactId>
131135
</dependency>
132136

133137
<dependency>
@@ -164,6 +168,11 @@
164168
<artifactId>flink-orc</artifactId>
165169
</dependency>
166170

171+
<dependency>
172+
<groupId>org.apache.orc</groupId>
173+
<artifactId>orc-core</artifactId>
174+
</dependency>
175+
167176
<dependency>
168177
<groupId>org.apache.parquet</groupId>
169178
<artifactId>parquet-avro</artifactId>

‎flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public long getTs() {
5454
return message.getTs();
5555
}
5656

57-
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName())
57+
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), null)
5858
.fields()
5959
.name("message").type(Message.SCHEMA$).noDefault()
6060
.name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault()

‎flink-cyber/flink-alert-scoring/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@
9595
<groupId>org.apache.flink</groupId>
9696
<artifactId>flink-statebackend-rocksdb</artifactId>
9797
</dependency>
98+
<dependency>
99+
<groupId>com.google.guava</groupId>
100+
<artifactId>guava</artifactId>
101+
<scope>test</scope>
102+
</dependency>
98103

99104
</dependencies>
100105
<build>

‎flink-cyber/flink-commands/json-commands/pom.xml

+5-6
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,14 @@
7070
<version>${log4j.version}</version>
7171
<scope>provided</scope>
7272
</dependency>
73-
<dependency>
74-
<groupId>com.google.guava</groupId>
75-
<artifactId>guava</artifactId>
76-
<version>${guava.version}</version>
77-
</dependency>
7873
<dependency>
7974
<groupId>com.fasterxml.jackson.core</groupId>
8075
<artifactId>jackson-databind</artifactId>
81-
<version>${jackson.version}</version>
76+
</dependency>
77+
<dependency>
78+
<groupId>com.google.guava</groupId>
79+
<artifactId>guava</artifactId>
80+
<scope>test</scope>
8281
</dependency>
8382
</dependencies>
8483

‎flink-cyber/flink-commands/scoring-commands/pom.xml

+18
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@
6060
<version>${jackson.datatype.version}</version>
6161
<scope>compile</scope>
6262
</dependency>
63+
<dependency>
64+
<groupId>com.hortonworks.registries</groupId>
65+
<artifactId>schema-registry-serdes</artifactId>
66+
</dependency>
67+
<dependency>
68+
<groupId>com.google.guava</groupId>
69+
<artifactId>guava</artifactId>
70+
</dependency>
6371
<!-- Logging and Metrics -->
6472
<dependency>
6573
<groupId>org.slf4j</groupId>
@@ -90,6 +98,10 @@
9098
<artifactId>avro</artifactId>
9199
<scope>compile</scope>
92100
</dependency>
101+
<dependency>
102+
<groupId>org.apache.flink</groupId>
103+
<artifactId>flink-java</artifactId>
104+
</dependency>
93105
</dependencies>
94106
<build>
95107
<plugins>
@@ -128,6 +140,12 @@
128140
</excludes>
129141
</filter>
130142
</filters>
143+
<relocations>
144+
<relocation>
145+
<pattern>com.google.</pattern>
146+
<shadedPattern>com.cloudera.cyber.com.google.</shadedPattern>
147+
</relocation>
148+
</relocations>
131149
<transformers>
132150
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
133151
<mainClass>com.cloudera.cyber.scoring.ScoringJobKafka</mainClass>

‎flink-cyber/flink-common/pom.xml

+14-1
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,22 @@
4343
<groupId>org.apache.flink</groupId>
4444
<artifactId>flink-connector-kafka</artifactId>
4545
</dependency>
46+
<dependency>
47+
<groupId>org.apache.kafka</groupId>
48+
<artifactId>kafka-clients</artifactId>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.apache.flink</groupId>
52+
<artifactId>flink-connector-cloudera-registry</artifactId>
53+
</dependency>
54+
<dependency>
55+
<groupId>com.hortonworks.registries</groupId>
56+
<artifactId>schema-registry-serdes</artifactId>
57+
</dependency>
4658
<dependency>
4759
<groupId>org.apache.flink</groupId>
48-
<artifactId>flink-cloudera-registry</artifactId>
60+
<artifactId>flink-connector-base</artifactId>
61+
<version>${flink.version}</version>
4962
</dependency>
5063
<dependency>
5164
<groupId>org.apache.avro</groupId>

‎flink-cyber/flink-cyber-api/pom.xml

+19-1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,19 @@
3232
</plugins>
3333
</build>
3434

35+
<dependencyManagement>
36+
<dependencies>
37+
<dependency>
38+
<!-- Import dependency management from Spring Boot -->
39+
<groupId>org.springframework.boot</groupId>
40+
<artifactId>spring-boot-dependencies</artifactId>
41+
<version>${spring-boot.version}</version>
42+
<type>pom</type>
43+
<scope>import</scope>
44+
</dependency>
45+
</dependencies>
46+
</dependencyManagement>
47+
3548
<dependencies>
3649
<dependency>
3750
<groupId>org.projectlombok</groupId>
@@ -55,10 +68,15 @@
5568
<artifactId>flink-table-common</artifactId>
5669
</dependency>
5770

71+
72+
<dependency>
73+
<groupId>org.springframework</groupId>
74+
<artifactId>spring-core</artifactId>
75+
</dependency>
76+
5877
<dependency>
5978
<groupId>org.apache.avro</groupId>
6079
<artifactId>avro</artifactId>
61-
<version>${avro.version}</version>
6280
</dependency>
6381

6482
<dependency>

‎flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import lombok.Data;
1919
import lombok.NoArgsConstructor;
2020
import lombok.NonNull;
21+
import org.apache.avro.LogicalTypes;
2122
import org.apache.avro.Schema;
2223
import org.apache.avro.SchemaBuilder;
2324
import org.apache.avro.specific.SpecificRecord;
@@ -56,7 +57,7 @@ public class Message extends SpecificRecordBase implements SpecificRecord, Ident
5657
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName())
5758
.fields()
5859
.requiredString("id")
59-
.requiredLong("ts")
60+
.name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))).noDefault()
6061
.name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault()
6162
.requiredString("message")
6263
.name("threats").type().optional().type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$)))

‎flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java

+11-3
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,16 @@
55
public class AvroSchemas {
66

77
public static SchemaBuilder.RecordBuilder<org.apache.avro.Schema> createRecordBuilder(String namespace, String recordName) {
8-
return SchemaBuilder.record(recordName).namespace(namespace)
9-
.prop("ssb.rowtimeAttribute", "ts")
10-
.prop("ssb.watermarkExpression", "`ts` - INTERVAL '30' SECOND");
8+
return createRecordBuilder(namespace, recordName, "ts");
9+
}
10+
11+
public static SchemaBuilder.RecordBuilder<org.apache.avro.Schema> createRecordBuilder(String namespace, String recordName, String tsFieldName) {
12+
SchemaBuilder.RecordBuilder<org.apache.avro.Schema> recordBuilder = SchemaBuilder.record(recordName).namespace(namespace);
13+
if (tsFieldName != null) {
14+
recordBuilder
15+
.prop("ssb.rowtimeAttribute", tsFieldName)
16+
.prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName));
17+
}
18+
return recordBuilder;
1119
}
1220
}

‎flink-cyber/flink-dedupe/pom.xml

+6-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,12 @@
4545

4646
<dependency>
4747
<groupId>org.apache.flink</groupId>
48-
<artifactId>flink-cloudera-registry</artifactId>
48+
<artifactId>flink-connector-cloudera-registry</artifactId>
49+
</dependency>
50+
51+
<dependency>
52+
<groupId>com.hortonworks.registries</groupId>
53+
<artifactId>schema-registry-serdes</artifactId>
4954
</dependency>
5055

5156
<dependency>

‎flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@
9090
<classifier>tests</classifier>
9191
</dependency>
9292

93+
<dependency>
94+
<groupId>com.google.guava</groupId>
95+
<artifactId>guava</artifactId>
96+
</dependency>
97+
9398
<dependency>
9499
<groupId>com.hortonworks.smm</groupId>
95100
<artifactId>monitoring-interceptors</artifactId>

‎flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,11 @@
156156
<version>${commons-validator.version}</version>
157157
</dependency>
158158

159+
<dependency>
160+
<groupId>com.google.guava</groupId>
161+
<artifactId>guava</artifactId>
162+
</dependency>
163+
159164
<dependency>
160165
<groupId>org.assertj</groupId>
161166
<artifactId>assertj-core</artifactId>

‎flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml

+6-12
Original file line numberDiff line numberDiff line change
@@ -72,25 +72,19 @@
7272
<dependency>
7373
<groupId>org.apache.flink</groupId>
7474
<artifactId>flink-connector-hbase-${flink.hbase.version}</artifactId>
75-
<version>${flink.version}</version>
76-
<exclusions>
77-
<exclusion>
78-
<groupId>org.glassfish</groupId>
79-
<artifactId>javax.el</artifactId>
80-
</exclusion>
81-
<exclusion>
82-
<groupId>org.apache.hbase</groupId>
83-
<artifactId>hbase-client</artifactId>
84-
</exclusion>
85-
</exclusions>
8675
</dependency>
8776

8877
<dependency>
8978
<groupId>org.apache.flink</groupId>
90-
<artifactId>flink-cloudera-registry</artifactId>
79+
<artifactId>flink-connector-cloudera-registry</artifactId>
9180
<scope>compile</scope>
9281
</dependency>
9382

83+
<dependency>
84+
<groupId>com.hortonworks.registries</groupId>
85+
<artifactId>schema-registry-serdes</artifactId>
86+
</dependency>
87+
9488
<dependency>
9589
<groupId>com.cloudera.cyber</groupId>
9690
<artifactId>flink-cyber-api</artifactId>

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml

+8
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@
3939
<artifactId>lombok</artifactId>
4040
<scope>provided</scope>
4141
</dependency>
42+
<dependency>
43+
<groupId>com.fasterxml.jackson.core</groupId>
44+
<artifactId>jackson-core</artifactId>
45+
</dependency>
46+
<dependency>
47+
<groupId>com.fasterxml.jackson.core</groupId>
48+
<artifactId>jackson-databind</artifactId>
49+
</dependency>
4250

4351
</dependencies>
4452

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml

+6-8
Original file line numberDiff line numberDiff line change
@@ -52,21 +52,19 @@
5252
<dependency>
5353
<groupId>org.apache.flink</groupId>
5454
<artifactId>flink-connector-hbase-${flink.hbase.version}</artifactId>
55-
<version>${flink.version}</version>
56-
<exclusions>
57-
<exclusion>
58-
<groupId>org.glassfish</groupId>
59-
<artifactId>javax.el</artifactId>
60-
</exclusion>
61-
</exclusions>
6255
</dependency>
6356

6457
<dependency>
6558
<groupId>org.apache.flink</groupId>
66-
<artifactId>flink-cloudera-registry</artifactId>
59+
<artifactId>flink-connector-cloudera-registry</artifactId>
6760
<scope>provided</scope>
6861
</dependency>
6962

63+
<dependency>
64+
<groupId>com.hortonworks.registries</groupId>
65+
<artifactId>schema-registry-serdes</artifactId>
66+
</dependency>
67+
7068
<dependency>
7169
<groupId>com.cloudera.cyber</groupId>
7270
<artifactId>flink-cyber-api</artifactId>

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml

+6-1
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,15 @@
5050

5151
<dependency>
5252
<groupId>org.apache.flink</groupId>
53-
<artifactId>flink-cloudera-registry</artifactId>
53+
<artifactId>flink-connector-cloudera-registry</artifactId>
5454
<scope>provided</scope>
5555
</dependency>
5656

57+
<dependency>
58+
<groupId>com.hortonworks.registries</groupId>
59+
<artifactId>schema-registry-serdes</artifactId>
60+
</dependency>
61+
5762
<dependency>
5863
<groupId>com.cloudera.cyber</groupId>
5964
<artifactId>flink-cyber-api</artifactId>

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml

+14-5
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,15 @@
5151

5252
<dependency>
5353
<groupId>org.apache.flink</groupId>
54-
<artifactId>flink-cloudera-registry</artifactId>
54+
<artifactId>flink-connector-cloudera-registry</artifactId>
5555
<scope>provided</scope>
5656
</dependency>
5757

58+
<dependency>
59+
<groupId>com.hortonworks.registries</groupId>
60+
<artifactId>schema-registry-serdes</artifactId>
61+
</dependency>
62+
5863
<dependency>
5964
<groupId>com.cloudera.cyber</groupId>
6065
<artifactId>flink-cyber-api</artifactId>
@@ -118,10 +123,14 @@
118123
<version>${global.httpclient.version}</version>
119124
<scope>compile</scope>
120125
</dependency>
126+
<dependency>
127+
<groupId>junit</groupId>
128+
<artifactId>junit</artifactId>
129+
</dependency>
121130
<dependency>
122131
<groupId>org.mock-server</groupId>
123-
<artifactId>mockserver-netty</artifactId>
124-
<version>5.11.1</version>
132+
<artifactId>mockserver-netty-no-dependencies</artifactId>
133+
<version>RELEASE</version>
125134
<scope>test</scope>
126135
</dependency>
127136
<dependency>
@@ -196,8 +205,8 @@
196205
<artifactId>maven-surefire-plugin</artifactId>
197206
<version>${maven-surefire-plugin.version}</version>
198207
<configuration>
199-
<!-- use separate jvm to run each test. mockserver secure and insecure get confused if tests run in the same jvm -->
200-
<forkMode>always</forkMode>
208+
<forkCount>1</forkCount>
209+
<reuseForks>false</reuseForks>
201210
</configuration>
202211
</plugin>
203212
</plugins>

0 commit comments

Comments
 (0)