Skip to content

Commit

Permalink
test: fix ci test
Browse files Browse the repository at this point in the history
  • Loading branch information
@sagar-salvi-apptware
sagar-salvi-apptware committed Nov 8, 2024
1 parent 4527497 commit d8ecb9c
Show file tree
Hide file tree
Showing 2 changed files with 346 additions and 34 deletions.
346 changes: 346 additions & 0 deletions metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,346 @@
[
{
"entityType": "container",
"entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "cassandra",
"env": "PROD",
"keyspace": "cass_test_1"
},
"name": "cass_test_1",
"env": "PROD"
}
},
"systemMetadata": {
"lastObserved": 1731087079359,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "schemaMetadata",
"aspect": {
"json": {
"schemaName": "information",
"platform": "urn:li:dataPlatform:cassandra",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"hash": "",
"platformSchema": {
"com.linkedin.schema.OtherSchema": {
"rawSchema": "[{\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"details\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1, \"type\": \"text\"}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"last_updated\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1, \"type\": \"timestamp\"}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"person_id\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0, \"type\": \"int\"}]"
}
},
"fields": [
{
"fieldPath": "[version=2.0].[type=text].details",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.StringType": {}
}
},
"nativeDataType": "text",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "[version=2.0].[type=timestamp].last_updated",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.TimeType": {}
}
},
"nativeDataType": "timestamp",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "[version=2.0].[type=int].person_id",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.NumberType": {}
}
},
"nativeDataType": "int",
"recursive": false,
"isPartOfKey": false
}
]
}
},
"systemMetadata": {
"lastObserved": 1731087079371,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:cassandra"
}
},
"systemMetadata": {
"lastObserved": 1731087079360,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1731087079360,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Keyspace"
]
}
},
"systemMetadata": {
"lastObserved": 1731087079360,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1731087079361,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1731087079373,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Table"
]
}
},
"systemMetadata": {
"lastObserved": 1731087079373,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05"
}
},
"systemMetadata": {
"lastObserved": 1731087079374,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05",
"urn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05"
}
]
}
},
"systemMetadata": {
"lastObserved": 1731087079374,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {
"bloom_filter_fp_chance": "0.01",
"caching": "{'keys': 'ALL', 'rows_per_partition': 'NONE'}",
"cdc": "None",
"compaction": "{'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}",
"compression": "{'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}",
"max_index_interval": "2048"
},
"name": "information",
"qualifiedName": "cass_test_1.information",
"description": "",
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1731087079374,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1731087079383,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "cassandra",
"env": "PROD",
"keyspace": "cass_test_2"
},
"name": "cass_test_2",
"env": "PROD"
}
},
"systemMetadata": {
"lastObserved": 1731087079383,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Keyspace"
]
}
},
"systemMetadata": {
"lastObserved": 1731087079384,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1731087079385,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:cassandra"
}
},
"systemMetadata": {
"lastObserved": 1731087079384,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
}
]
34 changes: 0 additions & 34 deletions metadata-ingestion/tests/integration/cassandra/docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,3 @@
# version: "1"
# services:
# test-cassandra:
# image: cassandra:latest
# container_name: test-cassandra
# # environment:
# # No env vars needed. AllowAll authenticator is used by default (https://docs.datastax.com/en/archived/cassandra/3.0/cassandra/configuration/configCassandra_yaml.html?scroll=configCassandra_yaml__endpoint_snitch)
# ports:
# - 9042:9042
# volumes:
# - ./setup/cassandra.yaml:/etc/cassandra/cassandra.yaml
# - ./setup/init_keyspaces.cql:/docker-entrypoint-initdb.d/init_keyspaces.cql
# networks:
# - testnet
# # referencing netflix's approach to loading cassandra with initial data https://github.com/Netflix/osstracker/blob/master/docker-compose.yml
# test-cassandra-load-keyspace:
# container_name: test-cassandra-load-keyspace
# image: cassandra:latest
# depends_on:
# - test-cassandra
# volumes:
# - ./setup/init_keyspaces.cql:/init_keyspaces.cql
# command: /bin/bash -c "sleep 50 && echo loading cassandra keyspace && cqlsh test-cassandra -f init_keyspaces.cql"
# deploy:
# restart_policy:
# condition: on-failure
# delay: 5s
# max_attempts: 3
# window: 100s
# networks:
# - testnet
# networks:
# testnet:

version: "1"
services:
test-cassandra:
Expand Down

0 comments on commit d8ecb9c

Please sign in to comment.