feat(etl): running etl in the same network (#22)

etl/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - 9200:9200
       - 9300:9300
     networks:
-      - devnet
+      - compose-services_devnet
   kibana-service:
     image: docker.elastic.co/kibana/kibana-oss:6.5.4
     container_name: kibana-service
@@ -25,15 +25,15 @@ services:
     ports:
       - 5601:5601
     networks:
-      - devnet
+      - compose-services_devnet
     depends_on:
       - esproxy-service
   tube-service:
     image: "quay.io/cdis/tube:master"
     container_name: tube-service
-    command: bash -c "python run_config.py && sleep 50 && python run_import.py"
+    command: bash -c "while true; do sleep 5; done"
     networks:
-      - devnet
+      - compose-services_devnet
     environment:
       - DICTIONARY_URL=https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json
       - ES_URL=esproxy-service
@@ -45,7 +45,6 @@ services:
       - ./Secrets/etlMapping.yaml:/usr/share/gen3/tube/etlMapping.yaml
     depends_on:
       - spark-service
-      - sheepdog-service
   spark-service:
     image: "quay.io/cdis/gen3-spark:master"
     container_name: spark-service
@@ -57,9 +56,10 @@ services:
       - 8032
       - 9000
     networks:
-      - devnet
+      - compose-services_devnet
     environment:
       - HADOOP_URL=hdfs://0.0.0.0:9000
       - HADOOP_HOST=0.0.0.0
 networks:
-  devnet:
+  compose-services_devnet:
+    external: true

templates/etlMapping.yaml

@@ -46,6 +46,16 @@ mappings:
       - name: _submitted_somatic_mutations_count
         path: samples.aliquots.read_groups.submitted_somatic_mutations
         fn: count
+    joining_props:
+      - index: file
+        join_on: subject_id
+        props:
+          - name: data_format
+            src: data_format
+            fn: set
+          - name: data_type
+            src: data_type
+            fn: set
   - name: file
     doc_type: file
     type: collector
@@ -65,13 +75,3 @@ mappings:
     target_nodes:
       - name: slide_image
         path: slides.samples.cases
-      - name: submitted_methylation
-        path: aliquots.samples.cases
-      - name: submitted_copy_number
-        path: read_groups.aliquots.samples.cases
-      - name: submitted_aligned_reads
-        path: read_groups.aliquots.samples.cases
-      - name: submitted_unaligned_reads
-        path: read_groups.aliquots.samples.cases
-      - name: submitted_somatic_mutation
-        path: read_groups.aliquots.samples.cases

templates/user.yaml

@@ -13,19 +13,17 @@ users:
     admin: True
     projects:
     - auth_id: jnkns
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: jenkins
-      privilege: [create, read, update, delete, upload, read-storage]
-    projects:
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: DEV
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: project1
-      privilege: [create, read, update, delete, upload, read-storage]
-    projects:
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: QA
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: project1
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
   #
   # The integration test suite assumes this user exists - you can
   # delete it if you dont want to run the test suite:
@@ -35,16 +33,14 @@ users:
     admin: True
     projects:
     - auth_id: jnkns
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: jenkins
-      privilege: [create, read, update, delete, upload, read-storage]
-    projects:
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: DEV
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: project1
-      privilege: [create, read, update, delete, upload, read-storage]
-    projects:
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: QA
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']
     - auth_id: project1
-      privilege: [create, read, update, delete, upload, read-storage]
+      privilege: ['create', 'read', 'update', 'delete', 'upload', 'read-storage']