Sunbird-Obsrv · SadanandGowda · Dec 23, 2022 · Dec 23, 2022 · Dec 23, 2022 · Dec 23, 2022
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml
@@ -10,8 +10,8 @@ channel_data_exhaust_bucket: dev-data-store
 secrets_path: '{{inventory_dir}}/secrets.yml'
 artifacts_container: "{{dp_vault_artifacts_container}}"
 
-report_azure_account_name: "{{sunbird_private_storage_account_name}}"
-report_azure_storage_secret: "{{sunbird_private_storage_account_key}}"
+report_azure_account_name: "{{cloud_private_storage_accountname}}"
+report_azure_storage_secret: "{{cloud_private_storage_secret}}"
 
 redis_host: "{{ groups['redis'][0] }}"
 metadata_redis_host: "{{ groups['redis'][0] }}"
@@ -50,8 +50,8 @@ secor:
   artifact_dir: /mount/secor
   artifact_ver: "0.29"
   azure:
-    account_name: "{{sunbird_private_storage_account_name}}"
-    account_key: "{{sunbird_private_storage_account_key}}"
+    account_name: "{{cloud_private_storage_accountname}}"
+    account_key: "{{cloud_private_storage_secret}}"
     container_name: "{{channel_data_exhaust_bucket}}"
   paths: ['/mount/secor', '/mount/secor/reports', '/mount/secor/logs', '/home/analytics/sbin', '/mount/data/analytics']
   channel: "{{secor_alerts_slack_channel}}"

diff --git a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-jobmanager.yaml
@@ -21,6 +21,7 @@ spec:
         workingDir: /opt/flink
         command: ["/opt/flink/bin/standalone-job.sh"]
 {{- $job-config-key := .Release.Name }}
+
         args: ["start-foreground",
                "--job-classname={{ index .Values $job-config-key.job_classname }}", 
                "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
@@ -29,7 +30,20 @@ spec:
                "-Dblob.server.port=6124", 
                "-Dqueryable-state.server.ports=6125",
                "-Djobmanager.heap.size={{ index .Values $job-config-key.job_manager_heap_size }}",
-               "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+               "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }} 
+               "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+               "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+               "-Ds3.endpoint={{ .Values.cloud_private_endpoint }}",
+               "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+               "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+               "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+               "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}"
+{{- end }}
                "-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
         ports:
         - containerPort: 6123

diff --git a/...ble/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml b/...ble/kubernetes/helm_charts/datapipeline/flink-jobs/templates/job-cluster-taskmanager.yaml
@@ -29,7 +29,20 @@ spec:
           "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
           "-Dtaskmanager.rpc.port=6122",
           "-Dtaskmanager.heap.size={{ index .Values $job-config-key.task_manager_heap_size }}",
-          "-Dfs.azure.account.key.{{ .Values.azure_storage_account }}.blob.core.windows.net: {{ .Values.azure_storage_secret }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
+          "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net: {{ .Values.cloud_storage_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "aws" }} 
+          "-Ds3.access-key={{ .Values.cloud_storage_key }}",
+          "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
+          "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
+          "-Ds3.path.style.access={{ .Values.cloud_storage_path_style_access }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "gcloud" }}
+          "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
+          "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+          "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
+{{- end }}
           "-Dconfig.file=/opt/flink/conf/{{ .Release.Name }}.conf"]
         ports:
         - containerPort: 6122

diff --git a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2
@@ -6,8 +6,12 @@ dockerhub: {{ dockerhub }}
 repository: {{ datapipeline_repository|default('data-pipeline') }}
 image_tag: {{ image_tag }}
 
-azure_storage_account={{ sunbird_private_storage_account_name }}
-azure_storage_secret={{ sunbird_private_storage_account_key }}
+checkpoint_store_type: {{ cloud_service_provider }} 
+cloud_storage_key: {{ cloud_private_storage_accountname }}
+cloud_storage_secret: {{ cloud_private_storage_secret }}
+cloud_storage_endpoint: {{ cloud_private_storage_endpoint }}
+cloud_storage_path_style_access: {{ cloud_storage_pathstyle_access }}
+cloud_storage_project_id: {{ cloud_private_storage_project }}
 
 telemetry-extractor:
   job_name=telemetry-extractor

diff --git a/ansible/lpa_data-products_deploy.yml b/ansible/lpa_data-products_deploy.yml
@@ -5,7 +5,7 @@
   become: yes
   become_user: "{{ analytics_user }}"
   environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+    AZURE_STORAGE_ACCOUNT: "{{cloud_private_storage_accountname}}"
+    AZURE_STORAGE_KEY: "{{cloud_private_storage_secret}}"
   roles:
    - data-products-deploy
diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
@@ -8,8 +8,8 @@
     line: 'export {{item.var}}={{item.value}}'
     regexp: "export {{ item.var }}.*"
   with_items:
-    - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
-    - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
+    - {var: 'azure_storage_key', value: '{{ cloud_private_storage_accountname }}'}
+    - {var: 'azure_storage_secret', value: '{{ cloud_private_storage_secret }}'}
 
 - name: Adding ENV Vars to spark servers environment.
   become: yes
@@ -18,20 +18,21 @@
     line: '{{item.var}}={{item.value}}'
     regexp: "{{ item.var }}.*"
   with_items:
-    - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
-    - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
-    - {var: 'AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_private_storage_account_name }}'}
-    - {var: 'AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_private_storage_account_key }}'}
-    - {var: 'PUBLIC_AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_public_storage_account_name }}'}
-    - {var: 'PUBLIC_AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_public_storage_account_key }}'}
-    - {var: 'PRIVATE_REPORT_CONTAINER', value: '{{ sunbird_private_azure_report_container_name }}'}
-    - {var: 'PUBLIC_REPORT_CONTAINER', value: '{{ sunbird_public_azure_report_container_name }}'}
+    - {var: 'azure_storage_key', value: '{{ cloud_private_storage_accountname }}'}
+    - {var: 'azure_storage_secret', value: '{{ cloud_private_storage_secret }}'}
+    - {var: 'AZURE_STORAGE_ACCOUNT', value: '{{ cloud_private_storage_accountname }}'}
+    - {var: 'AZURE_STORAGE_ACCESS_KEY', value: '{{ cloud_private_storage_secret }}'}
+    - {var: 'PUBLIC_AZURE_STORAGE_ACCOUNT', value: '{{ cloud_public_storage_accountname }}'}
+    - {var: 'PUBLIC_AZURE_STORAGE_ACCESS_KEY', value: '{{ cloud_public_storage_secret }}'}
+    - {var: 'PRIVATE_REPORT_CONTAINER', value: '{{ cloud_storage_privatereports_bucketname }}'}
+    - {var: 'PUBLIC_REPORT_CONTAINER', value: '{{ cloud_storage_publicreports_bucketname }}'}
     - {var: 'REPORT_BACKUP_CONTAINER', value: 'portal-reports-backup'}
     - {var: 'GOOGLE_CREDENTIALS_PATH', value: '/home/analytics/credentials'}
     - {var: 'STORAGE_PROVIDER', value: 'AZURE'}
     - {var: 'ENV', value: '{{env}}'}
     - {var: 'KAFKA_BROKER_HOST', value: "{{groups['processing-cluster-kafka'][0]}}:9092"}
 
+
 - name: Install required python packages
   become: yes
   action: apt pkg={{ item }} state=present update_cache=yes

diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml
@@ -31,7 +31,7 @@ druid_request_logging_type: "file"
 
 #Druid Extensions
 
-druid_storage_type: "azure"
+druid_storage_type: "{{ cloud_storage_telemetry_type }}"
 
 druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"'
 
@@ -40,7 +40,7 @@ druid_community_extensions:
 
 # End of druid_extensions
 
-druid_indexing_logs_type: azure
+druid_indexing_logs_type:  "{{ cloud_storage_telemetry_type }}"
 druid_indexing_log_dir: /var/druid/indexing-logs
 druid_indexing_storage_type : metadata
 druid_indexing_task_basedir : "/var/task"
@@ -126,24 +126,31 @@ default_druid_configs:
    druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
    #Druid Azure Details 
    druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
-   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
-   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
-   azure_container: "{{ druid_azure_container_name }}"
+   azure_account_name: "{{ cloud_private_storage_accountname }}"
+   azure_storage_secret: "{{ cloud_private_storage_secret }}"
+   azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    #Logging the indexing logs to azure
-   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    druid_log_azure_folder: "druidlogs"
    #Druid S3 Details
-   druid_storage_type: "{{ druid_storage_type }}"
-   s3_access_key: "{{ s3_storage_key }}"
-   s3_secret_key: "{{ s3_storage_secret }}"
-   s3_bucket: "{{ s3_storage_container }}"
-   s3_endpoint: "{{ s3_storage_endpoint }}"
+   druid_storage_type: "{{ cloud_storage_telemetry_type }}"
+   s3_access_key: "{{ cloud_private_storage_accountname }}"
+   s3_secret_key: "{{ cloud_private_storage_secret }}"
+   s3_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+   s3_endpoint: "{{ cloud_private_storage_endpoint }}"
    s3_segment_dir: "druid/raw/segments"
-   s3_path_like_access: "{{ s3_path_style_access }}"
-   s3_v4_sign_region: "{{ s3_default_bucket_location }}"
+   s3_path_like_access: "{{ cloud_storage_pathstyle_access }}"
+   s3_v4_sign_region: "{{ cloud_private_storage_region }}"
    #Logging the indexing logs to s3
-   s3_logging_bucket: "{{ s3_storage_container }}"
+   s3_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
    s3_indexer_logs_dir: "druid/raw/stage/indexing_logs"
+   #Druid GCS Details
+   gcloud_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+   gcloud_prefix: ""
+   gcloud_max_list_length: 1024
+   #Logging the indexing logs to GCS
+   gcloud_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+   gcloud_indexer_logs_dir: "druid/raw/stage/indexing_logs"
    #Druid coordinator node configuration
    druid_coordinator_heap_size: 128m
    druid_coordinator_period: PT30S
@@ -200,24 +207,31 @@ default_druid_configs:
    druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
    #Druid Azure Details
    druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
-   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
-   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
-   azure_container: "{{ druid_azure_container_name }}"
+   azure_account_name: "{{ cloud_private_storage_accountname }}"
+   azure_storage_secret: "{{ cloud_private_storage_secret }}"
+   azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    #Logging the indexing logs to azure
-   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_container: "{{ cloud_storage_telemetry_bucketname }}"
    druid_log_azure_folder: "druidlogs"
    #Druid S3 Details
-   druid_storage_type: "{{ druid_storage_type }}"
+   druid_storage_type: "{{ cloud_storage_telemetry_type }}"
    s3_access_key: "{{ s3_storage_key }}"
    s3_secret_key: "{{ s3_storage_secret }}"
    s3_bucket: "{{ s3_storage_container }}"
    s3_endpoint: "{{ s3_storage_endpoint }}"
    s3_segment_dir: "druid/rollup/segments"
-   s3_path_like_access: "{{ s3_path_style_access }}"
-   s3_v4_sign_region: "{{ s3_default_bucket_location }}"
+   s3_path_like_access: "{{ cloud_storage_pathstyle_access }}"
+   s3_v4_sign_region: "{{ cloud_private_storage_region }}"
    #Logging the indexing logs to s3
-   s3_logging_bucket: "{{ s3_storage_container }}"
+   s3_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
    s3_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
+   #Druid GCS Details
+   druid.google.bucket: "{{ cloud_storage_telemetry_bucketname }}"
+   druid.google.prefix: ""
+   druid.google.maxListingLength: 1024
+   #Logging the indexing logs to gcs
+   gcloud_logging_bucket: "{{ cloud_storage_telemetry_bucketname }}"
+   gcloud_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
    #Druid coordinator node configuration
    druid_coordinator_heap_size: 128m
    druid_coordinator_period: PT30S

diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
@@ -70,6 +70,10 @@ druid.s3.endpoint.signingRegion={{ druid_configs[cluster].s3_v4_sign_region }}
 # druid.storage.sse.type=s3
 # uncomment to disable chunk encoding
 # druid.s3.disableChunkedEncoding=true
+{% elif druid_storage_type == "google" %}
+druid.google.bucket={{ druid_configs[cluster].gcloud_bucket }}
+druid.google.prefix={{ druid_configs[cluster].gcloud_prefix }}
+druid.google.maxListingLength={{ druid_configs[cluster].gcloud_max_list_length }}
 {% endif %}
 
 # Indexing service logs
@@ -83,6 +87,9 @@ druid.indexer.logs.prefix= {{ druid_configs[cluster].druid_log_azure_folder }}
 druid.indexer.logs.s3Bucket={{ druid_configs[cluster].s3_logging_bucket }}
 # path to logs within the bucker
 druid.indexer.logs.s3Prefix={{ druid_configs[cluster].s3_indexer_logs_dir }}
+{% elif druid_indexing_logs_type == "google" %}
+druid.indexer.logs.bucket={{ druid_configs[cluster].gcloud_logging_bucket }}
+druid.indexer.logs.prefix={{ druid_configs[cluster].gcloud_indexer_logs_dir }}
 {% endif %}
 
 # Service discovery

diff --git a/ansible/roles/analytics-spark-provision/templates/spark-env.j2 b/ansible/roles/analytics-spark-provision/templates/spark-env.j2
@@ -70,12 +70,12 @@ export SPARK_WORKER_MEMORY={{ spark.worker.memory }}
 export SPARK_WORKER_INSTANCES={{ spark.worker.instances }}
 export SPARK_EXECUTOR_MEMORY={{ spark.executor.memory }}
 export SPARK_PUBLIC_DNS="{{ spark.public_dns }}"
-export reports_storage_key={{sunbird_private_storage_account_name}}
-export reports_storage_secret={{sunbird_private_storage_account_key}}
-export azure_storage_key={{sunbird_private_storage_account_name}}
-export azure_storage_secret={{sunbird_private_storage_account_key}}
-export druid_storage_account_key={{sunbird_public_storage_account_name}}
-export druid_storage_account_secret={{sunbird_public_storage_account_key}}
-export aws_storage_key={{ s3_storage_key }}
-export aws_storage_secret={{ s3_storage_secret }}
+export reports_storage_key={{cloud_private_storage_accountname}}
+export reports_storage_secret={{cloud_private_storage_secret}}
+export azure_storage_key={{cloud_private_storage_accountname}}
+export azure_storage_secret={{cloud_private_storage_secret}}
+export druid_storage_account_key={{cloud_private_storage_accountname}}
+export druid_storage_account_secret={{cloud_private_storage_secret}}
+export aws_storage_key={{ cloud_private_storage_accountname }}
+export aws_storage_secret={{ cloud_private_storage_secret }}
 
diff --git a/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2 b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2
@@ -8,8 +8,8 @@ headnode_size="{{headnode_size}}"
 location="{{location}}"
 http_user=admin
 http_password="{{azure_spark_cluster_http_password}}"
-storage_account_name="{{sunbird_private_storage_account_name}}"
-storage_account_key="{{sunbird_private_storage_account_key}}"
+storage_account_name="{{cloud_private_storage_accountname}}"
+storage_account_key="{{cloud_private_storage_secret}}"
 storage_container="{{spark_storage_container}}"
 subnet_name="{{subnet_name}}"
 vnet_name="{{vnet_name}}"

diff --git a/ansible/roles/content-snapshot-indexer/defaults/main.yml b/ansible/roles/content-snapshot-indexer/defaults/main.yml
@@ -46,8 +46,8 @@ cloud_storage:
   container: "telemetry-data-store" # Container is different in all env so override this.
   object_key: "druid-content-snapshot/snapshot.txt"
   provider: "azure"
-  account_name: "{{sunbird_public_storage_account_name}}"
-  account_key: "{{sunbird_public_storage_account_key}}"
+  account_name: "{{cloud_public_storage_accountname}}"
+  account_key: "{{cloud_public_storage_secret}}"
 
 cassandra:
   host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address.

diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
@@ -10,7 +10,7 @@ redis.max.pipeline.size="{{ redis.max_pipeline_size }}"
 
 redis.dialcode.database.index=6
 
-cloudStorage.accountName="{{sunbird_private_storage_account_name}}"
-cloudStorage.accountKey="{{sunbird_private_storage_account_key}}"
+cloudStorage.accountName="{{cloud_private_storage_accountname}}"
+cloudStorage.accountKey="{{cloud_private_storage_secret}}"
 cloudStorage.container="{{ bucket | default('telemetry-data-store') }}"
 cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv"