Configure Workflow Service with DAGS loaded

charts/osdu-developer-init/templates/workflow-init.yaml

@@ -0,0 +1,158 @@
+{{- $enabled := eq (include "osdu-developer-init.isEnabled" .) "1" -}}
+{{- $namespace := .Release.Namespace -}}
+{{- if and $enabled .Values.jobs.workflowInit }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: workflow-init
+  namespace: {{ $namespace }}
+spec:
+  ttlSecondsAfterFinished: 120
+  template:
+    spec:
+      volumes:
+      - name: script
+        configMap:
+          name: workflow-init-script
+          defaultMode: 0500
+      initContainers:
+      - name: data-seed
+        image: alpine
+        command:
+        - script/init.sh
+        volumeMounts:
+          - name: script
+            mountPath: "/script"
+        env:
+          - name: NAMESPACE
+            value: {{ $namespace }}
+          - name: PARTITION
+            value: {{ .Values.partition | quote }}
+          - name: AZURE_TENANT_ID
+            value: {{ .Values.tenantId | quote }}
+          - name: AZURE_CLIENT_ID
+            value: {{ .Values.clientId | quote }}
+          - name: AZURE_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.clientSecret.name | quote }}
+                key: {{ .Values.clientSecret.key | quote }}
+      containers:
+      - name: sleep
+        image: istio/base
+        command: ["/bin/sleep", "30"]
+        volumeMounts:  # Ensure this container also mounts the volume if needed
+          - name: script
+            mountPath: "/script"
+      restartPolicy: Never
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-init-script
+  namespace: {{ $namespace }}
+data:
+  init.sh: |
+    #!/bin/sh
+    set -eu
+
+    apk add --no-cache curl jq
+
+    echo "=================================================================="
+    echo "  Creating Bearer Token for Application: ${AZURE_CLIENT_ID}       "
+    echo "=================================================================="
+    echo "  Identity Client Id: ${AZURE_CLIENT_ID}"
+
+    OUTPUT=$(curl -s -w "%{http_code}" --request POST \
+        --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \
+        --header "content-type: application/x-www-form-urlencoded" \
+        --data "grant_type=client_credentials" \
+        --data "client_id=${AZURE_CLIENT_ID}" \
+        --data "client_secret=${AZURE_CLIENT_SECRET}" \
+        --data "resource=${AZURE_CLIENT_ID}")
+
+    HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$')
+    BODY=${OUTPUT%???}
+
+    if [[ "$HTTP_STATUS_CODE" != "200" ]]; then
+      echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE"
+      exit 1
+    fi
+
+    TOKEN=$(echo "$BODY" | jq .access_token | tr -d '"')
+
+    # First workflow
+    WORKFLOW_NAME="Osdu_ingest"
+    WORKFLOW_DESCRIPTION="Manifest Ingest workflow for OSDU"
+
+    echo "Registering workflow: $WORKFLOW_NAME"
+    OUTPUT=$(curl -s -w "%{http_code}" --request POST \
+      --url http://workflow.osdu-core/api/workflow/v1/workflow \
+      --header "Host: workflow.osdu-core" \
+      --header "accept: application/json" \
+      --header "content-type: application/json" \
+      --header "authorization: Bearer $TOKEN" \
+      --header "data-partition-id: ${PARTITION}" \
+      --data "{
+        \"workflowName\": \"$WORKFLOW_NAME\",
+        \"description\": \"$WORKFLOW_DESCRIPTION\",
+        \"registrationInstructions\": {
+          \"active\": true,
+          \"dagName\": \"$WORKFLOW_NAME\",
+          \"concurrentWorkflowRun\": 5,
+          \"concurrentTaskRun\": 5,
+          \"workflowDetailContent\": \"\",
+          \"etc\": \"autotest\"
+        }
+      }")
+
+    HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$')
+    BODY=${OUTPUT%???}
+
+    if [ "$HTTP_STATUS_CODE" = "200" ]; then
+      echo "Success: $(echo "$BODY" | jq .)"
+    else
+      echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE"
+      echo "Response body: $BODY"
+      exit 1
+    fi
+
+    # Second workflow
+    WORKFLOW_NAME="Osdu_ingest_by_reference"
+    WORKFLOW_DESCRIPTION="ManifestIngest by reference workflow for OSDU"
+
+    echo "Registering workflow: $WORKFLOW_NAME"
+    OUTPUT=$(curl -s -w "%{http_code}" --request POST \
+      --url http://workflow.osdu-core/api/workflow/v1/workflow \
+      --header "Host: workflow.osdu-core" \
+      --header "accept: application/json" \
+      --header "content-type: application/json" \
+      --header "authorization: Bearer $TOKEN" \
+      --header "data-partition-id: ${PARTITION}" \
+      --data "{
+        \"workflowName\": \"$WORKFLOW_NAME\",
+        \"description\": \"$WORKFLOW_DESCRIPTION\",
+        \"registrationInstructions\": {
+          \"active\": true,
+          \"dagName\": \"$WORKFLOW_NAME\",
+          \"concurrentWorkflowRun\": 5,
+          \"concurrentTaskRun\": 5,
+          \"workflowDetailContent\": \"\",
+          \"etc\": \"autotest\"
+        }
+      }")
+
+    HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$')
+    BODY=${OUTPUT%???}
+
+    if [ "$HTTP_STATUS_CODE" = "200" ]; then
+      echo "Success: $(echo "$BODY" | jq .)"
+    else
+      echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE"
+      echo "Response body: $BODY"
+      exit 1
+    fi
+
+    exit 0
+{{- end }}

docs/src/osdu_core.md

@@ -15,6 +15,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: partition-azure
 Class: opengroup.osdu.partition.provider.azure.PartitionApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 The following environment variables are necessary to run the Partition Service.
@@ -67,6 +68,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: entitlements-v2-azure
 Class: org.opengroup.osdu.entitlements.v2.azure.EntitlementsV2Application
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -110,6 +112,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: legal-azure
 Class: org.opengroup.osdu.legal.azure.LegalApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -163,6 +166,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: os-schema-azure
 Class: org.opengroup.osdu.schema.azure.SchemaApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -204,6 +208,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: storage-azure
 Class: org.opengroup.osdu.storage.provider.azure.StorageApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -244,6 +249,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: indexer-azure
 Class: org.opengroup.osdu.indexer.azure.IndexerAzureApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -279,6 +285,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: search-azure
 Class: org.opengroup.osdu.search.provider.azure.SearchApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -317,6 +324,7 @@ Build and Run Configuration: SpringBoot
 Java SDK:  java zulu-17
 Module: file-azure
 Class: org.opengroup.osdu.file.provider.azure.FileAzureApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
 ```
 
 | Variable                             | Value                                          | Description                                |
@@ -333,3 +341,44 @@ Class: org.opengroup.osdu.file.provider.azure.FileAzureApplication
 | `AZURE_PAAS_PODIDENTITY_ISENABLED`   | `false`                                        | Azure PaaS pod identity enabled            |
 | `LOG_PREFIX`                         | `file`                                         | Log prefix                                 |
 
+## Workflow Service
+
+The workflow service can be run locally in IntelliJ with the following run configuration
+
+```
+Build and Run Configuration: SpringBoot
+---------------------------------------
+Java SDK:  java zulu-17
+Module: file-azure
+Class: org.opengroup.osdu.workflow.provider.azure.WorkflowAzureApplication
+VM Options: -javaagent:/<your_full_path>/osdu-developer/src/applicationinsights-agent.jar
+```
+
+
+| Variable                             | Value                                          | Description                                |
+|--------------------------------------|------------------------------------------------|--------------------------------------------|
+| `APPINSIGHTS_KEY`                    | `<your_appinsights_key>`                       | Application Insights key                   |
+| `APPLICATIONINSIGHTS_CONNECTION_STRING` | `<your_appinsights_connection>`             | Application Insights Connection            |
+| `KEYVAULT_URI`                       | `<your_keyvault_uri>`                          | Key Vault URI                              |
+| `AZURE_HOST`                         | `<your_host_ip>`                               | Azure host IP                              |
+| `PARTITION_SERVICE_ENDPOINT`         | `http://${AZURE_HOST}/api/partition/v1/`       | Partition service endpoint                 |
+| `OSDU_ENTITLEMENTS_URL`              | `http://${AZURE_HOST}/api/entitlements/v2`     | Entitlements service endpoint              |
+| `OSDU_AIRFLOW_URL`                   | `https://${AZURE_HOST}/airflow`                | Airflow URL                                |
+| `AAD_CLIENT_ID`                      | `<your_aad_client_id>`                         | Active Directory client ID                 |
+| `SPRING_APPLICATION_NAME`            | `workflow`                                     | Spring application name                    |
+| `LOG_PREFIX`                         | `workflow`                                     | Log prefix                                 |
+| `COSMOSDB_DATABASE`                  | `osdu-db`                                      | Cosmos DB database name                    |
+| `COSMOSDB_SYSTEM_DATABASE`           | `osdu-system-db`                               | Cosmos DB system database name             |
+| `AIRFLOW_STORAGE_ACCOUNT_NAME`       | `<your_storage_account_name>`                  | Airflow storage account name               |
+| `AZURE_PAAS_PODIDENTITY`             | `false`                                        | Azure PaaS pod identity                    |
+| `AZURE_ISTIOAUTH_ENABLED`            | `true`                                         | Turn Istio auth on                         |
+| `AZURE_PAAS_PODIDENTITY_ISENABLED`   | `false`                                        | Azure PaaS pod identity enabled            |
+| `OSDU_AIRFLOW_USERNAME`              | `<your_airflow_username>`                      | Airflow username                           |
+| `OSDU_AIRFLOW_PASSWORD`              | `<your_airflow_password`                       | Airflow password                           |
+| `OSDU_AIRFLOW_VERSION2_ENABLED`      | `true`                                         | Enable Airflow version 2                   |
+| `DP_AIRFLOW_FOR_SYSTEM_DAG`          | `false`                                        | Use Airflow for system DAGs                |
+| `IGNORE_DAGCONTENT`                  | `true`                                         | Ignore DAG content                         |
+| `IGNORE_CUSTOMOPERATORCONTENT`       | `true`                                         | Ignore custom operator content             |
+| `SERVER_PORT`                        | `8080`                                         | Server port                                |
+
+// ... end of file ...