diff --git a/charts/osdu-developer-init/templates/workflow-init.yaml b/charts/osdu-developer-init/templates/workflow-init.yaml new file mode 100644 index 00000000..00e3299a --- /dev/null +++ b/charts/osdu-developer-init/templates/workflow-init.yaml @@ -0,0 +1,158 @@ +{{- $enabled := eq (include "osdu-developer-init.isEnabled" .) "1" -}} +{{- $namespace := .Release.Namespace -}} +{{- if and $enabled .Values.jobs.workflowInit }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: workflow-init + namespace: {{ $namespace }} +spec: + ttlSecondsAfterFinished: 120 + template: + spec: + volumes: + - name: script + configMap: + name: workflow-init-script + defaultMode: 0500 + initContainers: + - name: data-seed + image: alpine + command: + - script/init.sh + volumeMounts: + - name: script + mountPath: "/script" + env: + - name: NAMESPACE + value: {{ $namespace }} + - name: PARTITION + value: {{ .Values.partition | quote }} + - name: AZURE_TENANT_ID + value: {{ .Values.tenantId | quote }} + - name: AZURE_CLIENT_ID + value: {{ .Values.clientId | quote }} + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.clientSecret.name | quote }} + key: {{ .Values.clientSecret.key | quote }} + containers: + - name: sleep + image: istio/base + command: ["/bin/sleep", "30"] + volumeMounts: # Ensure this container also mounts the volume if needed + - name: script + mountPath: "/script" + restartPolicy: Never +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: workflow-init-script + namespace: {{ $namespace }} +data: + init.sh: | + #!/bin/sh + set -eu + + apk add --no-cache curl jq + + echo "==================================================================" + echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " + echo "==================================================================" + echo " Identity Client Id: ${AZURE_CLIENT_ID}" + + OUTPUT=$(curl -s -w "%{http_code}" --request POST \ + --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ + --header "content-type: application/x-www-form-urlencoded" \ + --data "grant_type=client_credentials" \ + --data "client_id=${AZURE_CLIENT_ID}" \ + --data "client_secret=${AZURE_CLIENT_SECRET}" \ + --data "resource=${AZURE_CLIENT_ID}") + + HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') + BODY=${OUTPUT%???} + + if [[ "$HTTP_STATUS_CODE" != "200" ]]; then + echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" + exit 1 + fi + + TOKEN=$(echo "$BODY" | jq .access_token | tr -d '"') + + # First workflow + WORKFLOW_NAME="Osdu_ingest" + WORKFLOW_DESCRIPTION="Manifest Ingest workflow for OSDU" + + echo "Registering workflow: $WORKFLOW_NAME" + OUTPUT=$(curl -s -w "%{http_code}" --request POST \ + --url http://workflow.osdu-core/api/workflow/v1/workflow \ + --header "Host: workflow.osdu-core" \ + --header "accept: application/json" \ + --header "content-type: application/json" \ + --header "authorization: Bearer $TOKEN" \ + --header "data-partition-id: ${PARTITION}" \ + --data "{ + \"workflowName\": \"$WORKFLOW_NAME\", + \"description\": \"$WORKFLOW_DESCRIPTION\", + \"registrationInstructions\": { + \"active\": true, + \"dagName\": \"$WORKFLOW_NAME\", + \"concurrentWorkflowRun\": 5, + \"concurrentTaskRun\": 5, + \"workflowDetailContent\": \"\", + \"etc\": \"autotest\" + } + }") + + HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') + BODY=${OUTPUT%???} + + if [ "$HTTP_STATUS_CODE" = "200" ]; then + echo "Success: $(echo "$BODY" | jq .)" + else + echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" + echo "Response body: $BODY" + exit 1 + fi + + # Second workflow + WORKFLOW_NAME="Osdu_ingest_by_reference" + WORKFLOW_DESCRIPTION="ManifestIngest by reference workflow for OSDU" + + echo "Registering workflow: $WORKFLOW_NAME" + OUTPUT=$(curl -s -w "%{http_code}" --request POST \ + --url http://workflow.osdu-core/api/workflow/v1/workflow \ + --header "Host: workflow.osdu-core" \ + --header "accept: application/json" \ + --header "content-type: application/json" \ + --header "authorization: Bearer $TOKEN" \ + --header "data-partition-id: ${PARTITION}" \ + --data "{ + \"workflowName\": \"$WORKFLOW_NAME\", + \"description\": \"$WORKFLOW_DESCRIPTION\", + \"registrationInstructions\": { + \"active\": true, + \"dagName\": \"$WORKFLOW_NAME\", + \"concurrentWorkflowRun\": 5, + \"concurrentTaskRun\": 5, + \"workflowDetailContent\": \"\", + \"etc\": \"autotest\" + } + }") + + HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') + BODY=${OUTPUT%???} + + if [ "$HTTP_STATUS_CODE" = "200" ]; then + echo "Success: $(echo "$BODY" | jq .)" + else + echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" + echo "Response body: $BODY" + exit 1 + fi + + exit 0 +{{- end }} \ No newline at end of file diff --git a/docs/src/osdu_core.md b/docs/src/osdu_core.md index 573a2317..7d336657 100644 --- a/docs/src/osdu_core.md +++ b/docs/src/osdu_core.md @@ -15,6 +15,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: partition-azure Class: opengroup.osdu.partition.provider.azure.PartitionApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` The following environment variables are necessary to run the Partition Service. @@ -67,6 +68,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: entitlements-v2-azure Class: org.opengroup.osdu.entitlements.v2.azure.EntitlementsV2Application +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -110,6 +112,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: legal-azure Class: org.opengroup.osdu.legal.azure.LegalApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -163,6 +166,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: os-schema-azure Class: org.opengroup.osdu.schema.azure.SchemaApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -204,6 +208,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: storage-azure Class: org.opengroup.osdu.storage.provider.azure.StorageApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -244,6 +249,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: indexer-azure Class: org.opengroup.osdu.indexer.azure.IndexerAzureApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -279,6 +285,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: search-azure Class: org.opengroup.osdu.search.provider.azure.SearchApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -317,6 +324,7 @@ Build and Run Configuration: SpringBoot Java SDK: java zulu-17 Module: file-azure Class: org.opengroup.osdu.file.provider.azure.FileAzureApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar ``` | Variable | Value | Description | @@ -333,3 +341,44 @@ Class: org.opengroup.osdu.file.provider.azure.FileAzureApplication | `AZURE_PAAS_PODIDENTITY_ISENABLED` | `false` | Azure PaaS pod identity enabled | | `LOG_PREFIX` | `file` | Log prefix | +## Workflow Service + +The workflow service can be run locally in IntelliJ with the following run configuration + +``` +Build and Run Configuration: SpringBoot +--------------------------------------- +Java SDK: java zulu-17 +Module: file-azure +Class: org.opengroup.osdu.workflow.provider.azure.WorkflowAzureApplication +VM Options: -javaagent://osdu-developer/src/applicationinsights-agent.jar +``` + + +| Variable | Value | Description | +|--------------------------------------|------------------------------------------------|--------------------------------------------| +| `APPINSIGHTS_KEY` | `` | Application Insights key | +| `APPLICATIONINSIGHTS_CONNECTION_STRING` | `` | Application Insights Connection | +| `KEYVAULT_URI` | `` | Key Vault URI | +| `AZURE_HOST` | `` | Azure host IP | +| `PARTITION_SERVICE_ENDPOINT` | `http://${AZURE_HOST}/api/partition/v1/` | Partition service endpoint | +| `OSDU_ENTITLEMENTS_URL` | `http://${AZURE_HOST}/api/entitlements/v2` | Entitlements service endpoint | +| `OSDU_AIRFLOW_URL` | `https://${AZURE_HOST}/airflow` | Airflow URL | +| `AAD_CLIENT_ID` | `` | Active Directory client ID | +| `SPRING_APPLICATION_NAME` | `workflow` | Spring application name | +| `LOG_PREFIX` | `workflow` | Log prefix | +| `COSMOSDB_DATABASE` | `osdu-db` | Cosmos DB database name | +| `COSMOSDB_SYSTEM_DATABASE` | `osdu-system-db` | Cosmos DB system database name | +| `AIRFLOW_STORAGE_ACCOUNT_NAME` | `` | Airflow storage account name | +| `AZURE_PAAS_PODIDENTITY` | `false` | Azure PaaS pod identity | +| `AZURE_ISTIOAUTH_ENABLED` | `true` | Turn Istio auth on | +| `AZURE_PAAS_PODIDENTITY_ISENABLED` | `false` | Azure PaaS pod identity enabled | +| `OSDU_AIRFLOW_USERNAME` | `` | Airflow username | +| `OSDU_AIRFLOW_PASSWORD` | ` Edit Configurations**. +2. Select your application configuration +3. In **VM Options**, add: + ``` + -javaagent:/path/to/applicationinsights-agent.jar -DAPPINSIGHTS_LOGGING_ENABLED=true + ``` +4. In **Environment Variables**, add: + - **Key**: `APPLICATIONINSIGHTS_CONNECTION_STRING` + - **Value**: `InstrumentationKey=` +5. Save and run the configuration. + +### Option 3: VS Code Configuration + +1. Create or edit `.vscode/launch.json`. +2. Add the following configuration: + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "type": "java", + "name": "Launch WorkflowAzureApplication", + "request": "launch", + "mainClass": "org.opengroup.osdu.", + "vmArgs": "-javaagent:/path/to/applicationinsights-agent.jar -DAPPINSIGHTS_LOGGING_ENABLED=true", + "env": { + "APPLICATIONINSIGHTS_CONNECTION_STRING": "InstrumentationKey=" + } + } + ] + } + ``` diff --git a/tools/rest-scripts/local.http b/tools/rest-scripts/local.http index d8a3bc31..fd79de4d 100644 --- a/tools/rest-scripts/local.http +++ b/tools/rest-scripts/local.http @@ -34,13 +34,14 @@ grant_type=client_credentials @data_partition_id = opendes + ### # @name infoPartition GET {{PARTITION_HOST}}/info -Authorization: Basic user:password -# Authorization: Bearer {{access_token}} +# Authorization: Basic user:password +Authorization: Bearer {{access_token}} Accept: application/json - +# x-payload: ### # @name createPartition @@ -371,4 +372,40 @@ Accept: application/json GET {{STORAGE_HOST}}/records/123 Authorization: Bearer {{access_token}} Accept: application/json -data-partition-id: {{DATA_PARTITION}} \ No newline at end of file +data-partition-id: {{DATA_PARTITION}} + + + +# ----------------------- +# API: workflow +# ----------------------- +@WORKFLOW_HOST = {{HOST_IP}}/api/workflow/v1 + +### +# @name info +GET {{WORKFLOW_HOST}}/info +Authorization: Bearer {{access_token}} +Accept: application/json +# x-payload: {{JWT_SECTION}} + + +### +# @name getAllWorkflow +GET {{WORKFLOW_HOST}}/workflow +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}} +# x-payload: {{JWT_SECTION}} + + +### +# @name runWorkflowExecution +POST {{WORKFLOW_HOST}}/workflow/test_bash_operator/workflowRun +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}} + +{ + "executionContext": { + } +} \ No newline at end of file diff --git a/tools/rest-scripts/workflow.http b/tools/rest-scripts/workflow.http index a564ab75..20adfffe 100644 --- a/tools/rest-scripts/workflow.http +++ b/tools/rest-scripts/workflow.http @@ -37,7 +37,7 @@ grant_type=refresh_token @access_token = {{refresh.response.body.access_token}} @SERVICE_HOST = {{HOST}}/api/workflow/v1 @data_partition_id = opendes -@workflow_name = test_fetch_remote +@workflow_name = test_bash_operator # ----------------------- @@ -76,9 +76,11 @@ data-partition-id: {{data_partition_id}} "description": "Test Register a Workflow", "registrationInstructions": { "active": true, + "dagName": "{{workflow_name}}", "concurrentWorkflowRun": 5, "concurrentTaskRun": 5, - "workflowDetailContent": "" + "workflowDetailContent": "", + "etc": "autotest" } } @@ -91,6 +93,20 @@ Content-Type: application/json data-partition-id: {{data_partition_id}} +### +# @name runWorkflowExecution +POST {{SERVICE_HOST}}/workflow/{{workflow_name}}/workflowRun +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}} + +{ + "executionContext": { + } +} + + + ### # @name deleteWorkflow DELETE {{SERVICE_HOST}}/workflow/{{workflow_name}}