github workflow changes for flex flows

.github/actions/aml_real_deployment/action.yml

@@ -24,11 +24,20 @@ inputs:
   KEY_VAULT_NAME:
     description: "Key vault name"
     required: true
-
+  ENV_VARS:
+    description: 'The API key for authentication'
+    required: false
 
 runs:
   using: composite
   steps:
+    - name: Create .env file
+      shell: bash
+      run: echo "${{ inputs.ENV_VARS }}" > .env
+    - name: load .env file
+      shell: bash
+      run: python -c "from dotenv import load_dotenv; load_dotenv()"
+
     - name: Provision AML Online Endpoint
       uses: ./.github/actions/execute_script
       with:

.github/actions/configure_azureml_agent/action.yml

@@ -6,7 +6,9 @@ inputs:
   versionSpec:
     description: "The Python version to use in the environment."
     default: "3.9"
-
+  base_path:
+    description: "path to the base directory of the flow"
+    required: true
 
 runs:
   using: composite
@@ -22,9 +24,24 @@ runs:
       shell: bash
       run: |
         set -e # fail on error
-
+        process_requirements() {
+          local folder=$1
+
+          # Loop through all files in the current folder
+          for file in "$folder"/*; do
+              if [ -d "$file" ]; then
+                  # If the file is a directory, recursively process it
+                  process_requirements "$file"
+              elif [ -f "$file" ] && [ "$(basename "$file")" == "requirements.txt" ]; then
+                  # If the file is requirements.txt, install the dependencies
+                  echo "Installing dependencies from $file"
+                  python -m pip install -r "$file"
+              fi
+          done
+        }
         python -m pip install --upgrade pip
         python -m pip install -r .github/requirements/execute_job_requirements.txt
-        python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] PyPDF2 faiss-cpu
+        
+        process_requirements "${{ inputs.base_path }}"
 
         az version

.github/actions/kubernetes_deployment/action.yml

@@ -29,6 +29,12 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Create .env file
+      shell: bash
+      run: echo "${{ inputs.ENV_VARS }}" > .env
+    - name: load .env file
+      shell: bash
+      run: python -c "from dotenv import load_dotenv; load_dotenv()"
     - name: Provision Kubernetes Online Endpoint
       uses: ./.github/actions/execute_script
       with:

.github/actions/webapp_deployment/action.yml

@@ -9,17 +9,23 @@ inputs:
   DEPLOY_ENVIRONMENT:
     description: "env stage e.g. dev, test, prod"
     required: true
-
+  ENV_VARS:
+    description: "env stage e.g. dev, test, prod"
+    required: true
 
 runs:
   using: composite
   steps:
-    - name: prepare docker image
-      uses: ./.github/actions/prepare_docker_image
-      with:
-        USE_CASE_BASE_PATH: ${{ inputs.USE_CASE_BASE_PATH }}
-        DEPLOY_ENVIRONMENT: ${{ inputs.DEPLOY_ENVIRONMENT }}
+    - name: Convert .env to environment variables
+      shell: bash
+      run: |
+        while IFS='=' read -r key value; do
+          echo "$key=$value" >> $GITHUB_ENV
+        done <<< "${{ inputs.env_vars }}"
 
+    - name: create docker image
+      shell: bash
+      run: use_case_base_path=${{ inputs.USE_CASE_BASE_PATH }} deploy_environment=${{ inputs.DEPLOY_ENVIRONMENT }} build_id=${{ github.run_id }} ./llmops/common/scripts/gen_docker_image.sh 
 
     - name: create webapp
       shell: bash

.github/requirements/build_validation_requirements.txt

@@ -5,9 +5,11 @@ pytest-cov==3.0.0
 pytest-azurepipelines==1.0.3
 pytest-mock==3.7.0
 pytest==7.1.2
-azure-ai-ml==1.13.0
-azure-identity==1.11.0
+azure-ai-ml>=1.16.0
+azure-identity>=1.15.0
 mldesigner==0.1.0b4
 pandas==2.2.1
 python-dotenv>=0.10.3
-promptflow==1.6.0
+promptflow>=1.11.0
+python-dotenv
+promptflow-azure>=1.11.0

.github/requirements/execute_job_requirements.txt

@@ -1,6 +1,11 @@
-azure-ai-ml==1.13.0
-azure-identity==1.14.0
-python-dotenv>=0.10.3
-azureml-mlflow>=1.51
+promptflow>=1.11.0
+promptflow-azure>=1.11.0
+promptflow-tools>=1.4.0
+promptflow-evals>=0.3.0
+azure-ai-ml>=1.16.0
+azure-identity==1.15.0
+python-dotenv>=1.0.0
+azureml-mlflow>=1.56.0
 keyrings.alt
-bs4
+python-dotenv
+

.github/workflows/build_validation_workflow.yml

@@ -19,27 +19,34 @@ jobs:
     steps:
       - name: Checkout Actions
         uses: actions/checkout@v4
+
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+
       - name: Load all build validation related dependencies
         shell: bash
         run: |
           set -e # fail on error
           python -m pip install --upgrade pip
           python -m pip install -r .github/requirements/build_validation_requirements.txt
           az version
+
       - name: Azure login
         uses: azure/login@v1
         with:
           creds: ${{ secrets.azure_credentials }}
+
       - name: load the current Azure subscription details
         id: subscription_details
         shell: bash
         run: |
           export subscriptionId=$(az account show --query id -o tsv)
           echo "SUBSCRIPTION_ID=$subscriptionId" >> $GITHUB_OUTPUT
 
+      - name: Python code linting
+        run:  flake8 ./llmops ./tests
+
       - name: Execute Unit Tests
         shell: bash
         run: |

.github/workflows/chat_with_pdf_ci_dev_workflow.yml

@@ -23,9 +23,9 @@ on:
       - main
       - development
     paths:
+      - 'chat_with_pdf/**'
       - '.github/**'
       - 'llmops/**'
-      - 'chat_with_pdf/**'
 
 
 #=====================================
@@ -37,8 +37,8 @@ jobs:
     with:
       env_name: ${{ inputs.env_name || 'dev'}}
       use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }}
-      deployment_type: ${{ inputs.deployment_type|| 'aml' }}
+      deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
     secrets:
       azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
-      connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }}
       registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
+      env_vars: ${{ secrets.ENV_VARS }}

.github/workflows/chat_with_pdf_pr_dev_workflow.yml

@@ -18,9 +18,9 @@ on:
       - main
       - development
     paths:
+      - 'chat_with_pdf/**'
       - '.github/**'
       - 'llmops/**'
-      - 'chat_with_pdf/**'
 
 #=====================================
 # Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
@@ -33,3 +33,4 @@ jobs:
       use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }}
     secrets:
       azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+      env_vars: ${{ secrets.ENV_VARS }}

.github/workflows/class_flows_ci_dev_workflow.yml

@@ -0,0 +1,42 @@
+name: class_flows_ci_dev_workflow
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The flow usecase to execute"
+        required: true
+        default: "class_flows"
+      deployment_type:
+        type: string
+        description: "Determine type of deployment - aml, aks, docker, webapp"
+        required: true
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - 'class_flows/**'
+      - '.github/**'
+      - 'llmops/**'
+
+#=====================================
+# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
+#=====================================
+jobs:
+  execute-platform-flow-ci:
+    uses: ./.github/workflows/platform_ci_dev_workflow.yml
+    with:
+      env_name: ${{ inputs.env_name || 'dev'}}
+      use_case_base_path: ${{ inputs.use_case_base_path || 'class_flows' }}
+      deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
+    secrets:
+      azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+      registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
+      env_vars: ${{ secrets.ENV_VARS }}

.github/workflows/class_flows_pr_dev_workflow.yml

@@ -0,0 +1,37 @@
+name: class_flows_ci_dev_workflow
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The flow usecase to execute"
+        required: true
+        default: "class_flows"
+  pull_request:
+    branches:
+      - main
+      - development
+    paths:
+      - '.github/**'
+      - 'llmops/**'
+      - 'class_flows/**'
+
+#=====================================
+# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
+#=====================================
+jobs:
+  execute-platform-pr-workflow:
+    uses: ./.github/workflows/platform_pr_dev_workflow.yml
+    with:
+      env_name: ${{ inputs.env_name || 'pr'}}
+      use_case_base_path: ${{ inputs.flow_type || 'class_flows' }}
+    secrets:
+      azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+      env_vars: ${{ secrets.ENV_VARS }}
+

.github/workflows/flex_flows_ci_dev_workflow.yml

@@ -0,0 +1,42 @@
+name: function_flows_ci_dev_workflow
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The flow usecase to execute"
+        required: true
+        default: "function_flows"
+      deployment_type:
+        type: string
+        description: "Determine type of deployment - aml, aks, docker, webapp"
+        required: true
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - 'function_flows/**'
+      - '.github/**'
+      - 'llmops/**'
+
+#=====================================
+# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
+#=====================================
+jobs:
+  execute-platform-flow-ci:
+    uses: ./.github/workflows/platform_ci_dev_workflow.yml
+    with:
+      env_name: ${{ inputs.env_name || 'dev'}}
+      use_case_base_path: ${{ inputs.use_case_base_path || 'function_flows' }}
+      deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
+    secrets:
+      azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+      registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
+      env_vars: ${{ secrets.ENV_VARS }}

.github/workflows/flex_flows_pr_dev_workflow.yml

@@ -0,0 +1,37 @@
+name: function_flows_pr_dev_workflow
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The flow usecase to execute"
+        required: true
+        default: "function_flows"
+  pull_request:
+    branches:
+      - main
+      - development
+    paths:
+      - 'function_flows/**'
+      - '.github/**'
+      - 'llmops/**'
+
+#=====================================
+# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
+#=====================================
+jobs:
+  execute-platform-pr-workflow:
+    uses: ./.github/workflows/platform_pr_dev_workflow.yml
+    with:
+      env_name: ${{ inputs.env_name || 'pr'}}
+      use_case_base_path: ${{ inputs.flow_type || 'function_flows' }}
+    secrets:
+      azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+      env_vars: ${{ secrets.ENV_VARS }}
+

.github/workflows/math_coding_ci_dev_workflow.yml

@@ -22,9 +22,9 @@ on:
       - main
       - development
     paths:
+      - 'math_coding/**'
       - '.github/**'
       - 'llmops/**'
-      - 'math_coding/**'
 
 #=====================================
 # Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
@@ -35,8 +35,8 @@ jobs:
     with:
       env_name: ${{ inputs.env_name || 'dev'}}
       use_case_base_path: ${{ inputs.use_case_base_path || 'math_coding' }}
-      deployment_type: ${{ inputs.deployment_type|| 'aml' }}
+      deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
     secrets:
       azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
-      connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }}
       registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
+      env_vars: ${{ secrets.ENV_VARS }}