Skip to content

Commit c19124e

Browse files
ritesh-modiritesh-modi
committed
github workflow changes for flex flows
1 parent d658e6a commit c19124e

25 files changed

+302
-44
lines changed

‎.github/actions/aml_real_deployment/action.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,20 @@ inputs:
2424
KEY_VAULT_NAME:
2525
description: "Key vault name"
2626
required: true
27-
27+
ENV_VARS:
28+
description: 'The API key for authentication'
29+
required: false
2830

2931
runs:
3032
using: composite
3133
steps:
34+
- name: Create .env file
35+
shell: bash
36+
run: echo "${{ inputs.ENV_VARS }}" > .env
37+
- name: load .env file
38+
shell: bash
39+
run: python -c "from dotenv import load_dotenv; load_dotenv()"
40+
3241
- name: Provision AML Online Endpoint
3342
uses: ./.github/actions/execute_script
3443
with:

‎.github/actions/configure_azureml_agent/action.yml

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ inputs:
66
versionSpec:
77
description: "The Python version to use in the environment."
88
default: "3.9"
9-
9+
base_path:
10+
description: "path to the base directory of the flow"
11+
required: true
1012

1113
runs:
1214
using: composite
@@ -22,9 +24,24 @@ runs:
2224
shell: bash
2325
run: |
2426
set -e # fail on error
25-
27+
process_requirements() {
28+
local folder=$1
29+
30+
# Loop through all files in the current folder
31+
for file in "$folder"/*; do
32+
if [ -d "$file" ]; then
33+
# If the file is a directory, recursively process it
34+
process_requirements "$file"
35+
elif [ -f "$file" ] && [ "$(basename "$file")" == "requirements.txt" ]; then
36+
# If the file is requirements.txt, install the dependencies
37+
echo "Installing dependencies from $file"
38+
python -m pip install -r "$file"
39+
fi
40+
done
41+
}
2642
python -m pip install --upgrade pip
2743
python -m pip install -r .github/requirements/execute_job_requirements.txt
28-
python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] PyPDF2 faiss-cpu
44+
45+
process_requirements "${{ inputs.base_path }}"
2946
3047
az version

‎.github/actions/kubernetes_deployment/action.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ inputs:
2929
runs:
3030
using: composite
3131
steps:
32+
- name: Create .env file
33+
shell: bash
34+
run: echo "${{ inputs.ENV_VARS }}" > .env
35+
- name: load .env file
36+
shell: bash
37+
run: python -c "from dotenv import load_dotenv; load_dotenv()"
3238
- name: Provision Kubernetes Online Endpoint
3339
uses: ./.github/actions/execute_script
3440
with:

‎.github/actions/webapp_deployment/action.yml

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,23 @@ inputs:
99
DEPLOY_ENVIRONMENT:
1010
description: "env stage e.g. dev, test, prod"
1111
required: true
12-
12+
ENV_VARS:
13+
description: "env stage e.g. dev, test, prod"
14+
required: true
1315

1416
runs:
1517
using: composite
1618
steps:
17-
- name: prepare docker image
18-
uses: ./.github/actions/prepare_docker_image
19-
with:
20-
USE_CASE_BASE_PATH: ${{ inputs.USE_CASE_BASE_PATH }}
21-
DEPLOY_ENVIRONMENT: ${{ inputs.DEPLOY_ENVIRONMENT }}
19+
- name: Convert .env to environment variables
20+
shell: bash
21+
run: |
22+
while IFS='=' read -r key value; do
23+
echo "$key=$value" >> $GITHUB_ENV
24+
done <<< "${{ inputs.env_vars }}"
2225
26+
- name: create docker image
27+
shell: bash
28+
run: use_case_base_path=${{ inputs.USE_CASE_BASE_PATH }} deploy_environment=${{ inputs.DEPLOY_ENVIRONMENT }} build_id=${{ github.run_id }} ./llmops/common/scripts/gen_docker_image.sh
2329

2430
- name: create webapp
2531
shell: bash

‎.github/requirements/build_validation_requirements.txt

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ pytest-cov==3.0.0
55
pytest-azurepipelines==1.0.3
66
pytest-mock==3.7.0
77
pytest==7.1.2
8-
azure-ai-ml==1.13.0
9-
azure-identity==1.11.0
8+
azure-ai-ml>=1.16.0
9+
azure-identity>=1.15.0
1010
mldesigner==0.1.0b4
1111
pandas==2.2.1
1212
python-dotenv>=0.10.3
13-
promptflow==1.6.0
13+
promptflow>=1.11.0
14+
python-dotenv
15+
promptflow-azure>=1.11.0

‎.github/requirements/execute_job_requirements.txt

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
1-
azure-ai-ml==1.13.0
2-
azure-identity==1.14.0
3-
python-dotenv>=0.10.3
4-
azureml-mlflow>=1.51
1+
promptflow>=1.11.0
2+
promptflow-azure>=1.11.0
3+
promptflow-tools>=1.4.0
4+
promptflow-evals>=0.3.0
5+
azure-ai-ml>=1.16.0
6+
azure-identity==1.15.0
7+
python-dotenv>=1.0.0
8+
azureml-mlflow>=1.56.0
59
keyrings.alt
6-
bs4
10+
python-dotenv
11+

‎.github/workflows/build_validation_workflow.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,27 +19,34 @@ jobs:
1919
steps:
2020
- name: Checkout Actions
2121
uses: actions/checkout@v4
22+
2223
- uses: actions/setup-python@v4
2324
with:
2425
python-version: "3.9"
26+
2527
- name: Load all build validation related dependencies
2628
shell: bash
2729
run: |
2830
set -e # fail on error
2931
python -m pip install --upgrade pip
3032
python -m pip install -r .github/requirements/build_validation_requirements.txt
3133
az version
34+
3235
- name: Azure login
3336
uses: azure/login@v1
3437
with:
3538
creds: ${{ secrets.azure_credentials }}
39+
3640
- name: load the current Azure subscription details
3741
id: subscription_details
3842
shell: bash
3943
run: |
4044
export subscriptionId=$(az account show --query id -o tsv)
4145
echo "SUBSCRIPTION_ID=$subscriptionId" >> $GITHUB_OUTPUT
4246
47+
- name: Python code linting
48+
run: flake8 ./llmops ./tests
49+
4350
- name: Execute Unit Tests
4451
shell: bash
4552
run: |

‎.github/workflows/chat_with_pdf_ci_dev_workflow.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ on:
2323
- main
2424
- development
2525
paths:
26+
- 'chat_with_pdf/**'
2627
- '.github/**'
2728
- 'llmops/**'
28-
- 'chat_with_pdf/**'
2929

3030

3131
#=====================================
@@ -37,8 +37,8 @@ jobs:
3737
with:
3838
env_name: ${{ inputs.env_name || 'dev'}}
3939
use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }}
40-
deployment_type: ${{ inputs.deployment_type|| 'aml' }}
40+
deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
4141
secrets:
4242
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
43-
connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }}
4443
registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
44+
env_vars: ${{ secrets.ENV_VARS }}

‎.github/workflows/chat_with_pdf_pr_dev_workflow.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ on:
1818
- main
1919
- development
2020
paths:
21+
- 'chat_with_pdf/**'
2122
- '.github/**'
2223
- 'llmops/**'
23-
- 'chat_with_pdf/**'
2424

2525
#=====================================
2626
# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
@@ -33,3 +33,4 @@ jobs:
3333
use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }}
3434
secrets:
3535
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
36+
env_vars: ${{ secrets.ENV_VARS }}

‎.github/workflows/class_flows_ci_dev_workflow.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: class_flows_ci_dev_workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
env_name:
7+
type: string
8+
description: "Execution Environment"
9+
required: true
10+
default: "dev"
11+
use_case_base_path:
12+
type: string
13+
description: "The flow usecase to execute"
14+
required: true
15+
default: "class_flows"
16+
deployment_type:
17+
type: string
18+
description: "Determine type of deployment - aml, aks, docker, webapp"
19+
required: true
20+
push:
21+
branches:
22+
- main
23+
- development
24+
paths:
25+
- 'class_flows/**'
26+
- '.github/**'
27+
- 'llmops/**'
28+
29+
#=====================================
30+
# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
31+
#=====================================
32+
jobs:
33+
execute-platform-flow-ci:
34+
uses: ./.github/workflows/platform_ci_dev_workflow.yml
35+
with:
36+
env_name: ${{ inputs.env_name || 'dev'}}
37+
use_case_base_path: ${{ inputs.use_case_base_path || 'class_flows' }}
38+
deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
39+
secrets:
40+
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
41+
registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
42+
env_vars: ${{ secrets.ENV_VARS }}

‎.github/workflows/class_flows_pr_dev_workflow.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: class_flows_ci_dev_workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
env_name:
7+
type: string
8+
description: "Execution Environment"
9+
required: true
10+
default: "dev"
11+
use_case_base_path:
12+
type: string
13+
description: "The flow usecase to execute"
14+
required: true
15+
default: "class_flows"
16+
pull_request:
17+
branches:
18+
- main
19+
- development
20+
paths:
21+
- '.github/**'
22+
- 'llmops/**'
23+
- 'class_flows/**'
24+
25+
#=====================================
26+
# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
27+
#=====================================
28+
jobs:
29+
execute-platform-pr-workflow:
30+
uses: ./.github/workflows/platform_pr_dev_workflow.yml
31+
with:
32+
env_name: ${{ inputs.env_name || 'pr'}}
33+
use_case_base_path: ${{ inputs.flow_type || 'class_flows' }}
34+
secrets:
35+
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
36+
env_vars: ${{ secrets.ENV_VARS }}
37+

‎.github/workflows/flex_flows_ci_dev_workflow.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: function_flows_ci_dev_workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
env_name:
7+
type: string
8+
description: "Execution Environment"
9+
required: true
10+
default: "dev"
11+
use_case_base_path:
12+
type: string
13+
description: "The flow usecase to execute"
14+
required: true
15+
default: "function_flows"
16+
deployment_type:
17+
type: string
18+
description: "Determine type of deployment - aml, aks, docker, webapp"
19+
required: true
20+
push:
21+
branches:
22+
- main
23+
- development
24+
paths:
25+
- 'function_flows/**'
26+
- '.github/**'
27+
- 'llmops/**'
28+
29+
#=====================================
30+
# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
31+
#=====================================
32+
jobs:
33+
execute-platform-flow-ci:
34+
uses: ./.github/workflows/platform_ci_dev_workflow.yml
35+
with:
36+
env_name: ${{ inputs.env_name || 'dev'}}
37+
use_case_base_path: ${{ inputs.use_case_base_path || 'function_flows' }}
38+
deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
39+
secrets:
40+
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
41+
registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
42+
env_vars: ${{ secrets.ENV_VARS }}

‎.github/workflows/flex_flows_pr_dev_workflow.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: function_flows_pr_dev_workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
env_name:
7+
type: string
8+
description: "Execution Environment"
9+
required: true
10+
default: "dev"
11+
use_case_base_path:
12+
type: string
13+
description: "The flow usecase to execute"
14+
required: true
15+
default: "function_flows"
16+
pull_request:
17+
branches:
18+
- main
19+
- development
20+
paths:
21+
- 'function_flows/**'
22+
- '.github/**'
23+
- 'llmops/**'
24+
25+
#=====================================
26+
# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
27+
#=====================================
28+
jobs:
29+
execute-platform-pr-workflow:
30+
uses: ./.github/workflows/platform_pr_dev_workflow.yml
31+
with:
32+
env_name: ${{ inputs.env_name || 'pr'}}
33+
use_case_base_path: ${{ inputs.flow_type || 'function_flows' }}
34+
secrets:
35+
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
36+
env_vars: ${{ secrets.ENV_VARS }}
37+

‎.github/workflows/math_coding_ci_dev_workflow.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ on:
2222
- main
2323
- development
2424
paths:
25+
- 'math_coding/**'
2526
- '.github/**'
2627
- 'llmops/**'
27-
- 'math_coding/**'
2828

2929
#=====================================
3030
# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
@@ -35,8 +35,8 @@ jobs:
3535
with:
3636
env_name: ${{ inputs.env_name || 'dev'}}
3737
use_case_base_path: ${{ inputs.use_case_base_path || 'math_coding' }}
38-
deployment_type: ${{ inputs.deployment_type|| 'aml' }}
38+
deployment_type: ${{ inputs.deployment_type|| 'webapp' }}
3939
secrets:
4040
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
41-
connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }}
4241
registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}
42+
env_vars: ${{ secrets.ENV_VARS }}

0 commit comments

Comments
 (0)