Skip to content

Commit

Permalink
Updating to use new command
Browse files Browse the repository at this point in the history
  • Loading branch information
@CodeByDrescher
CodeByDrescher committed Dec 19, 2024
1 parent e9b055a commit 6123a9f
Show file tree
Hide file tree
Showing 2 changed files with 219 additions and 121 deletions.
335 changes: 214 additions & 121 deletions .github/workflows/NightlyBMDB_CLI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,102 +117,129 @@ jobs:
uses: mxschmitt/action-tmate@v3
if: ${{ failure() }}

published:
runs-on: ubuntu-20.04
needs: build
strategy:
matrix:
sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
steps:
- name: Create Docker Image Dir
run: mkdir /tmp/docker

- name: Download Docker Image from `build` Job
uses: actions/download-artifact@v3
with:
name: docker-image
path: /tmp/docker

- name: Load Downloaded Docker Image
run: docker load --input /tmp/docker/$(ls /tmp/docker)

# Get files from BMDB
- name: Checkout Input Files
uses: actions/checkout@v4
with:
repository: virtualcell/vcdb

- name: Prepare sub directory
run: |
SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
MOD_VALUE=${{ matrix.sets }}
file_index=0
# make target dir
mkdir $TARGET_DIR
# loop through files in directory, increment indexing and separating files
for file in $SOURCE_DIR/*; do
file_index=$((file_index+1))
if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
cp ${file} $TARGET_DIR/
fi
done
# Do the execution
- name: Begin Execution
#$(docker image ls | grep "<none>" | awk '{print $3;}') to get image id
run: |
# The /* goes on the outside, otherwise bash just interprets a string!
for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
echo "\"$file\""
if [ -f "$file" ]; then
extensionless_name="${file%.*}"
base_name="$(basename ${file})"
base_extless_name="$(basename ${extensionless_name})"
mkdir "${extensionless_name}_output"
echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
sim_input="/root/${base_name}"
sim_output="/root/${base_extless_name}_output"
docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
echo -e "\n\n\n\n\n" # give ourselves some line separation
# Check for individual Success
if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
then
# Copy error log to github workspace
touch ${{ github.workspace }}/errors.report
cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
else
echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
fi
fi
done
- name: Check For Success
run: |
# If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
then
exit 0
fi
echo "Models that failed:"
cat ${{ github.workspace }}/errors.report
- name: Upload failures
uses: actions/upload-artifact@v3
with:
name: set_${{ matrix.sets }}.report
path: ${{ github.workspace }}/errors.report

# Report a Problem to Slack
- name: Report Problem to Slack
if: ${{ failure() }}
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage

- name: Setup tmate session 3
uses: mxschmitt/action-tmate@v3
if: ${{ failure() }}
# published:
# runs-on: ubuntu-20.04
# needs: build
# strategy:
# matrix:
# sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
# steps:
# - name: Create Docker Image Dir
# run: mkdir /tmp/docker
#
# - name: Download Docker Image from `build` Job
# uses: actions/download-artifact@v3
# with:
# name: docker-image
# path: /tmp/docker
#
# - name: Load Downloaded Docker Image
# run: docker load --input /tmp/docker/$(ls /tmp/docker)
#
# # Get files from BMDB
# - name: Checkout Input Files
# uses: actions/checkout@v4
# with:
# repository: virtualcell/vcdb
#
# - name: Prepare sub directory
# run: |
# SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
# TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
# MOD_VALUE=${{ matrix.sets }}
# file_index=0
#
# # make target dir
# mkdir $TARGET_DIR
# # loop through files in directory, increment indexing and separating files
# for file in $SOURCE_DIR/*; do
# file_index=$((file_index+1))
# if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
# cp ${file} $TARGET_DIR/
# fi
# done
#
# # Do the execution
# - name: Begin Execution
# #$(docker image ls | grep "<none>" | awk '{print $3;}') to get image id
# run: |
# # Prepare files
# echo "[]" > ${{ github.workspace }}/total_exec_summary.json
# echo "[]" > ${{ github.workspace }}/full_tracer.json
# # The /* goes on the outside, otherwise bash just interprets a string!
# for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
# echo "\"$file\""
# if [ -f "$file" ]; then
# extensionless_name="${file%.*}"
# base_name="$(basename ${file})"
# base_extless_name="$(basename ${extensionless_name})"
# mkdir "${extensionless_name}_output"
# echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
# sim_input="/root/${base_name}"
# sim_output="/root/${base_extless_name}_output"
# vcell_result=$(docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute-omex -d -i ${sim_input} -o ${sim_output} --timeout_ms=300000)
#
# echo -e "\n\n\n\n\n" # give ourselves some line separation
#
# # Grab Summary
# if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json | wc -l) -ne 0 ]
# then
# # Append to master summary in github workspace
# jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
# mv temp.json ${{ github.workspace }}/total_exec_summary.json
# else
# echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
# fi
#
# # Check for individual Failure (a.k.a. if we failed early)
# status=$(cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) | jq -r '.status')
# if [[ "$status" == "FAILED" ]] || [ ${vcell_result} -ne 0 ]
# then
# # Copy error log to github workspace
# echo "${base_extless_name}" >> $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt
# touch ${{ github.workspace }}/errors.report
# cat $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt >> ${{ github.workspace }}/errors.report
# else
# echo "Run of ${base_extless_name} succeeded (set_${{ matrix.sets }})"
# fi
#
# # Grab Trace
# # Check for individual Success
# if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json | wc -l) -ne 0 ]
# then
# # Append to master trace in github workspace
# jq '. + inputs' ${{ github.workspace }}/full_tracer.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json) > temp_2.json
# mv temp_2.json ${{ github.workspace }}/full_tracer.json
# else
# echo "No trace found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
# fi
# fi
# done
#
# - name: Check For Success
# run: |
# # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
# if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
# then
# exit 0
# fi
# echo "Models that failed:"
# cat ${{ github.workspace }}/errors.report
#
# - name: Upload failures
# uses: actions/upload-artifact@v3
# with:
# name: set_${{ matrix.sets }}.report
# path: ${{ github.workspace }}/errors.report
#
# # Report a Problem to Slack
# - name: Report Problem to Slack
# if: ${{ failure() }}
# run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
#
# - name: Setup tmate session 3
# uses: mxschmitt/action-tmate@v3
# if: ${{ failure() }}

bmdb:
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -271,13 +298,18 @@ jobs:
fi
done
- name: Install JQ
run: sudo apt update && sudo apt install jq -y

# Do the execution
- name: Begin Execution
#$(docker image ls | grep "<none>" | awk '{print $3;}') to get image id
run: |
# Prepare files
echo "[]" > ${{ github.workspace }}/total_exec_summary.json
echo "[]" > ${{ github.workspace }}/full_tracer.json
# The /* goes on the outside, otherwise bash just interprets a string!
for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
echo "\"$file\""
if [ -f "$file" ]; then
extensionless_name="${file%.*}"
base_name="$(basename ${file})"
Expand All @@ -286,22 +318,56 @@ jobs:
echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
sim_input="/root/${base_name}"
sim_output="/root/${base_extless_name}_output"
docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
vcell_result=$(docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute-omex -d -i ${sim_input} -o ${sim_output} --timeout_ms=300000)
echo -e "\n\n\n\n\n" # give ourselves some line separation
# Check for individual Success
if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
# Grab Summary
echo -n "Obtaining Summary..."
if [ -f "${GITHUB_WORKSPACE}/set_${{ matrix.sets }}/${base_extless_name}_output/exec_summary.json" ]
then
# Append to master summary in github workspace
jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
mv temp.json ${{ github.workspace }}/total_exec_summary.json
echo "Done!"
else
echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
fi
# Check for individual Failure (a.k.a. if we failed early)
echo -n "Obtaining Execution Result..."
status=$(cat "${GITHUB_WORKSPACE}/set_${{ matrix.sets }}/${base_extless_name}_output/exec_summary.json" | jq -r '.status')
echo "${status}"
# Note: `cat`ing "$vcell_result" will be the runting logging!
if [ "${status}" = "FAILED" ]; then
# Copy error log to github workspace
echo -n "Copying Summary..."
echo "${base_extless_name}" >> $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt
touch ${{ github.workspace }}/errors.report
cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
cat $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt >> ${{ github.workspace }}/errors.report
echo "Done!"
else
echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
echo "Run of ${base_extless_name} succeeded (set_${{ matrix.sets }})"
fi
# Grab Trace
# Check for individual Success
echo -n "Obtaining Execution Tracer Logs..."
if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json | wc -l) -ne 0 ]
then
echo -n "Copying Tracer Logs..."
# Append to master trace in github workspace
jq '. + inputs' ${{ github.workspace }}/full_tracer.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json) > temp_2.json
mv temp_2.json ${{ github.workspace }}/full_tracer.json
echo "Done!"
else
echo "No trace found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
fi
fi
echo "Completed processing of \"${file}\"!"
echo "\n\n-----------------------------------------------------------------------------------------------\n\n"
done
#run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
- name: Check For Success
run: |
Expand All @@ -319,6 +385,18 @@ jobs:
name: set_${{ matrix.sets }}.report
path: ${{ github.workspace }}/errors.report

- name: Upload summary
uses: actions/upload-artifact@v3
with:
name: set_${{ matrix.sets }}.summary
path: ${{ github.workspace }}/total_exec_summary.json

- name: Upload tracer
uses: actions/upload-artifact@v3
with:
name: set_${{ matrix.sets }}.tracer
path: ${{ github.workspace }}/full_tracer.json

# Report a Problem to Slack
- name: Report Problem to Slack
if: ${{ failure() }}
Expand All @@ -328,24 +406,24 @@ jobs:
uses: mxschmitt/action-tmate@v3
if: ${{ failure() }}

published_results:
runs-on: ubuntu-20.04
needs: published

steps:
- name: Gather Artifacts
uses: actions/download-artifact@v3
with:
path: ${{ github.workspace }}/published-results

- name: Combine .report artifacts into one file
run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt

- name: Post results to slack part 1
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage

- name: Post results to slack part 2
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
# published_results:
# runs-on: ubuntu-20.04
# needs: published
#
# steps:
# - name: Gather Artifacts
# uses: actions/download-artifact@v3
# with:
# path: ${{ github.workspace }}/published-results
#
# - name: Combine .report artifacts into one file
# run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
#
# - name: Post results to slack part 1
# run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
#
# - name: Post results to slack part 2
# run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage


bmdb_results:
Expand All @@ -358,11 +436,26 @@ jobs:
with:
path: ${{ github.workspace }}/bmdb-results

- name: Install JQ
run: sudo apt update && sudo apt install jq -y

- name: Combine .report artifacts into one file
run: cat $(find $GITHUB_WORKSPACE/bmdb-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt

- name: Combine .summary artifacts into one file
run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json

- name: Combine .summary artifacts into one file
run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'full_tracer.json') > $GITHUB_WORKSPACE/tracer.json

- name: Post results to slack part 1
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) BMDB models that didn't pass:" https://slack.com/api/chat.postMessage

- name: Post results to slack part 2
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage

- name: Post results to slack part 3
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F file=$GITHUB_WORKSPACE/summary.json https://slack.com/api/files.upload

- name: Post results to slack part 4
run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F file=$GITHUB_WORKSPACE/tracer.json https://slack.com/api/files.upload
Loading

0 comments on commit 6123a9f

Please sign in to comment.