Skip to content

Commit 47a1e71

Browse files
agitteragitter
authored
Merge pull request #198 from jhiemstrawisc/container-workflow-overhaul
Container workflow overhaul
2 parents 775ebe4 + 14ad177 commit 47a1e71

File tree

4 files changed

+134
-136
lines changed

4 files changed

+134
-136
lines changed

‎.github/workflows/build-and-remove-template.yml

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Build and Remove Docker Image
2+
# A workflow template that handles building+removing SPRAS containers.
3+
4+
# Required inputs are:
5+
# - path: The path to a container's Dockerfile directory, relative to the repo root.
6+
# - container: The registry/container to be built, without a tag.
7+
# - always_build: A boolean indicating whether the action should condition the container build
8+
# on detected changes.
9+
# - context: The optional build context path, relative to the repo root.
10+
on:
11+
workflow_call:
12+
inputs:
13+
path:
14+
required: true
15+
type: string
16+
container:
17+
required: true
18+
type: string
19+
always_build:
20+
required: false
21+
type: boolean
22+
default: false
23+
context:
24+
required: false
25+
type: string
26+
27+
jobs:
28+
build-and-remove-docker-image:
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Checkout repository
32+
uses: actions/checkout@v2
33+
34+
# Given the input path to the directory containing definitions of the container,
35+
# check whether anything at that path has been modified relative to master's head.
36+
# If so, we'll set `env.changed=true` to trigger the build in a later step
37+
- name: Check for changes
38+
if: ${{ inputs.always_build == false }}
39+
id: check_changes
40+
run: |
41+
if git fetch origin master && git diff --name-only origin/master | grep -q "$(basename ${{ inputs.path }})"; then
42+
echo "Changes detected to ${{inputs.path}}."
43+
echo "changed=true" >> $GITHUB_ENV
44+
else
45+
echo "No changes detected to ${{ inputs.path}}"
46+
echo "changed=false" >> $GITHUB_ENV
47+
fi
48+
49+
- name: Toggle container build if 'always_build'
50+
if: ${{ inputs.always_build }}
51+
run: |
52+
echo "Input for job 'always_build' is true -- building regardless of detected changes."
53+
echo "changed=true" >> $GITHUB_ENV
54+
55+
- name: Build Docker image
56+
if: env.changed == 'true'
57+
uses: docker/build-push-action@v6
58+
with:
59+
context: ${{ inputs.context || inputs.path }}/.
60+
file: ${{ inputs.path }}/Dockerfile
61+
tags: ${{ inputs.container }}:latest
62+
# The cache will always use the `latest` tag, as we assume that results in the highest
63+
# amount of reusable container stuff.
64+
cache-from: type=registry,ref=${{ inputs.container }}:latest
65+
push: false
66+
67+
# Clean/remove the container as we go -- we don't actually need it around, and it
68+
# counts against our action's disk quota.
69+
# Here we use `|| true` to prevent the job from failing if the image doesn't exist or
70+
# can't be removed for some reason
71+
- name: Remove Docker image
72+
if: env.changed == 'true'
73+
run: docker rmi ${{ inputs.container }}:latest || true

‎.github/workflows/build-containers.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Build SPRAS Containers
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened]
6+
push:
7+
branches:
8+
- master
9+
10+
jobs:
11+
build-and-remove-omics1:
12+
uses: "./.github/workflows/build-and-remove-template.yml"
13+
with:
14+
path: docker-wrappers/OmicsIntegrator1
15+
container: reedcompbio/omics-integrator-1
16+
build-and-remove-omics2:
17+
uses: "./.github/workflows/build-and-remove-template.yml"
18+
with:
19+
path: docker-wrappers/OmicsIntegrator2
20+
container: reedcompbio/omics-integrator-2
21+
build-and-remove-pathlinker:
22+
uses: "./.github/workflows/build-and-remove-template.yml"
23+
with:
24+
path: docker-wrappers/PathLinker
25+
container: reedcompbio/pathlinker
26+
build-and-remove-meo:
27+
uses: "./.github/workflows/build-and-remove-template.yml"
28+
with:
29+
path: docker-wrappers/MEO
30+
container: reedcompbio/meo
31+
build-and-remove-mincostflow:
32+
uses: "./.github/workflows/build-and-remove-template.yml"
33+
with:
34+
path: docker-wrappers/MinCostFlow
35+
container: reedcompbio/mincostflow
36+
build-and-remove-allpairs:
37+
uses: "./.github/workflows/build-and-remove-template.yml"
38+
with:
39+
path: docker-wrappers/AllPairs
40+
container: reedcompbio/allpairs
41+
build-and-remove-domino:
42+
uses: "./.github/workflows/build-and-remove-template.yml"
43+
with:
44+
path: docker-wrappers/DOMINO
45+
container: reedcompbio/domino
46+
build-and-remove-cytoscape:
47+
uses: "./.github/workflows/build-and-remove-template.yml"
48+
with:
49+
path: docker-wrappers/Cytoscape
50+
container: reedcompbio/py4cytoscape
51+
build-and-remove-spras:
52+
uses: "./.github/workflows/build-and-remove-template.yml"
53+
with:
54+
path: docker-wrappers/SPRAS
55+
container: reedcompbio/spras
56+
# Since any change to the SPRAS codebase would constitute a change to
57+
# the container we produce in this step, build the container regardless
58+
# of detected changes.
59+
always_build: true
60+
context: ./

‎.github/workflows/test-spras.yml

Lines changed: 0 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -61,141 +61,6 @@ jobs:
6161
shell: bash --login {0}
6262
run: snakemake --cores 2 --configfile config/config.yaml --show-failed-logs
6363

64-
# Builds the Docker images
65-
docker:
66-
name: Build Docker images
67-
runs-on: ${{ matrix.os }}
68-
strategy:
69-
matrix:
70-
os: [ubuntu-latest]
71-
steps:
72-
- name: Checkout repository
73-
uses: actions/checkout@v2
74-
# Pull from Docker Hub to use the cache
75-
# https://medium.com/mobileforgood/coding-tips-patterns-for-continuous-integration-with-docker-on-travis-ci-9cedb8348a62
76-
# https://github.com/docker/build-push-action/issues/7
77-
- name: Pull Docker images
78-
run: |
79-
docker pull reedcompbio/omics-integrator-1:latest
80-
docker pull reedcompbio/omics-integrator-2:v2
81-
docker pull reedcompbio/pathlinker:v2
82-
docker pull reedcompbio/meo:latest
83-
docker pull reedcompbio/mincostflow:latest
84-
docker pull reedcompbio/allpairs:v2
85-
docker pull reedcompbio/domino:latest
86-
docker pull reedcompbio/py4cytoscape:v3
87-
docker pull reedcompbio/spras:v0.1.0
88-
- name: Build Omics Integrator 1 Docker image
89-
uses: docker/build-push-action@v1
90-
with:
91-
path: docker-wrappers/OmicsIntegrator1/.
92-
dockerfile: docker-wrappers/OmicsIntegrator1/Dockerfile
93-
repository: reedcompbio/omics-integrator-1
94-
tags: latest
95-
cache_froms: reedcompbio/omics-integrator-1:latest
96-
push: false
97-
- name: Remove Omics Integrator 1 Docker image
98-
# Remove the image to prevent the cache from being used. Here we use
99-
# `|| true` to prevent the job from failing if the image doesn't exist or
100-
# can't be removed for some reason
101-
run: docker rmi reedcompbio/omics-integrator-1:latest || true
102-
103-
- name: Build Omics Integrator 2 Docker image
104-
uses: docker/build-push-action@v1
105-
with:
106-
path: docker-wrappers/OmicsIntegrator2/.
107-
dockerfile: docker-wrappers/OmicsIntegrator2/Dockerfile
108-
repository: reedcompbio/omics-integrator-2
109-
tags: v2
110-
cache_froms: reedcompbio/omics-integrator-2:latest
111-
push: false
112-
- name: Remove Omics Integrator 2 Docker image
113-
run: docker rmi reedcompbio/omics-integrator-2:latest || true
114-
115-
- name: Build PathLinker Docker image
116-
uses: docker/build-push-action@v1
117-
with:
118-
path: docker-wrappers/PathLinker/.
119-
dockerfile: docker-wrappers/PathLinker/Dockerfile
120-
repository: reedcompbio/pathlinker
121-
tags: v2
122-
cache_froms: reedcompbio/pathlinker:latest
123-
push: false
124-
- name: Remove PathLinker Docker image
125-
run: docker rmi reedcompbio/pathlinker:latest || true
126-
127-
- name: Build Maximum Edge Orientation Docker image
128-
uses: docker/build-push-action@v1
129-
with:
130-
path: docker-wrappers/MEO/.
131-
dockerfile: docker-wrappers/MEO/Dockerfile
132-
repository: reedcompbio/meo
133-
tags: latest
134-
cache_froms: reedcompbio/meo:latest
135-
push: false
136-
- name: Remove MEO Docker image
137-
run: docker rmi reedcompbio/meo:latest || true
138-
139-
- name: Build MinCostFlow Docker image
140-
uses: docker/build-push-action@v1
141-
with:
142-
path: docker-wrappers/MinCostFlow/.
143-
dockerfile: docker-wrappers/MinCostFlow/Dockerfile
144-
repository: reedcompbio/mincostflow
145-
tags: latest
146-
cache_froms: reedcompbio/mincostflow:latest
147-
push: false
148-
- name: Remove MinCostFlow Docker image
149-
run: docker rmi reedcompbio/mincostflow:latest || true
150-
151-
- name: Build All Pairs Shortest Paths Docker image
152-
uses: docker/build-push-action@v1
153-
with:
154-
path: docker-wrappers/AllPairs/.
155-
dockerfile: docker-wrappers/AllPairs/Dockerfile
156-
repository: reedcompbio/allpairs
157-
tags: v2
158-
cache_froms: reedcompbio/allpairs:latest
159-
push: false
160-
- name: Remove All Pairs Shortest Paths Docker image
161-
run: docker rmi reedcompbio/allpairs:latest || true
162-
163-
- name: Build DOMINO Docker image
164-
uses: docker/build-push-action@v1
165-
with:
166-
path: docker-wrappers/DOMINO/.
167-
dockerfile: docker-wrappers/DOMINO/Dockerfile
168-
repository: reedcompbio/domino
169-
tags: latest
170-
cache_froms: reedcompbio/domino:latest
171-
push: false
172-
- name: Remove DOMINO Docker image
173-
run: docker rmi reedcompbio/domino:latest || true
174-
175-
- name: Build Cytoscape Docker image
176-
uses: docker/build-push-action@v1
177-
with:
178-
path: docker-wrappers/Cytoscape/.
179-
dockerfile: docker-wrappers/Cytoscape/Dockerfile
180-
repository: reedcompbio/py4cytoscape
181-
tags: v3
182-
cache_froms: reedcompbio/py4cytoscape:v3
183-
push: false
184-
- name: Remove Cytoscape Docker image
185-
run: docker rmi reedcompbio/py4cytoscape:v3 || true
186-
187-
- name: Build SPRAS Docker image
188-
uses: docker/build-push-action@v1
189-
with:
190-
path: .
191-
dockerfile: docker-wrappers/SPRAS/Dockerfile
192-
repository: reedcompbio/spras
193-
tags: v0.2.0
194-
cache_froms: reedcompbio/spras:v0.2.0
195-
push: false
196-
- name: Remove SPRAS Docker image
197-
run: docker rmi reedcompbio/spras:v0.2.0 || true
198-
19964
# Run pre-commit checks on source files
20065
pre-commit:
20166
name: Run pre-commit checks

‎CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ The pull request will be closed so that the `master` branch of the fork stays sy
224224
1. Document the usage of the Docker wrapper and the assumptions made when implementing the wrapper
225225
1. Add example usage for the new algorithm and its parameters to the template config file
226226
1. Write test functions and provide example input data in a new test subdirectory `test/<algorithm>`. Provide example data and algorithm/expected files names to lists or dicts in `test/generate-inputs` and `test/parse-outputs`. Use the full path with the names of the test files.
227-
1. Extend `.github/workflows/test-spras.yml` to pull and build the new Docker image
227+
1. Extend `.github/workflows/build-containers.yml` to pull and build the new Docker image
228228

229229
When adding new algorithms, there are many other considerations that are not relevant with the simple Local Neighborhood example.
230230
Most algorithms require dependencies that need to be installed in the `Dockerfile`.

0 commit comments

Comments
 (0)