Skip to content

Commit 90c4379

Browse files
jack1902jack1902
committed
[core] Move git logic and upload to agent bash script
* Moved the git logic, plugin disabled and pipeline upload to `command` script. This removes the need to check for git_mirrors and other agent specific setups. * Updated the testing for the above * Removed the documentation around git_mirrors * Updated the docs to reflect that git_diff can be a comma-seperated string
1 parent 49682ee commit 90c4379

File tree

9 files changed

+156
-268
lines changed

9 files changed

+156
-268
lines changed

‎Dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
FROM python:3.7-slim
22

33
RUN apt-get update && \
4-
apt-get upgrade -y && \
5-
apt-get install -y git
4+
apt-get upgrade -y
65

76
WORKDIR "/buildkite"
87

‎README.md

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,16 @@ The above example `initial_pipeline` will skip the `build and deploy lambda` ste
5353

5454
## Configuration
5555

56-
| Option | Required | Type | Default | Description |
57-
| ---------------- | :------: | :-------: | :-----: | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
58-
| dynamic_pipeline | Yes | `string` | | The name including the path to the pipeline that contains all the actual `steps` |
59-
| git_mirrors_path | No | `string` | | The Path that the agent uses for `git-mirror-path`, This will be mounted into the container. See [git mirrors](https://github.com/buildkite/agent/blob/master/EXPERIMENTS.md) |
60-
| disable_plugin | No | `boolean` | `false` | This can be used to pass the entire `dynamic_pipeline` pipeline straight to buildkite without skipping a single step. |
61-
| diff | No | `string` | | Can be used to override the default commands (see below for a better explanation of the defaults) |
62-
| log_level | No | `string` | `INFO` | The Level of logging to be used by the python script underneath. Pass `DEBUG` for verbose logging if errors occur |
63-
| steps | Yes | `array` | | Each Step should contain a `label` with the `include`/`exclude` settings relevant to the label it applies to within the `dynamic_pipeline` file |
64-
| label | Yes | `string` | | The `label` these conditions apply to within the `dynamic_pipeline` file. (These should be an EXACT match) |
65-
| include | No | `array` | | If any element is found within the `git diff` then this step will NOT be skipped |
66-
| exclude | No | `array` | | If any alement is found within the `git diff` then this step will be SKIPPED |
56+
| Option | Required | Type | Default | Description |
57+
| ---------------- | :------: | :-------: | :-----: | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
58+
| dynamic_pipeline | Yes | `string` | | The name including the path to the pipeline that contains all the actual `steps` |
59+
| disable_plugin | No | `boolean` | `false` | This can be used to pass the entire `dynamic_pipeline` pipeline straight to buildkite without skipping a single step. |
60+
| diff | No | `string` | | Can be used to override the default commands (see below for a better explanation of the defaults) Pass a comma-seperated string of git diff commands if you want multiple custom git diff commands run |
61+
| log_level | No | `string` | `INFO` | The Level of logging to be used by the python script underneath. Pass `DEBUG` for verbose logging if errors occur |
62+
| steps | Yes | `array` | | Each Step should contain a `label` with the `include`/`exclude` settings relevant to the label it applies to within the `dynamic_pipeline` file |
63+
| label | Yes | `string` | | The `label` these conditions apply to within the `dynamic_pipeline` file. (These should be an EXACT match) |
64+
| include | No | `array` | | If any element is found within the `git diff` then this step will NOT be skipped |
65+
| exclude | No | `array` | | If any alement is found within the `git diff` then this step will be SKIPPED |
6766

6867
Other useful things to note:
6968
- Both `include` and `exclude` make use of Unix shell-style wildcards (Look at `.gitignore` files for inspiration)

‎hooks/command

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,53 @@
11
#!/bin/bash
22
set -euo pipefail
33

4+
set +u
5+
# Check if the plugin is disabled
6+
if [[ ! -z "$BUILDKITE_PLUGIN_GIT_DIFF_CONDITIONAL_DISABLE_PLUGIN" ]]; then
7+
echo "Plugin disable flag detected, passing entire pipeline to buildkite"
8+
buildkite-agent pipeline upload $BUILDKITE_PLUGIN_GIT_DIFF_CONDITIONAL_DYNAMIC_PIPELINE
9+
exit 0
10+
fi
11+
set -u
12+
13+
f_get_diff() {
14+
local default_diff_commands
15+
local diff_commands
16+
17+
default_diff_commands="git diff --name-only origin/master...HEAD,git diff --name-only HEAD HEAD~1"
18+
19+
IFS=','
20+
read -a diff_commands <<< "${BUILDKITE_PLUGIN_GIT_DIFF_CONDITIONAL_DIFF:-$default_diff_commands}"
21+
22+
for diff_command in ${diff_commands[@]}; do
23+
echo >&2 "Checking for diff using: ($diff_command)"
24+
25+
diff=$(eval $diff_command)
26+
if [[ ! -z "$diff" ]]; then
27+
echo >&2 "Found diff using command ($diff_command)"
28+
break
29+
fi
30+
done
31+
32+
echo "$diff"
33+
}
34+
435
basedir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
536

37+
diff=$(f_get_diff)
38+
if [[ -z "$diff" ]]; then
39+
echo "No diff detected"
40+
exit 0
41+
else
42+
mkdir -p .git_diff_conditional
43+
echo $diff > .git_diff_conditional/git_diff
44+
fi
45+
646
docker build $basedir -t buildkite-pyyaml > /dev/null
747

8-
set +u
9-
MIRROS_PATH=${BUILDKITE_PLUGIN_GIT_DIFF_CONDITIONAL_GIT_MIRRORS_PATH}
10-
set -u
48+
docker run --rm -v "$PWD:/buildkite" --env-file <(env | grep BUILDKITE) buildkite-pyyaml
1149

12-
if [[ -z "${MIRROS_PATH}" ]]; then
13-
docker run --rm -v "$PWD:/buildkite" --env-file <(env | grep BUILDKITE) buildkite-pyyaml
14-
else
15-
docker run --rm -v "$PWD:/buildkite" -v "${MIRROS_PATH}":"${MIRROS_PATH}" --env-file <(env | grep BUILDKITE) buildkite-pyyaml
50+
if [[ -s ".git_diff_conditional/pipeline_output" ]]; then
51+
echo "Uploading the pipeline to the buildkite-agent"
52+
buildkite-agent pipeline upload .git_diff_conditional/pipeline_output
1653
fi

‎hooks/post-command

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
echo "Cleaning up plugin-cache"
5+
6+
rm -rf .git_diff_conditional

‎plugin.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ requirements:
55
- docker
66
configuration:
77
properties:
8-
git_mirrors_path:
9-
type: string
108
dynamic_pipeline:
119
type: string
1210
disable_plugin:

‎scripts/generate_pipeline.py

Lines changed: 16 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -148,56 +148,14 @@ def check_if_skip(conditional_steps: dict, step: dict) -> bool:
148148
return conditional_steps[label]
149149

150150

151-
def run_command(diff_command: str) -> list:
152-
"""Get the `git diff` based on given command
153-
154-
Args:
155-
diff_command (str): The git command to use to get the diff
156-
157-
Returns:
158-
list: Contains all the files changed according to git
159-
"""
151+
def get_diff():
160152
try:
161-
result = subprocess.run(
162-
diff_command, check=True, stdout=subprocess.PIPE, shell=True
163-
).stdout.decode("utf-8")
164-
except subprocess.CalledProcessError as e:
165-
LOG.debug(e)
166-
log_and_exit("error", f"Error getting diff using command: {diff_command}", 1)
153+
with open(".git_diff_conditional/git_diff", "r") as _fp:
154+
diff = [_file.strip() for _file in _fp.readlines() if _file.strip() != ""]
155+
except FileNotFoundError:
156+
log_and_exit("error", "Error getting diff from file", 1)
167157
else:
168-
return [_file for _file in result.replace(" ", "").split("\n") if _file != ""]
169-
170-
171-
def get_diff(plugin_prefix):
172-
default_diff_commands = [
173-
"git diff --name-only origin/master...HEAD",
174-
"git diff --name-only HEAD HEAD~1",
175-
]
176-
diff_command = os.getenv(f"{plugin_prefix}_DIFF", default_diff_commands)
177-
178-
diff = None
179-
if isinstance(diff_command, list):
180-
for command in diff_command:
181-
diff = run_command(command)
182-
if diff:
183-
break
184-
else:
185-
command = diff_command
186-
diff = run_command(command)
187-
188-
LOG.info("Got diff using command (%s)", command)
189-
190-
return diff
191-
192-
193-
def upload_pipeline(pipeline):
194-
out = yaml.dump(pipeline, default_flow_style=False)
195-
196-
try:
197-
subprocess.run([f'echo "{out}" | buildkite-agent pipeline upload'], shell=True)
198-
except subprocess.CalledProcessError as e:
199-
LOG.debug(e)
200-
log_and_exit("error", "Error uploading pipeline", 1)
158+
return diff
201159

202160

203161
def handler():
@@ -208,31 +166,32 @@ def handler():
208166
LOG.setLevel(log_level)
209167

210168
# Get the git diff
211-
diff = get_diff(plugin_prefix)
169+
diff = get_diff()
212170

213171
# Instantiate the Class
214172
git_diff_conditions = GitDiffConditional(diff, plugin_prefix)
215173

216174
# Get the dynamic_pipeline
217175
dynamic_pipeline = git_diff_conditions.load_dynamic_pipeline("DYNAMIC_PIPELINE")
218176

219-
if os.getenv(f"{plugin_prefix}_DISABLE_PLUGIN"):
220-
LOG.warning(
221-
"Plugin disable flag detected, passing entire pipeline to buildkite"
222-
)
223-
return upload_pipeline(dynamic_pipeline)
224-
225177
# Get the conditions
226178
conditions = git_diff_conditions.load_conditions_from_environment()
227179

228180
# Generate the pipeline
229181
pipeline = git_diff_conditions.generate_pipeline_from_conditions(
230182
dynamic_pipeline, conditions
231183
)
232-
if len(pipeline["steps"]) == 0:
184+
185+
if not pipeline["steps"]:
233186
log_and_exit("info", f"No pipeline generated for diff: ({diff})", 0)
187+
else:
188+
LOG.info("Dynamic pipeline generated, saving for agent upload")
234189

235-
upload_pipeline(pipeline)
190+
try:
191+
with open(".git_diff_conditional/pipeline_output", "w") as _fp:
192+
yaml.dump(pipeline, _fp, default_flow_style=False)
193+
except Exception:
194+
log_and_exit("error", "error saving pipeline to disk", 1)
236195

237196

238197
if __name__ == "__main__":

‎tests/unit/test_get_diff.py

Lines changed: 14 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -3,145 +3,41 @@
33
import pytest
44

55
from CONSTANTS import PLUGIN_PREFIX
6-
from scripts.generate_pipeline import get_diff, run_command
6+
from scripts.generate_pipeline import get_diff
77

88
#
9-
# function run_command tests
9+
# function get_diff tests
1010
#
1111

1212

1313
@pytest.mark.parametrize(
14-
"subprocess_return_value,expected_result",
14+
"file_contents,expected_result",
1515
[
1616
(
1717
"""test.py
1818
folder_a/test.tf
1919
folder_a/folder_b/test.txt""",
2020
["test.py", "folder_a/test.tf", "folder_a/folder_b/test.txt"],
21-
),
22-
("", []),
21+
), # Diff present
22+
("\n", []), # No Diff
2323
],
2424
)
25-
def test_run_command(mocker, subprocess_return_value, expected_result):
26-
"""
27-
Checks that the expected_result is returned given an expected input
28-
"""
29-
30-
subprocess_mock = mocker.patch("scripts.generate_pipeline.subprocess.run")
31-
32-
test_command = "git diff"
33-
subprocess_mock.return_value = mocker.Mock(
34-
stdout=bytes(subprocess_return_value, "UTF-8")
25+
def test_get_diff(mocker, file_contents, expected_result):
26+
open_mock = mocker.patch(
27+
"scripts.generate_pipeline.open", mocker.mock_open(read_data=file_contents)
3528
)
3629

37-
result = run_command(test_command)
38-
39-
# Tests
30+
result = get_diff()
4031
assert result == expected_result
41-
subprocess_mock.assert_called_once_with(
42-
test_command, check=True, stdout=-1, shell=True
43-
)
4432

4533

46-
def test_run_command_raises_error(mocker, logger, log_and_exit_mock):
47-
test_command = "Error"
48-
exit_code = 1
49-
subprocess_mock = mocker.patch(
50-
"scripts.generate_pipeline.subprocess.run",
51-
side_effect=subprocess.CalledProcessError(exit_code, test_command),
34+
def test_get_diff_no_file(mocker, log_and_exit_mock):
35+
open_mock = mocker.patch(
36+
"scripts.generate_pipeline.open", side_effect=FileNotFoundError
5237
)
5338

54-
result = run_command(test_command)
55-
56-
# Tests
39+
result = get_diff()
5740
assert result is None
58-
assert logger.record_tuples == [
59-
("cli", 10, f"Command '{test_command}' returned non-zero exit status 1."),
60-
]
61-
subprocess_mock.assert_called_once_with(
62-
test_command, check=True, stdout=subprocess.PIPE, shell=True
63-
)
6441
log_and_exit_mock.assert_called_once_with(
65-
"error", f"Error getting diff using command: {test_command}", exit_code
66-
)
67-
68-
69-
#
70-
# function get_diff tests
71-
#
72-
73-
74-
@pytest.mark.parametrize(
75-
"command_to_return_on,expected_calls",
76-
[
77-
(
78-
"git diff --name-only origin/master...HEAD", # Test feature branch
79-
["git diff --name-only origin/master...HEAD"],
80-
),
81-
(
82-
"git diff --name-only HEAD HEAD~1", # Test master against master - 1 commmit
83-
[
84-
"git diff --name-only origin/master...HEAD",
85-
"git diff --name-only HEAD HEAD~1",
86-
],
87-
),
88-
],
89-
)
90-
def test_get_diff(mocker, command_to_return_on, expected_calls):
91-
return_value = ["test.py", "folder_a/test.py"]
92-
93-
def side_effect(command):
94-
if command == command_to_return_on:
95-
result = return_value
96-
else:
97-
result = []
98-
99-
return result
100-
101-
run_command_mock = mocker.patch(
102-
"scripts.generate_pipeline.run_command", side_effect=side_effect
103-
)
104-
105-
result = get_diff(PLUGIN_PREFIX)
106-
107-
# Tests
108-
run_command_mock.assert_has_calls(
109-
[mocker.call(call) for call in expected_calls], any_order=False,
110-
)
111-
assert result == return_value
112-
113-
114-
def test_get_diff_custom(monkeypatch, mocker):
115-
"Test that run_command is called with the passed custom git diff command"
116-
117-
custom_diff_command = "custom diff command"
118-
monkeypatch.setenv(f"{PLUGIN_PREFIX}_DIFF", custom_diff_command)
119-
run_command_mock = mocker.patch(
120-
"scripts.generate_pipeline.run_command", return_value=["diff.py"]
121-
)
122-
123-
result = get_diff(PLUGIN_PREFIX)
124-
125-
# Tests
126-
run_command_mock.assert_called_once_with(custom_diff_command)
127-
assert result == run_command_mock.return_value
128-
129-
130-
def test_get_diff_no_diff(mocker):
131-
"Test that run_command is called twice with both default commands"
132-
133-
run_command_mock = mocker.patch(
134-
"scripts.generate_pipeline.run_command", return_value=[]
135-
)
136-
137-
result = get_diff(PLUGIN_PREFIX)
138-
139-
# Tests
140-
run_command_mock.assert_has_calls(
141-
[
142-
mocker.call("git diff --name-only origin/master...HEAD"),
143-
mocker.call("git diff --name-only HEAD HEAD~1"),
144-
],
145-
any_order=False,
42+
"error", "Error getting diff from file", 1
14643
)
147-
assert result == run_command_mock.return_value

0 commit comments

Comments
 (0)