Skip to content

Commit 3dc0642

Browse files
jw098jw098
committed
clang query bash script that only runs on changed files
1 parent 54d133b commit 3dc0642

1 file changed

Lines changed: 239 additions & 0 deletions

File tree

‎.github/scripts/clang-query.sh‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
# This script will get the diff between the current branch and the main branch.
5+
# It will then find all files that depend on these diff files, by
6+
# using clang-scan-deps to generate the dependency graph.
7+
# The clang-query is then run on these diffed files (and their dependants).
8+
# One weakness of this is that this diff only compares the current branch and main branch,
9+
# and so is mainly helpful for PRs. But doesn't help when committing directly to main, or
10+
# after merging the PR. This isn't insurmountable, but I felt it was getting too messy.
11+
12+
13+
# to get this script to work in Github CI:
14+
# Update cpp-ci-serial-programs-base.yml:
15+
# - Set fetch-depth to 0 to fetch all commits for the diff. with blob:none so you're not downloading all that data
16+
# with:
17+
# path: 'Arduino-Source'
18+
# submodules: 'recursive'
19+
# fetch-depth: 0
20+
# filter: blob:none
21+
#
22+
# - when installing clang-tools, specify a version. e.g. clang-tools-18
23+
# sudo apt install clang-tools-18 libopencv-dev
24+
#
25+
# - under run clang query: set working directory. run this script
26+
# name: Run clang query
27+
# if: inputs.run-clang-query
28+
# working-directory: ./Arduino-Source
29+
# run : bash ./.github/scripts/clang-query.sh
30+
# - this script should be placed within the folder .github/scripts
31+
#
32+
# other files to consider updating
33+
# .gitattributes: *.sh text eol=lf
34+
35+
36+
# to get this script to work locally in Windows:
37+
# Open Git Bash. cd to root of the repo (Arduino-Source). Run the following command:
38+
# sh .github/scripts/clang-query.sh
39+
40+
41+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
42+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
43+
TMP_DIR="$REPO_ROOT/.ci_tmp"
44+
mkdir -p "$TMP_DIR"
45+
46+
# Define the cleanup function
47+
cleanup() {
48+
echo "Cleaning up temporary files..."
49+
rm -rf "$TMP_DIR"
50+
}
51+
52+
# Register the trap: run cleanup on EXIT, plus common signals like INT (Ctrl+C) or TERM
53+
# trap cleanup EXIT INT TERM
54+
55+
56+
cd "$REPO_ROOT"
57+
58+
# find path to compile_commands.json
59+
if [ -f "$REPO_ROOT/SerialPrograms/bin/compile_commands.json" ]; then
60+
DB_PATH="$REPO_ROOT/SerialPrograms/bin/compile_commands.json"
61+
elif [ -f "$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" ]; then
62+
DB_PATH="$REPO_ROOT/build/RelWithDebInfo/compile_commands.json"
63+
else
64+
echo "Error: compile_commands.json not found!"
65+
exit 1
66+
fi
67+
68+
69+
echo "Generating clang-scan-deps experimental-full > deps.json."
70+
71+
# in ubuntu, the command is clang-scan-deps-18. in Windows, it is clang-scan-deps
72+
SCAN_DEPS=$(command -v clang-scan-deps-18 || command -v clang-scan-deps)
73+
74+
# Safety check: Exit if the tool isn't found
75+
if [ -z "$SCAN_DEPS" ]; then
76+
echo "Error: clang-scan-deps (or version -18) not found in PATH."
77+
exit 1
78+
fi
79+
80+
81+
# filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format
82+
jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json"
83+
84+
# get dependency graph
85+
"$SCAN_DEPS" -compilation-database "$TMP_DIR/compile_commands_filtered.json" -format experimental-full > "$TMP_DIR/deps.json"
86+
87+
# normalize slashes
88+
# sed 's|\\\\|/|g' deps.json > normalized_deps.json
89+
sed -i 's|\\\\|/|g' "$TMP_DIR/deps.json"
90+
91+
# check if deps.json has the expected keys
92+
# because we are relying on clang-scan-deps experimental-full, where the names of the keys can change.
93+
TU_KEY="translation-units"
94+
CMD_KEY="commands"
95+
DEPS="file-deps"
96+
INPUT="input-file"
97+
98+
JQ_SCRIPT=$(cat << 'EOF'
99+
# 1. Access the target object
100+
(.[$TU][0][$CMD][0]) as $target
101+
102+
# 2. Define the required keys
103+
104+
| [$DEPS, $INPUT] as $required
105+
106+
| (
107+
if .[$TU] == null then
108+
"Missing: \($TU). Keys found at top-level: \(keys_unsorted)"
109+
elif .[$TU][0] == null then
110+
"Missing: \($TU)[0]"
111+
elif .[$TU][0].[$CMD] == null then
112+
"Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)"
113+
elif $target == null then
114+
"Missing: \($TU)[0].[$CMD][0]"
115+
elif ($required | all(. as $req | $target | has($req)) | not) then
116+
"Missing: One or more required keys \($required). Found: \($target | keys_unsorted)"
117+
# elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then
118+
# "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)"
119+
else
120+
"All keys \($required) found in \($TU)[0].\($CMD)[0]"
121+
end
122+
) as $result
123+
124+
| if ($result | type == "string" and startswith("Missing:")) then
125+
("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error
126+
else $result end
127+
EOF
128+
)
129+
130+
echo "Checking keys in deps.json."
131+
jq -r "$JQ_SCRIPT" \
132+
--arg TU "$TU_KEY" \
133+
--arg CMD "$CMD_KEY" \
134+
--arg DEPS "$DEPS" \
135+
--arg INPUT "$INPUT" \
136+
"$TMP_DIR/deps.json"
137+
138+
echo "Generating changed_files.txt from git diff."
139+
140+
# git diff with relative paths
141+
git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt"
142+
143+
echo "Generating files_to_query.txt, based on changed_files.txt and deps.json."
144+
145+
# for each line in changed_files_unix.txt, search deps.json to find all their dependants
146+
jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \
147+
--arg TU "$TU_KEY" \
148+
--arg CMD "$CMD_KEY" \
149+
--arg DEPS "$DEPS" \
150+
--arg INPUT "$INPUT" '
151+
# 1. Clean the list of changed files
152+
($mod | split("\n") | map(select(length > 0))) as $changes |
153+
154+
# 2. Access the translation-units array
155+
[ .[$TU][] | .[$CMD][] |
156+
select(
157+
# 3. Check "file-deps" for matches
158+
.[$DEPS][] | . as $dp |
159+
any($changes[]; . as $c | $dp | endswith($c))
160+
) |
161+
# 4. Get the source file path
162+
.[$INPUT]
163+
] | unique[]
164+
' "$TMP_DIR/deps.json" | tr -d '\r' > "$TMP_DIR/files_to_query.txt"
165+
166+
167+
168+
cat << 'EOF' > "$TMP_DIR/query.txt"
169+
set output dump
170+
match invocation(
171+
isExpansionInFileMatching("SerialPrograms/"),
172+
hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))),
173+
hasArgument(0, hasType(asString("std::string")))
174+
)
175+
EOF
176+
177+
echo "Running clang-query."
178+
179+
# files=$(jq -r '.[].file' "$DB_PATH")
180+
DB_DIR=$(dirname "$DB_PATH")
181+
#echo "$files" | xargs --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> output.txt
182+
183+
# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" -- -Wno-unused-command-line-argument >> "$TMP_DIR/output.txt"
184+
185+
# this works
186+
# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
187+
# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
188+
189+
# also works
190+
# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 \
191+
# clang-query -p "$DB_DIR" \
192+
# --extra-arg="-Wno-unused-command-line-argument" \
193+
# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
194+
195+
# also works
196+
# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' | \
197+
# xargs -d '\n' --max-args=150 \
198+
# clang-query -p "$DB_DIR" \
199+
# --extra-arg="-Wno-unused-command-line-argument" \
200+
# --extra-arg="-Wno-unused-function" \
201+
# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
202+
203+
# in ubuntu, the command is clang-query-18. in Windows, it is clang-query
204+
CLANG_QUERY=$(command -v clang-query-18 || command -v clang-query)
205+
206+
if [ -z "$CLANG_QUERY" ]; then
207+
echo "Error: clang-query (or version -18) not found!"
208+
exit 1
209+
fi
210+
211+
ONLY_CHECK_CHANGED_FILES=true
212+
if [ "$ONLY_CHECK_CHANGED_FILES" = "true" ]; then
213+
LIST_FILE="$TMP_DIR/files_to_query.txt"
214+
else # check all files
215+
LIST_FILE="$TMP_DIR/file_list.txt"
216+
jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE"
217+
fi
218+
219+
> "$TMP_DIR/output.txt"
220+
221+
# Run clang-query using the list file
222+
# check if LIST_FILE has any data to analyze
223+
if [ ! -s "$LIST_FILE" ]; then
224+
echo "No files found to analyze. Skipping Clang-Query."
225+
else
226+
xargs -d '\n' -a "$LIST_FILE" --max-args=150 \
227+
"$CLANG_QUERY" -p "$DB_DIR" \
228+
--extra-arg="-Wno-unused-command-line-argument" \
229+
--extra-arg="-Wno-unused-function" \
230+
-f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
231+
fi
232+
233+
234+
235+
cat "$TMP_DIR/output.txt"
236+
if grep --silent "Match #" "$TMP_DIR/output.txt"; then
237+
echo "::error Forbidden std::filesystem::path construction detected!"
238+
exit 1
239+
fi

0 commit comments

Comments
 (0)