Repository navigation
219 lines (195 loc) · 7.77 KB
/
Copy pathbenchmark.yml
File metadata and controls
219 lines (195 loc) · 7.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
name: TUN Throughput Benchmark
on:
pull_request:
branches: ["master"]
paths:
- 'clash-lib/src/proxy/tun/**'
- 'clash-netstack/**'
- 'bench/**'
- '.github/workflows/benchmark.yml'
workflow_dispatch: # Allow manual trigger via GitHub UI
permissions:
contents: read
pull-requests: write # Required to post comments
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
RUST_TOOLCHAIN: "stable"
BENCH_DURATION: "10" # Duration in seconds for each benchmark run
jobs:
benchmark:
name: TUN Throughput Benchmark
runs-on: ubuntu-latest
steps:
- name: Checkout PR branch
uses: actions/checkout@v7
with:
fetch-depth: 0 # Full history for comparison
ref: ${{ needs.check-trigger.outputs.pr_sha || github.event.pull_request.head.sha }}
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: ${{ env.RUST_TOOLCHAIN }}
- name: Cache Rust dependencies
uses: Itsusinn/rust-cache@92e40e4e88556b42f44b1a0c0e417d27f4e30f90
with:
key: benchmark-${{ runner.os }}
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y iperf3 iproute2 cmake libclang-dev
- name: Setup TUN device support
run: |
# Load TUN kernel module
sudo modprobe tun
# Verify TUN device exists
ls -la /dev/net/tun
# Check kernel capabilities
echo "Kernel version:"
uname -a
echo "Network namespaces support:"
sudo ip netns list || echo "No namespaces yet"
# Clean up any leftover network state from previous runs
echo "Cleaning up any leftover network state..."
sudo ip netns del benchns 2>/dev/null || true
sudo ip link del veth0 2>/dev/null || true
sudo ip link del clash-bench 2>/dev/null || true
sudo pkill -9 iperf3 2>/dev/null || true
sudo pkill -9 clash-rs 2>/dev/null || true
echo "Cleanup complete"
- name: Build clash-rs (release mode)
run: |
cargo build --release --bin clash-rs --all-features
echo "CLASH_BINARY=$PWD/target/release/clash-rs" >> $GITHUB_ENV
- name: Install Python dependencies
run: |
python3 -m pip install --upgrade pip
- name: Run benchmark on PR code
id: pr_benchmark
run: |
echo "Starting TUN benchmark..."
sudo python3 bench/run_tun_benchmark.py \
--config bench/tun-benchmark.yaml \
--duration ${{ env.BENCH_DURATION }} \
--output benchmark-pr.json || {
echo "Benchmark failed with exit code: $?"
echo "Checking for partial results..."
if [ -f benchmark-pr.json ]; then
echo "Partial results found:"
cat benchmark-pr.json
fi
if [ -f clash-rs-bench.log ]; then
echo "Clash-rs log excerpt:"
tail -50 clash-rs-bench.log
fi
exit 1
}
- name: Upload PR benchmark results
uses: actions/upload-artifact@v7
if: always()
with:
name: benchmark-pr-results
path: benchmark-pr.json
retention-days: 30
- name: Upload clash-rs logs
uses: actions/upload-artifact@v7
if: always()
with:
name: clash-rs-bench-log
path: clash-rs-bench.log
retention-days: 30
# Check for committed baseline file
- name: Check for baseline file
id: check_baseline
run: |
if [ -f "bench/baseline-tun-benchmark.json" ]; then
echo "baseline_exists=true" >> $GITHUB_OUTPUT
echo "✓ Found committed baseline file"
cat bench/baseline-tun-benchmark.json | head -20
else
echo "baseline_exists=false" >> $GITHUB_OUTPUT
echo "⚠️ No baseline file found - comparison will be skipped"
echo "To create a baseline, run the benchmark on master and commit the result."
fi
- name: Compare results and generate comment
id: compare
if: steps.check_baseline.outputs.baseline_exists == 'true'
run: |
echo "Using committed baseline file"
python3 bench/compare_results.py \
--current benchmark-pr.json \
--baseline bench/baseline-tun-benchmark.json \
--output benchmark-comment.md
continue-on-error: true
- name: Post PR comment
if: github.event_name == 'pull_request'
uses: actions/github-script@v9
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs');
// Read comment from file or generate fallback
let commentBody;
if (fs.existsSync('benchmark-comment.md')) {
commentBody = fs.readFileSync('benchmark-comment.md', 'utf8');
} else if ('${{ steps.check_baseline.outputs.baseline_exists }}' === 'false') {
commentBody = '## ⚠️ TUN Throughput Benchmark Results\n\n';
commentBody += '**No baseline file found** - comparison skipped.\n\n';
commentBody += 'To enable benchmark comparisons:\n';
commentBody += '1. Run the benchmark script on the master branch\n';
commentBody += '2. Commit the generated `bench/baseline-tun-benchmark.json` file\n';
commentBody += '3. Future PRs will compare against this baseline\n\n';
commentBody += '### PR Benchmark\n';
if (fs.existsSync('benchmark-pr.json')) {
const prData = fs.readFileSync('benchmark-pr.json', 'utf8');
commentBody += '```json\n' + prData + '\n```\n';
} else {
commentBody += '❌ PR benchmark failed - check workflow logs\n';
}
} else {
commentBody = '⚠️ Benchmark comparison failed - check workflow logs for details.';
}
// Find existing benchmark comment
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const botComment = comments.find(c =>
c.user.type === 'Bot' &&
c.body.includes('TUN Throughput Benchmark Results')
);
if (botComment) {
// Update existing comment
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: botComment.id,
body: commentBody
});
console.log('Updated existing benchmark comment');
} else {
// Create new comment
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: commentBody
});
console.log('Created new benchmark comment');
}
- name: Upload benchmark artifacts
uses: actions/upload-artifact@v7
if: always()
with:
name: benchmark-artifacts
path: |
benchmark-*.json
benchmark-comment.md
clash-rs-bench.log
retention-days: 30
- name: Note on performance comparison
if: steps.compare.outcome == 'failure'
run: |
echo "::notice::Performance comparison detected a change (>10%) — see PR comment for details"