Skip to content

Commit e146d9c

Browse files
Aggarwal-RaghavAggarwal-Raghav
committed
TEZ-4718: Modernize Jenkins and Yetus integration to ensure full build and UT run on every PR
1 parent 383ffd9 commit e146d9c

4 files changed

Lines changed: 205 additions & 1114 deletions

File tree

‎Jenkinsfile‎

Lines changed: 124 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -23,188 +23,164 @@ pipeline {
2323

2424
options {
2525
buildDiscarder(logRotator(numToKeepStr: '5'))
26-
timeout (time: 20, unit: 'HOURS')
26+
timeout(time: 8, unit: 'HOURS')
2727
timestamps()
2828
checkoutToSubdirectory('src')
29+
// Ensure only one build runs per PR at a time
30+
disableConcurrentBuilds(abortPrevious: true)
31+
}
32+
33+
parameters {
34+
string(name: 'JIRA_ISSUE_KEY',
35+
defaultValue: '',
36+
description: 'The JIRA issue to comment on. Example: TEZ-1234')
2937
}
3038

3139
environment {
3240
SOURCEDIR = 'src'
33-
// will also need to change notification section below
41+
YETUSDIR = 'yetus'
3442
PATCHDIR = 'out'
3543
DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile"
36-
YETUS='yetus'
37-
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
38-
YETUS_VERSION='rel/0.15.1'
3944

40-
}
45+
// Credentials IDs
46+
GITHUB_CRED_ID = 'apache-tez-at-github.com'
47+
JIRA_CRED_ID = 'tez-ci'
4148

42-
parameters {
43-
string(name: 'JIRA_ISSUE_KEY',
44-
defaultValue: '',
45-
description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
49+
// Yetus version
50+
YETUS_VERSION = 'rel/0.15.1'
4651
}
4752

4853
stages {
49-
stage ('install yetus') {
54+
stage('Install Yetus') {
5055
steps {
51-
dir("${WORKSPACE}/${YETUS}") {
56+
dir("${WORKSPACE}/${env.YETUSDIR}") {
5257
checkout([
5358
$class: 'GitSCM',
5459
branches: [[name: "${env.YETUS_VERSION}"]],
55-
userRemoteConfigs: [[ url: 'https://github.com/apache/yetus']]]
60+
userRemoteConfigs: [[url: 'https://github.com/apache/yetus']]]
5661
)
5762
}
5863
}
5964
}
6065

61-
stage ('precommit-run') {
66+
stage('Yetus Analysis') {
6267
steps {
63-
withCredentials(
64-
[usernamePassword(credentialsId: 'apache-tez-at-github.com',
65-
passwordVariable: 'GITHUB_TOKEN',
66-
usernameVariable: 'GITHUB_USER'),
67-
usernamePassword(credentialsId: 'tez-ci',
68-
passwordVariable: 'JIRA_PASSWORD',
69-
usernameVariable: 'JIRA_USER')]) {
70-
sh '''#!/usr/bin/env bash
71-
72-
set -e
73-
74-
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
75-
76-
# this must be clean for every run
77-
if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
78-
rm -rf "${WORKSPACE}/${PATCHDIR}"
79-
fi
80-
mkdir -p "${WORKSPACE}/${PATCHDIR}"
81-
82-
# if given a JIRA issue, process it. If CHANGE_URL is set
83-
# (e.g., Github Branch Source plugin), process it.
84-
# otherwise exit, because we don't want Hadoop to do a
85-
# full build. We wouldn't normally do this check for smaller
86-
# projects. :)
87-
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
88-
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
89-
elif [[ -z "${CHANGE_URL}" ]]; then
90-
echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
91-
exit 0
92-
fi
93-
94-
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
95-
96-
# where the source is located
97-
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
98-
99-
# our project defaults come from a personality file
100-
YETUS_ARGS+=("--project=tez")
101-
102-
# lots of different output formats
103-
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
104-
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
105-
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
106-
107-
# enable writing back to Github
108-
YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
109-
110-
# auto-kill any surefire stragglers during unit test runs
111-
YETUS_ARGS+=("--reapermode=kill")
112-
113-
# set relatively high limits for ASF machines
114-
# changing these to higher values may cause problems
115-
# with other jobs on systemd-enabled machines
116-
YETUS_ARGS+=("--proclimit=5500")
117-
YETUS_ARGS+=("--dockermemlimit=20g")
118-
119-
# -1 spotbugs issues that show up prior to the patch being applied
120-
# YETUS_ARGS+=("--spotbugs-strict-precheck")
121-
# rsync these files back into the archive dir
122-
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
123-
124-
# URL for user-side presentation in reports and such to our artifacts
125-
# (needs to match the archive bits below)
126-
YETUS_ARGS+=("--build-url-artifacts=artifact/out")
127-
128-
# plugins to enable
129-
YETUS_ARGS+=("--plugins=all")
130-
131-
# use Hadoop's bundled shelldocs
132-
YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
133-
134-
# don't let these tests cause -1s because we aren't really paying that
135-
# much attention to them
136-
YETUS_ARGS+=("--tests-filter=checkstyle")
137-
138-
# run in docker mode and specifically point to our
139-
# Dockerfile since we don't want to use the auto-pulled version.
140-
YETUS_ARGS+=("--docker")
141-
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
142-
YETUS_ARGS+=("--mvn-custom-repos")
143-
144-
# effectively treat dev-suport as a custom maven module
145-
YETUS_ARGS+=("--skip-dirs=dev-support")
146-
147-
# help keep the ASF boxes clean
148-
YETUS_ARGS+=("--sentinel")
149-
150-
# test with Java 21 eclipse-temurin docker image path
151-
YETUS_ARGS+=("--java-home=/opt/java/openjdk")
152-
YETUS_ARGS+=("--debug")
153-
154-
# write Yetus report as GitHub comment (YETUS-1102)
155-
YETUS_ARGS+=("--github-write-comment")
156-
YETUS_ARGS+=("--github-use-emoji-vote")
157-
158-
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
159-
'''
68+
withCredentials([
69+
usernamePassword(credentialsId: env.GITHUB_CRED_ID, passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER'),
70+
usernamePassword(credentialsId: env.JIRA_CRED_ID, passwordVariable: 'JIRA_PASSWORD', usernameVariable: 'JIRA_USER')
71+
]) {
72+
sh '''#!/usr/bin/env bash
73+
set -e
74+
75+
TESTPATCHBIN="${WORKSPACE}/${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
76+
77+
rm -rf "${WORKSPACE}/${PATCHDIR}"
78+
mkdir -p "${WORKSPACE}/${PATCHDIR}"
79+
80+
cd "${WORKSPACE}/${SOURCEDIR}"
81+
# Ensure origin/master is available for diffing
82+
git fetch origin master || true
83+
git diff origin/master...HEAD > "${WORKSPACE}/${PATCHDIR}/local-pr.patch"
84+
cd "${WORKSPACE}"
85+
86+
YETUS_ARGS=()
87+
88+
# 1. Target Issue/PR
89+
# Pass the Jira issue or use the local patch for GitHub PRs
90+
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
91+
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
92+
elif [[ -n "${CHANGE_ID}" ]]; then
93+
echo "Processing PR: ${CHANGE_ID} using local patch"
94+
else
95+
echo "No PR or JIRA key provided. Skipping Yetus."
96+
exit 0
97+
fi
98+
99+
# 2. Core Paths & Project
100+
# Set the base directory, patch output directory, and project name
101+
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
102+
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
103+
YETUS_ARGS+=("--project=tez")
104+
105+
# 3. Docker & Environment
106+
# Run tests inside Docker, allowing dirty workspace (since Jenkins handles the checkout)
107+
YETUS_ARGS+=("--docker")
108+
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
109+
YETUS_ARGS+=("--java-home=/opt/java/openjdk")
110+
YETUS_ARGS+=("--dirty-workspace")
111+
YETUS_ARGS+=("--build-url-artifacts=artifact/out")
112+
113+
# 4. Plugins & Personality
114+
# Load all plugins eagerly and provide the custom Tez personality file.
115+
# Note: We copy the personality to PATCHDIR because Yetus' robot mode cleans untracked files in the workspace.
116+
cp "${WORKSPACE}/${SOURCEDIR}/dev-support/tez-personality.sh" "${WORKSPACE}/${PATCHDIR}/tez-personality.sh"
117+
YETUS_ARGS+=("--plugins=all")
118+
YETUS_ARGS+=("--personality=${WORKSPACE}/${PATCHDIR}/tez-personality.sh")
119+
YETUS_ARGS+=("--mvn-custom-repos")
120+
YETUS_ARGS+=("--skip-dirs=dev-support")
121+
YETUS_ARGS+=("--tests-filter=checkstyle")
122+
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
123+
YETUS_ARGS+=("--reapermode=kill")
124+
YETUS_ARGS+=("--sentinel")
125+
126+
# 5. Reporting & GitHub Integration
127+
# Configure report output files and pass the GitHub token for PR commenting
128+
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
129+
YETUS_ARGS+=("--github-write-comment")
130+
YETUS_ARGS+=("--github-use-emoji-vote")
131+
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
132+
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
133+
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
134+
135+
echo "Running Yetus with local patch overriding GitHub..."
136+
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${WORKSPACE}/${PATCHDIR}/local-pr.patch"
137+
'''
160138
}
161139
}
162140
}
163-
164141
}
165142

166143
post {
167144
always {
168-
script {
169-
// Yetus output
170-
archiveArtifacts "${env.PATCHDIR}/**"
171-
// Publish the HTML report so that it can be looked at
172-
// Has to be relative to WORKSPACE.
173-
publishHTML (target: [
174-
allowMissing: true,
175-
keepAll: true,
176-
alwaysLinkToLastBuild: true,
177-
// Has to be relative to WORKSPACE
178-
reportDir: "${env.PATCHDIR}",
179-
reportFiles: 'report.html',
180-
reportName: 'Yetus Report'
181-
])
182-
// Publish JUnit results
183-
try {
184-
junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
185-
} catch(e) {
186-
echo 'junit processing: ' + e.toString()
145+
script {
146+
sh '''
147+
# Fix permissions before attempting to read/zip directories created by root Docker processes
148+
echo "Cleaning up workspace permissions to prevent HADOOP-13951 and unzipping errors..."
149+
chmod -R u+rxw "${WORKSPACE}" || true
150+
151+
# Zip test logs for easier debugging and storage efficiency
152+
cd "${SOURCEDIR}" || exit 0
153+
find . -type d -name "surefire-reports" | while read -r dir; do
154+
rel_path=$(echo "${dir#./}" | tr '/' '_')
155+
zip -r -q "../${PATCHDIR}/test-logs-${rel_path}.zip" "${dir}" || true
156+
done
157+
'''
158+
159+
dir(env.SOURCEDIR) {
160+
// Safely publish junit results
161+
junit testResults: "**/target/surefire-reports/*.xml", allowEmptyResults: true
162+
}
163+
164+
// Archive Yetus output, reports, and zipped test logs
165+
archiveArtifacts artifacts: "${env.PATCHDIR}/**", allowEmptyArchive: true
166+
167+
// Publish Yetus HTML report to Jenkins UI sidebar only if it exists
168+
if (fileExists("${env.PATCHDIR}/report.html")) {
169+
publishHTML(target: [
170+
allowMissing: true,
171+
alwaysLinkToLastBuild: true,
172+
keepAll: true,
173+
reportDir: "${env.PATCHDIR}",
174+
reportFiles: 'report.html',
175+
reportName: 'Yetus Report'
176+
])
177+
}
187178
}
188-
}
189179
}
190180

191-
// Jenkins pipeline jobs fill slaves on PRs without this :(
192-
cleanup() {
181+
cleanup {
193182
script {
194-
sh '''
195-
# See YETUS-764
196-
if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
197-
echo "test-patch process appears to still be running: killing"
198-
kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
199-
sleep 10
200-
fi
201-
if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
202-
echo "test-patch container appears to still be running: killing"
203-
docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
204-
fi
205-
# See HADOOP-13951
206-
chmod -R u+rxw "${WORKSPACE}"
207-
'''
183+
// Wipe the workspace to prevent Jenkins agent disk exhaustion
208184
deleteDir()
209185
}
210186
}

0 commit comments

Comments
 (0)