Skip to content
Open
240 changes: 240 additions & 0 deletions .github/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [[ -z "$GITHUB_STEP_SUMMARY" ]]; then
echo "run_tests.sh is for use by CI (selected tests, timeout)."
echo "Users should go to opt/cachelib/tests and run make."
echo
fi

# Optional (e.g., flaky) tests. Issues a warning instead of an error.
OPTIONAL=()
OPTIONAL+=("allocator-test-AllocationClassTest") # Ubuntu 18 (segfault)
OPTIONAL+=("allocator-test-AllocatorResizeTypeTest") # Rocky 8.6
OPTIONAL+=("allocator-test-AllocatorTypeTest") # CentOS 8.5, Rocky 8.6
OPTIONAL+=("allocator-test-BlockCacheTest") # Rocky 8.6
OPTIONAL+=("allocator-test-MemoryAllocatorTest") # Ubuntu 18 (segfault)
OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault)
# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18
OPTIONAL+=("allocator-test-NavySetupTest")
OPTIONAL+=("allocator-test-NvmCacheTests") # Rocky 8.6
OPTIONAL+=("allocator-test-RefCountTest") # Ubuntu 18
OPTIONAL+=("allocator-test-SlabAllocatorTest") # Ubuntu 18
# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18
OPTIONAL+=("common-test-UtilTests")
OPTIONAL+=("datatype-test-MapTest") # Ubuntu 20
# CentOS 8.1, Rocky 9, Ubuntu 18
OPTIONAL+=("navy-test-BlockCacheTest")
# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Ubuntu 18, Ubuntu 20, Ubuntu 22
OPTIONAL+=("navy-test-DriverTest")
# CentOS 8.5, Rocky 9, Ubuntu 20
OPTIONAL+=("navy-test-MockJobSchedulerTest")
# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18, Ubuntu 20, Ubuntu 22
# Large pages need to be enabled
OPTIONAL+=("shm-test-test_page_size")

# Skip long-running benchmarks.
TO_SKIP=()
# TO_SKIP+=("allocator-test-AllocatorTypeTest") # 12 mins.
TO_SKIP+=("benchmark-test-CompactCacheBench") # 26 mins.
TO_SKIP+=("benchmark-test-MutexBench") # 60 mins.

TEST_TIMEOUT=30m
BENCHMARK_TIMEOUT=20m
PARALLELISM=10

print_test_log() {
logfile=$1
# Print last failed test
EXP_LOG=$(grep -Pazo \
"(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \
| sed 's/\x0/---------------\n/g')
# And contents of last test before core dumps
if grep -q -R "core dumped" $logfile; then
EXP_LOG+=$'\n'
EXP_LOG+=$(tac $logfile | sed '/\[ RUN \]/q' | tac)
fi
echo "::group::Logs:$logfile"
echo "$EXP_LOG"
echo
echo "::endgroup::"
echo

echo "#### $logfile" >> $MD_OUT
echo "\`\`\`" >> $MD_OUT
echo "$EXP_LOG" >> $MD_OUT
echo "\`\`\`" >> $MD_OUT
echo >> $MD_OUT
}

OPTIONAL_LIST=$(printf -- '%s\n' ${OPTIONAL[@]})
TO_SKIP_LIST=$(printf -- '%s\n' ${TO_SKIP[@]})

MD_OUT=${GITHUB_STEP_SUMMARY:-$PWD/summary.md}
if [[ "$MD_OUT" != "$GITHUB_STEP_SUMMARY" ]]; then
echo "Markdown summary will be saved in $MD_OUT. Truncating it."
echo
echo "Time started: $(date)" > $MD_OUT
fi

echo "See Summary page of job for a table of test results and log excerpts."
echo

dir=$(dirname "$0")
cd "$dir/.." || die "failed to change-dir into $dir/.."
test -d cachelib || die "failed to change-dir to expected root directory"

PREFIX="$PWD/opt/cachelib"
LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}"
export LD_LIBRARY_PATH

echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"

cd opt/cachelib/tests || die "failed to change-dir into opt/cachelib/tests"

TESTS_TO_RUN=$(find * -type f -not -name "*bench*" -executable \
| grep -vF "$TO_SKIP_LIST" \
| awk ' { print $1 ".log" } ')
N_TESTS=$(echo $TESTS_TO_RUN | wc -w)

if [[ $(< /proc/sys/vm/nr_hugepages) == "0" ]]; then
# GitHub's runners have 7GB of RAM (as of 2023)
echo
echo "Trying to allocate a 1GB huge page pool for shm-test-test_page_size"
sudo sysctl -w vm.nr_hugepages=512
fi

echo
echo "::group::Running tests for CI (total: $N_TESTS, max: $TEST_TIMEOUT)"
timeout --preserve-status $TEST_TIMEOUT make -j $PARALLELISM -s $TESTS_TO_RUN
echo "::endgroup::"
echo "Successful tests: $(find -name '*.ok' | wc -l)"
echo "Failed tests: $(find -name '*.fail' | wc -l)"
echo

BENCHMARKS_TO_RUN=$(find * -type f -name "*bench*" -executable \
| grep -vF "$TO_SKIP_LIST" \
| awk ' { print $1 ".log" } ')
N_BENCHMARKS=$(echo $BENCHMARKS_TO_RUN | wc -w)

echo "::group::Running benchmarks for CI (total: $N_BENCHMARKS, max: $BENCHMARK_TIMEOUT)"
timeout --preserve-status $BENCHMARK_TIMEOUT make -j $PARALLELISM -s $BENCHMARKS_TO_RUN
echo "::endgroup::"
echo "Successful benchmarks: $(find -name '*bench*.ok' | wc -l)"
echo "Failed benchmarks: $(find -name '*bench*.fail' | wc -l)"

TESTS_PASSED=$(find * -name '*.log.ok' | sed 's/\.log\.ok$//')
TESTS_FAILED=$(find * -name '*.log.fail' | sed 's/\.log\.fail$//')
TESTS_TIMEOUT=$(find * -type f -executable \
| grep -vF "$TESTS_PASSED" \
| grep -vF "$TESTS_FAILED" \
| grep -vF "$TO_SKIP_LIST")
TESTS_IGNORED=$(echo "$TESTS_FAILED" | grep -F "$OPTIONAL_LIST")
FAILURES_UNIGNORED=$(echo "$TESTS_FAILED" | grep -vF "$OPTIONAL_LIST")

N_TIMEOUT=$(echo $TESTS_TIMEOUT | wc -w)
N_PASSED=$(echo $TESTS_PASSED | wc -w)
N_FAILED=$(echo $TESTS_FAILED | wc -w)
N_IGNORED=$(echo $TESTS_IGNORED | wc -w)
N_FAILURES_UNIGNORED=$(echo $FAILURES_UNIGNORED | wc -w)
N_SKIPPED=$(echo $TO_SKIP_LIST | wc -w)

echo "## Test summary" >> $MD_OUT
echo "|Workflow| Passed | Failed | Ignored | Timeout | Skipped" >> $MD_OUT
echo "|--------|--------|--------|---------|---------|---------|" >> $MD_OUT
echo "| $GITHUB_JOB | $N_PASSED | $N_FAILED | $N_IGNORED | $N_TIMEOUT | $N_SKIPPED |" >> $MD_OUT

STATUS=0

if [[ $N_FAILED -ne 0 ]]; then

echo
echo "::group::Failures at a glance"
grep "core dumped" *.log || true
grep "FAILED.*ms" *.log || true
echo "::endgroup::"

echo >> $MD_OUT
echo "## Failures at a glance" >> $MD_OUT
echo "\`\`\`" >> $MD_OUT
grep "core dumped" *.log >> $MD_OUT || true
grep "FAILED.*ms" *.log >> $MD_OUT || true
echo "\`\`\`" >> $MD_OUT

if [ $N_FAILURES_UNIGNORED -eq 0 ]; then
echo "Only ignored tests failed."
else
STATUS=1
echo
echo "== Failing tests =="
echo "::error ::$N_FAILURES_UNIGNORED tests/benchmarks failed."

echo "$FAILURES_UNIGNORED"

echo >> $MD_OUT
echo "## Failing tests" >> $MD_OUT
echo "$FAILURES_UNIGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT


for failedtest in $FAILURES_UNIGNORED; do
echo "::error ::$failedtest failed. See job summary or log for details."
print_test_log "$failedtest.log"
done
fi

if [[ $N_IGNORED -ne 0 ]]; then
echo
echo "::group::Ignored test failures "
echo "$TESTS_IGNORED"
echo "::endgroup::"
echo "::warning ::$N_IGNORED tests/benchmarks failed and ignored."

echo >> $MD_OUT
echo "## Ignored test failures" >> $MD_OUT
echo "$TESTS_IGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT

for failedtest in $TESTS_IGNORED; do
echo "::warning ::$failedtest failed & ignored. See job summary or log for details."
print_test_log "$failedtest.log"
done
fi
else
echo
echo "All tests passed."
fi

echo
echo "::group::Skipped tests"
echo "$TO_SKIP_LIST"
echo "::endgroup::"

echo >> $MD_OUT
echo "## Skipped tests" >> $MD_OUT
echo "$TO_SKIP_LIST" | awk ' { print "1. " $1 } ' >> $MD_OUT

if [[ $N_TIMEOUT -ne 0 ]]; then
echo
echo "::error ::$N_TIMEOUT tests exceeded time limit." \
" Consider adding them to TO_SKIP or increasing TEST_TIMEOUT/BENCHMARK_TIMEOUT."
echo "::group::Timed out tests"
echo "$TESTS_TIMEOUT"
echo "::endgroup::"

echo "## Tests timed out" >> $MD_OUT
echo "$TESTS_TIMEOUT" | awk ' { print "1. " $1 } ' >> $MD_OUT
fi

# Comment out if you do not want (unignored) failing tests to fail the build
exit $STATUS
29 changes: 26 additions & 3 deletions .github/workflows/build-cachelib-centos-8-1.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ on:
jobs:
build-cachelib-centos8-1-1911:
#if: "!contains(github.event.head_commit.author.name, 'svcscm')"
name: "CentOS/8.1.1911 - Build CacheLib with all dependencies"
name: "CentOS/8.1.1911 Build & Test"
runs-on: ubuntu-latest
# Docker container image name
container: "centos:8.1.1911"
Expand Down Expand Up @@ -73,11 +73,21 @@ jobs:
echo === gcc -v ===
gcc -v
- name: "checkout sources"
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: "Install Prerequisites"
run: ./contrib/build.sh -S -B
- name: "Test: update-submodules"
run: ./contrib/update-submodules.sh
- name: "Use cache for built libraries of submodules"
uses: actions/cache@v3
id: cache-submodules
with:
key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }}
path: |
opt/cachelib/lib
opt/cachelib/include
opt/cachelib/bin/thrift1
opt/cachelib/bin/fizz*
- name: "Install dependency: zstd"
run: ./contrib/build-package.sh -j -v -i zstd
- name: "Install dependency: googleflags"
Expand All @@ -92,16 +102,22 @@ jobs:
run: ./contrib/build-package.sh -j -v -i fmt
- name: "Install dependency: folly"
run: ./contrib/build-package.sh -j -v -i folly
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "Install dependency: fizz"
run: ./contrib/build-package.sh -j -v -i fizz
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "Install dependency: wangle"
run: ./contrib/build-package.sh -j -v -i wangle
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "Install dependency: fbthrift"
run: ./contrib/build-package.sh -j -v -i fbthrift
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "build CacheLib"
# Build cachelib in debug mode (-d) and with all tests (-t)
run: ./contrib/build-package.sh -j -v -i -d -t cachelib
- uses: actions/upload-artifact@v2
- name: "Run tests"
run: ./.github/run_tests.sh
- uses: actions/upload-artifact@v3
if: failure()
with:
name: cachelib-cmake-logs
Expand All @@ -112,3 +128,10 @@ jobs:
build-cachelib/**/Makefile
if-no-files-found: warn
retention-days: 1
- uses: actions/upload-artifact@v3
with:
name: cachelib-test-logs
path: opt/cachelib/tests/*.log
if-no-files-found: warn
retention-days: 7
if: success() || failure()
29 changes: 26 additions & 3 deletions .github/workflows/build-cachelib-centos-8-5.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ on:
- cron: '0 9 * * *'
jobs:
build-cachelib-centos8-latest:
name: "CentOS/8.5 - Build CacheLib with all dependencies"
name: "CentOS/8.5 Build & Test"
runs-on: ubuntu-latest
# Docker container image name
container: "centos:latest"
Expand Down Expand Up @@ -72,11 +72,21 @@ jobs:
echo === gcc -v ===
gcc -v
- name: "checkout sources"
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: "Install Prerequisites"
run: ./contrib/build.sh -S -B
- name: "Test: update-submodules"
run: ./contrib/update-submodules.sh
- name: "Use cache for built libraries of submodules"
uses: actions/cache@v3
id: cache-submodules
with:
key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }}
path: |
opt/cachelib/lib
opt/cachelib/include
opt/cachelib/bin/thrift1
opt/cachelib/bin/fizz*
- name: "Install dependency: zstd"
run: ./contrib/build-package.sh -j -v -i zstd
- name: "Install dependency: googleflags"
Expand All @@ -91,16 +101,22 @@ jobs:
run: ./contrib/build-package.sh -j -v -i fmt
- name: "Install dependency: folly"
run: ./contrib/build-package.sh -j -v -i folly
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "Install dependency: fizz"
run: ./contrib/build-package.sh -j -v -i fizz
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "Install dependency: wangle"
run: ./contrib/build-package.sh -j -v -i wangle
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "Install dependency: fbthrift"
run: ./contrib/build-package.sh -j -v -i fbthrift
if: steps.cache-submodules.outputs.cache-hit != 'true'
- name: "build CacheLib"
# Build cachelib in debug mode (-d) and with all tests (-t)
run: ./contrib/build-package.sh -j -v -i -d -t cachelib
- uses: actions/upload-artifact@v2
- name: "Run tests"
run: ./.github/run_tests.sh
- uses: actions/upload-artifact@v3
if: failure()
with:
name: cachelib-cmake-logs
Expand All @@ -111,3 +127,10 @@ jobs:
build-cachelib/**/Makefile
if-no-files-found: warn
retention-days: 1
- uses: actions/upload-artifact@v3
with:
name: cachelib-test-logs
path: opt/cachelib/tests/*.log
if-no-files-found: warn
retention-days: 7
if: success() || failure()
Loading