Skip to content

Commit f731816

Browse files
sfodsfod
authored
Add sanitizer CI job (#703)
1 parent 889f2ec commit f731816

File tree

45 files changed

+368
-67
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+368
-67
lines changed

‎.builder/actions/build_samples.py‎

Lines changed: 5 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# SPDX-License-Identifier: Apache-2.0.
33

44
import Builder
5+
import itertools
56
import os
67
import sys
78

@@ -62,7 +63,7 @@ def run(self, env):
6263
'servicetests/tests/ShadowUpdate/',
6364
]
6465

65-
for sample_path in samples:
66+
for sample_path in itertools.chain(samples, servicetests, da_samples, defender_samples):
6667
build_path = os.path.join('build', sample_path)
6768
steps.append(['cmake',
6869
f'-B{build_path}',
@@ -71,43 +72,9 @@ def run(self, env):
7172
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
7273
# append extra cmake configs
7374
steps[-1].extend(cmd_args.cmake_extra)
74-
steps.append(['cmake',
75-
'--build', build_path,
76-
'--config', 'RelWithDebInfo'])
77-
78-
for sample_path in servicetests:
79-
build_path = os.path.join('build', sample_path)
80-
steps.append(['cmake',
81-
f'-B{build_path}',
82-
f'-H{sample_path}',
83-
f'-DCMAKE_PREFIX_PATH={env.install_dir}',
84-
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
85-
# append extra cmake configs
86-
steps[-1].extend(cmd_args.cmake_extra)
87-
steps.append(['cmake',
88-
'--build', build_path,
89-
'--config', 'RelWithDebInfo'])
90-
91-
for sample_path in da_samples:
92-
build_path = os.path.join('build', sample_path)
93-
steps.append(['cmake',
94-
f'-B{build_path}',
95-
f'-H{sample_path}',
96-
f'-DCMAKE_PREFIX_PATH={env.install_dir}',
97-
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
98-
# append extra cmake configs
99-
steps[-1].extend(cmd_args.cmake_extra)
100-
steps.append(['cmake',
101-
'--build', build_path,
102-
'--config', 'RelWithDebInfo'])
103-
104-
for sample_path in defender_samples:
105-
build_path = os.path.join('build', sample_path)
106-
steps.append(['cmake',
107-
f'-B{build_path}',
108-
f'-H{sample_path}',
109-
f'-DCMAKE_PREFIX_PATH={env.install_dir}',
110-
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
75+
# Currently, cmake_args sets only Linux-specific options.
76+
if sys.platform == "linux" or sys.platform == "linux2":
77+
steps[-1].extend(env.config['cmake_args'])
11178
steps.append(['cmake',
11279
'--build', build_path,
11380
'--config', 'RelWithDebInfo'])

‎.github/workflows/ci.yml‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,3 +685,193 @@ jobs:
685685
- name: Check for edits to code-generated files
686686
run: |
687687
./utils/check_codegen_edits.py
688+
689+
clang-sanitizers:
690+
runs-on: ubuntu-24.04 # latest
691+
strategy:
692+
matrix:
693+
sanitizer-variants: ["tsan", "asan"]
694+
permissions:
695+
id-token: write # This is required for requesting the JWT
696+
steps:
697+
- name: Build ${{ env.PACKAGE_NAME }} + consumers
698+
run: |
699+
echo "Downloading source"
700+
git clone --recursive https://github.com/aws/aws-iot-device-sdk-cpp-v2.git --branch ${{ env.HEAD_REF || github.ref_name }}
701+
echo "Running builder"
702+
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
703+
python builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant ${{ matrix.sanitizer-variants }}
704+
- name: Running samples in CI setup
705+
run: |
706+
python3 -m pip install boto3
707+
sudo apt-get update -y
708+
sudo apt-get install softhsm2 -y
709+
softhsm2-util --version
710+
- name: configure AWS credentials (Fleet provisioning)
711+
uses: aws-actions/configure-aws-credentials@v2
712+
with:
713+
role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }}
714+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
715+
- name: run Fleet Provisioning service client test for MQTT311
716+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
717+
run: |
718+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
719+
python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt3_fleet_provisioning_cfg.json --thing-name-prefix Fleet_Thing_
720+
- name: run Fleet Provisioning service client test for MQTT5
721+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
722+
run: |
723+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
724+
python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt5_fleet_provisioning_cfg.json --thing-name-prefix Fleet_Thing_
725+
- name: run Fleet Provisioning with CSR service client test for MQTT311
726+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
727+
run: |
728+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
729+
python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt3_fleet_provisioning_with_csr_cfg.json --thing-name-prefix Fleet_Thing_
730+
- name: run Fleet Provisioning with CSR service client test for MQTT5
731+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
732+
run: |
733+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
734+
python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt5_fleet_provisioning_with_csr_cfg.json --thing-name-prefix Fleet_Thing_
735+
- name: configure AWS credentials (Jobs)
736+
uses: aws-actions/configure-aws-credentials@v2
737+
with:
738+
role-to-assume: ${{ env.CI_JOBS_SERVICE_CLIENT_ROLE }}
739+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
740+
- name: run mqtt3 Jobs serviceTests
741+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
742+
run: |
743+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
744+
python3 ./test_cases/test_jobs_execution.py --config-file ${{ env.CI_SERVICE_TESTS_CFG_FOLDER }}/mqtt3_jobs_cfg.json
745+
- name: run mqtt5 Jobs serviceTests
746+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
747+
run: |
748+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
749+
python3 ./test_cases/test_jobs_execution.py --config-file ${{ env.CI_SERVICE_TESTS_CFG_FOLDER }}/mqtt5_jobs_cfg.json
750+
- name: configure AWS credentials (Shadow)
751+
uses: aws-actions/configure-aws-credentials@v2
752+
with:
753+
role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }}
754+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
755+
- name: run Shadow service client test for MQTT5
756+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
757+
run: |
758+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
759+
python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json
760+
- name: run Shadow service client test for MQTT311
761+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
762+
run: |
763+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
764+
python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json
765+
- name: run Named Shadow service client test for MQTT311
766+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
767+
run: |
768+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
769+
python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json
770+
- name: run Named Shadow service client test for MQTT5
771+
working-directory: ./aws-iot-device-sdk-cpp-v2/servicetests
772+
run: |
773+
export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-cpp-v2/utils
774+
python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json
775+
- name: configure AWS credentials (Connect and PubSub)
776+
uses: aws-actions/configure-aws-credentials@v2
777+
with:
778+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
779+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
780+
- name: run Basic Connect sample
781+
run: |
782+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_basic_connect_cfg.json
783+
- name: run Websocket Connect sample
784+
run: |
785+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_websocket_connect_cfg.json
786+
- name: run MQTT3 PubSub sample
787+
run: |
788+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_pubsub_cfg.json
789+
- name: run PKCS11 Connect sample
790+
run: |
791+
mkdir -p /tmp/tokens
792+
export SOFTHSM2_CONF=/tmp/softhsm2.conf
793+
echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf
794+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_pkcs11_connect_cfg.json
795+
- name: configure AWS credentials (MQTT5)
796+
uses: aws-actions/configure-aws-credentials@v2
797+
with:
798+
role-to-assume: ${{ env.CI_MQTT5_ROLE }}
799+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
800+
- name: run MQTT5 PubSub sample
801+
run: |
802+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_mqtt5_pubsub_cfg.json
803+
- name: run MQTT5 Shared Subscription sample
804+
run: |
805+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_mqtt5_shared_subscription_cfg.json
806+
- name: configure AWS credentials (Jobs)
807+
uses: aws-actions/configure-aws-credentials@v2
808+
with:
809+
role-to-assume: ${{ env.CI_JOBS_ROLE }}
810+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
811+
- name: run Jobs sample
812+
run: |
813+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_jobs_cfg.json
814+
- name: run Mqtt5 Jobs sample
815+
run: |
816+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_jobs_mqtt5_cfg.json
817+
- name: configure AWS credentials (Cognito)
818+
uses: aws-actions/configure-aws-credentials@v2
819+
with:
820+
role-to-assume: ${{ env.CI_COGNITO_ROLE }}
821+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
822+
- name: run CognitoConnect sample
823+
run: |
824+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_cognito_connect_cfg.json
825+
- name: configure AWS credentials (Custom Authorizer)
826+
uses: aws-actions/configure-aws-credentials@v2
827+
with:
828+
role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }}
829+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
830+
- name: run CustomAuthorizerConnect sample
831+
run: |
832+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_custom_authorizer_connect_cfg.json
833+
- name: configure AWS credentials (Shadow)
834+
uses: aws-actions/configure-aws-credentials@v2
835+
with:
836+
role-to-assume: ${{ env.CI_SHADOW_ROLE }}
837+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
838+
- name: run Shadow sample
839+
run: |
840+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_shadow_cfg.json
841+
- name: run Mqtt5 Shadow sample
842+
run: |
843+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_shadow_mqtt5_cfg.json
844+
- name: configure AWS credentials (Fleet provisioning)
845+
uses: aws-actions/configure-aws-credentials@v2
846+
with:
847+
role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }}
848+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
849+
- name: run Fleet Provisioning sample
850+
run: |
851+
echo "Generating UUID for IoT thing"
852+
Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())")
853+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_fleet_provisioning_cfg.json --input_uuid ${Sample_UUID}
854+
python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
855+
- name: run Mqtt5 Fleet Provisioning sample
856+
run: |
857+
echo "Generating UUID for IoT thing"
858+
Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())")
859+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_fleet_provisioning_mqtt5_cfg.json --input_uuid ${Sample_UUID}
860+
python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
861+
- name: configure AWS credentials (Secure tunneling)
862+
uses: aws-actions/configure-aws-credentials@v2
863+
with:
864+
role-to-assume: ${{ env.CI_SECURE_TUNNEL }}
865+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
866+
# Secure Tunneling has special requirements, so it uses a different Python file
867+
- name: run Secure Tunneling sample
868+
run: |
869+
python3 ${{ env.CI_UTILS_FOLDER }}/run_secure_tunnel_ci.py --sample_file "./aws-iot-device-sdk-cpp-v2/build/samples/secure_tunneling/secure_tunnel/secure-tunnel" --sample_region ${{ env.AWS_DEFAULT_REGION }}
870+
- name: configure AWS credentials (X509)
871+
uses: aws-actions/configure-aws-credentials@v2
872+
with:
873+
role-to-assume: ${{ env.CI_X509_ROLE }}
874+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
875+
- name: run X509 sample
876+
run: |
877+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_x509_connect_cfg.json

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ else()
7979
include(AwsFindPackage)
8080
endif()
8181

82+
include(AwsSanitizers)
83+
8284
aws_use_package(aws-crt-cpp)
8385

8486
add_subdirectory(jobs)

‎builder.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,18 @@
2929
"build"
3030
]
3131
},
32+
"tsan": {
33+
"cmake_args": [
34+
"-DENABLE_SANITIZERS=ON",
35+
"-DSANITIZERS=thread"
36+
]
37+
},
38+
"asan": {
39+
"cmake_args": [
40+
"-DENABLE_SANITIZERS=ON",
41+
"-DSANITIZERS=address,undefined"
42+
]
43+
},
3244
"build_gg_samples_only": {
3345
"!build_steps": [
3446
"build",

‎devicedefender/CMakeLists.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,9 @@ if (BUILD_DEPS)
8888
endif()
8989
endif()
9090

91-
target_link_libraries(IotDeviceDefender-cpp IotDeviceCommon-cpp)
91+
aws_add_sanitizers(IotDeviceDefender-cpp)
92+
93+
target_link_libraries(IotDeviceDefender-cpp PUBLIC IotDeviceCommon-cpp)
9294

9395
install(FILES ${AWS_IOTDEVICEDEFENDER_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/iotdevicedefender/" COMPONENT Development)
9496

‎devicedefender/tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,5 @@ if (UNIX AND NOT APPLE)
1919
add_net_test_case(Mqtt5DeviceDefenderCustomMetricSuccess)
2020
add_net_test_case(Mqtt5DeviceDefenderCustomMetricFail)
2121
generate_cpp_test_driver(${TEST_BINARY_NAME})
22+
aws_add_sanitizers(${TEST_BINARY_NAME})
2223
endif()

‎discovery/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ if (BUILD_DEPS)
8787
endif()
8888
endif()
8989

90-
target_link_libraries(Discovery-cpp ${DEP_AWS_LIBS})
90+
aws_add_sanitizers(Discovery-cpp)
91+
target_link_libraries(Discovery-cpp PUBLIC ${DEP_AWS_LIBS})
9192

9293
install(FILES ${AWS_DISCOVERY_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/discovery/" COMPONENT Development)
9394

‎eventstream_rpc/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,8 @@ if (NOT IS_SUBDIRECTORY_INCLUDE)
8585
aws_use_package(aws-crt-cpp)
8686
endif()
8787

88-
89-
target_link_libraries(EventstreamRpc-cpp ${DEP_AWS_LIBS})
88+
aws_add_sanitizers(EventstreamRpc-cpp)
89+
target_link_libraries(EventstreamRpc-cpp PUBLIC ${DEP_AWS_LIBS})
9090

9191
install(FILES ${AWS_EVENTSTREAMRPC_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/eventstreamrpc/" COMPONENT Development)
9292

‎eventstream_rpc/tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ add_test_case(OperateWhileDisconnected)
3636
#add_test_case(EchoOperation)
3737
#add_test_case(StressTestClient)
3838
generate_cpp_test_driver(${TEST_BINARY_NAME})
39+
aws_add_sanitizers(${TEST_BINARY_NAME})
3940
target_include_directories(${TEST_BINARY_NAME} PUBLIC
4041
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
4142
$<INSTALL_INTERFACE:include>)

‎greengrass_ipc/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ if (WIN32)
3333
endif()
3434

3535
add_library(GreengrassIpc-cpp ${AWS_GREENGRASSIPC_CPP_SRC})
36-
target_link_libraries(GreengrassIpc-cpp EventstreamRpc-cpp)
36+
target_link_libraries(GreengrassIpc-cpp PUBLIC EventstreamRpc-cpp)
3737

3838
set_target_properties(GreengrassIpc-cpp PROPERTIES LINKER_LANGUAGE CXX)
3939

@@ -93,8 +93,8 @@ if (NOT IS_SUBDIRECTORY_INCLUDE)
9393
aws_use_package(aws-crt-cpp)
9494
endif()
9595

96-
97-
target_link_libraries(GreengrassIpc-cpp ${DEP_AWS_LIBS})
96+
aws_add_sanitizers(GreengrassIpc-cpp)
97+
target_link_libraries(GreengrassIpc-cpp PUBLIC ${DEP_AWS_LIBS})
9898

9999
install(FILES ${AWS_GREENGRASSIPC_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/greengrass/" COMPONENT Development)
100100

0 commit comments

Comments
 (0)