Skip to content

Commit 428651f

Browse files
gty404gty404
committed
Merge branch 'main' into table-scan
2 parents fa25891 + ed49d1e commit 428651f

24 files changed

+563
-32
lines changed

‎.github/lsan-suppressions.txt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# Add specific leak suppressions here if needed
19+
# Format:
20+
# leak:SymbolName
21+
# leak:source_file.cc

‎.github/ubsan-suppressions.txt

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# Add specific undefined suppressions here if needed
19+
# Format:
20+
# signed-integer-overflow:source_file.cc

‎.github/workflows/sanitizer_test.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: ASAN and UBSAN Tests
19+
20+
on:
21+
push:
22+
branches:
23+
- '**'
24+
- '!dependabot/**'
25+
tags:
26+
- '**'
27+
pull_request:
28+
29+
concurrency:
30+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
31+
cancel-in-progress: true
32+
33+
permissions:
34+
contents: read
35+
36+
jobs:
37+
sanitizer-test:
38+
name: "ASAN and UBSAN Tests"
39+
runs-on: ubuntu-24.04
40+
strategy:
41+
fail-fast: false
42+
steps:
43+
- name: Checkout iceberg-cpp
44+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
45+
- name: Configure and Build with ASAN & UBSAN
46+
run: |
47+
mkdir build && cd build
48+
cmake .. -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON
49+
cmake --build . --verbose
50+
- name: Run Tests
51+
working-directory: build
52+
env:
53+
ASAN_OPTIONS: log_path=out.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=0:detect_container_overflow=0
54+
LSAN_OPTIONS: suppressions=${{ github.workspace }}/.github/lsan-suppressions.txt
55+
UBSAN_OPTIONS: log_path=out.log:halt_on_error=0:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt
56+
run: |
57+
ctest --output-on-failure
58+
- name: Save the test output
59+
if: always()
60+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
61+
with:
62+
name: test-output
63+
path: build/test/out.log*

‎CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ option(ICEBERG_BUILD_STATIC "Build static library" ON)
3737
option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
3838
option(ICEBERG_BUILD_TESTS "Build tests" ON)
3939
option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON)
40+
option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
41+
option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
4042

4143
include(GNUInstallDirs)
4244
include(FetchContent)
@@ -55,6 +57,7 @@ endif()
5557

5658
include(CMakeParseArguments)
5759
include(IcebergBuildUtils)
60+
include(IcebergSanitizer)
5861
include(IcebergThirdpartyToolchain)
5962
include(GenerateExportHeader)
6063

‎cmake_modules/IcebergBuildUtils.cmake

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ function(add_iceberg_lib LIB_NAME)
147147
"$<INSTALL_INTERFACE:${ARG_SHARED_INSTALL_INTERFACE_LIBS}>"
148148
PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
149149

150+
target_link_libraries(${LIB_NAME}_shared
151+
PUBLIC "$<BUILD_INTERFACE:iceberg_sanitizer_flags>")
152+
150153
install(TARGETS ${LIB_NAME}_shared
151154
EXPORT iceberg_targets
152155
ARCHIVE DESTINATION ${INSTALL_ARCHIVE_DIR}
@@ -202,6 +205,9 @@ function(add_iceberg_lib LIB_NAME)
202205
PUBLIC "$<BUILD_INTERFACE:${ARG_STATIC_LINK_LIBS}>")
203206
endif()
204207

208+
target_link_libraries(${LIB_NAME}_static
209+
PUBLIC "$<BUILD_INTERFACE:iceberg_sanitizer_flags>")
210+
205211
install(TARGETS ${LIB_NAME}_static
206212
EXPORT iceberg_targets
207213
ARCHIVE DESTINATION ${INSTALL_ARCHIVE_DIR}

‎cmake_modules/IcebergSanitizer.cmake

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
add_library(iceberg_sanitizer_flags INTERFACE)
19+
20+
if(ICEBERG_ENABLE_ASAN)
21+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
22+
target_compile_options(iceberg_sanitizer_flags INTERFACE -fsanitize=address
23+
-fno-omit-frame-pointer)
24+
target_link_options(iceberg_sanitizer_flags INTERFACE -fsanitize=address)
25+
message(STATUS "Address Sanitizer enabled")
26+
else()
27+
message(WARNING "Address Sanitizer is only supported for GCC and Clang compilers")
28+
endif()
29+
endif()
30+
31+
if(ICEBERG_ENABLE_UBSAN)
32+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
33+
target_compile_options(iceberg_sanitizer_flags INTERFACE -fsanitize=undefined
34+
-fno-omit-frame-pointer)
35+
target_link_options(iceberg_sanitizer_flags INTERFACE -fsanitize=undefined)
36+
message(STATUS "Undefined Behavior Sanitizer enabled")
37+
else()
38+
message(WARNING "Undefined Behavior Sanitizer is only supported for GCC and Clang compilers"
39+
)
40+
endif()
41+
endif()

‎src/iceberg/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ if(ICEBERG_BUILD_BUNDLE)
9696
arrow/demo_arrow.cc
9797
arrow/arrow_fs_file_io.cc
9898
avro/demo_avro.cc
99+
avro/avro_data_util.cc
100+
avro/avro_reader.cc
99101
avro/avro_schema_util.cc
100102
avro/avro_stream_internal.cc)
101103

‎src/iceberg/arrow/arrow_fs_file_io.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ class ICEBERG_BUNDLE_EXPORT ArrowFileSystemFileIO : public FileIO {
4646
/// \brief Delete a file at the given location.
4747
Status DeleteFile(const std::string& file_location) override;
4848

49+
/// \brief Get the Arrow file system.
50+
const std::shared_ptr<::arrow::fs::FileSystem>& fs() const { return arrow_fs_; }
51+
4952
private:
5053
std::shared_ptr<::arrow::fs::FileSystem> arrow_fs_;
5154
};

‎src/iceberg/avro/avro_data_util.cc

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/avro/avro_data_util_internal.h"
21+
22+
namespace iceberg::avro {
23+
24+
Status AppendDatumToBuilder(const ::avro::NodePtr& avro_node,
25+
const ::avro::GenericDatum& avro_datum,
26+
const SchemaProjection& projection,
27+
const Schema& arrow_schema,
28+
::arrow::ArrayBuilder* array_builder) {
29+
return NotImplemented("AppendDatumToBuilder is not yet implemented");
30+
}
31+
32+
} // namespace iceberg::avro

‎src/iceberg/avro/avro_data_util_internal.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
#include <arrow/array/builder_base.h>
23+
#include <avro/GenericDatum.hh>
24+
25+
#include "iceberg/schema_util.h"
26+
27+
namespace iceberg::avro {
28+
29+
Status AppendDatumToBuilder(const ::avro::NodePtr& avro_node,
30+
const ::avro::GenericDatum& avro_datum,
31+
const SchemaProjection& projection,
32+
const Schema& arrow_schema,
33+
::arrow::ArrayBuilder* array_builder);
34+
35+
} // namespace iceberg::avro

0 commit comments

Comments
 (0)