refactor: exprimental parquet reader

Author: dantengsky

Cargo.toml

@@ -161,6 +161,7 @@ databend-common-meta-types = { path = "src/meta/types" }
 databend-common-metrics = { path = "src/common/metrics" }
 databend-common-native = { path = "src/common/native" }
 databend-common-openai = { path = "src/common/openai" }
+databend-common-parquet-reader-experimental = { path = "src/common/experimental_parquet_reader" }
 databend-common-pipeline-core = { path = "src/query/pipeline/core" }
 databend-common-pipeline-sinks = { path = "src/query/pipeline/sinks" }
 databend-common-pipeline-sources = { path = "src/query/pipeline/sources" }
@@ -232,16 +233,16 @@ anyerror = { version = "=0.1.13" }
 anyhow = { version = "1.0.65" }
 apache-avro = { version = "0.17.0", features = ["snappy", "zstandard", "xz", "snappy", "bzip"] }
 approx = "0.5.1"
-arrow = { version = "55" }
-arrow-array = { version = "55" }
-arrow-buffer = { version = "55" }
-arrow-cast = { version = "55", features = ["prettyprint"] }
-arrow-data = { version = "55" }
-arrow-flight = { version = "55", features = ["flight-sql-experimental", "tls"] }
-arrow-ipc = { version = "55" }
-arrow-ord = { version = "55" }
-arrow-schema = { version = "55", features = ["serde"] }
-arrow-select = { version = "55" }
+arrow = { version = "55.2" }
+arrow-array = { version = "55.2" }
+arrow-buffer = { version = "55.2" }
+arrow-cast = { version = "55.2", features = ["prettyprint"] }
+arrow-data = { version = "55.2" }
+arrow-flight = { version = "55.2", features = ["flight-sql-experimental", "tls"] }
+arrow-ipc = { version = "55.2" }
+arrow-ord = { version = "55.2" }
+arrow-schema = { version = "55.2", features = ["serde"] }
+arrow-select = { version = "55.2" }
 arrow-udf-runtime = { version = "0.8.0", default-features = false }
 async-backtrace = "0.2"
 async-channel = "1.7.1"

scripts/ci/deploy/config/databend-query-node-1.toml

@@ -84,7 +84,7 @@ join_spilling_memory_ratio = 60
 [log]
 
 [log.file]
-level = "DEBUG"
+level = "INFO"
 format = "text"
 dir = "./.databend/logs_1"
 limit = 12 # 12 files, 1 file per hour
@@ -153,6 +153,9 @@ data_path = "./.databend/stateless_test_data"
 # use "disk" to enabled disk cache
 data_cache_storage = "none"
 
+
+table_data_deserialized_data_bytes = 1073741824
+
 [cache.disk]
 # cache path
 path = "./.databend/_cache"

src/common/column/src/binview/mod.rs

@@ -133,9 +133,9 @@ impl<T: ViewType + ?Sized> Clone for BinaryViewColumnGeneric<T> {
     }
 }
 
-unsafe impl<T: ViewType + ?Sized> Send for BinaryViewColumnGeneric<T> {}
+// impl<T: ViewType + ?Sized> Send for BinaryViewColumnGeneric<T> {}
 
-unsafe impl<T: ViewType + ?Sized> Sync for BinaryViewColumnGeneric<T> {}
+// unsafe impl<T: ViewType + ?Sized> Sync for BinaryViewColumnGeneric<T> {}
 
 impl<T: ViewType + ?Sized> BinaryViewColumnGeneric<T> {
     pub fn new_unchecked(

src/common/experimental_parquet_reader/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "databend-common-parquet-reader-experimental"
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+edition = { workspace = true }
+
+[features]
+
+[dependencies]
+databend-common-column = { workspace = true }
+databend-common-exception = { workspace = true }
+databend-common-expression = { workspace = true }
+databend-storages-common-table-meta = { workspace = true }
+
+bytes = { workspace = true }
+lz4_flex = { version = "^0.9" }
+parquet2 = { version = "0.17.0", default-features = false, features = ["serde_types", "async", "zstd", "snappy", "lz4"] }
+parquet-format-safe = "0.2.0"
+parquet = { workspace = true, features = ["experimental"] }
+streaming-decompression = "0.1.2"
+zstd = { workspace = true }
+
+[dev-dependencies]
+# used to test async readers
+
+[package.metadata.cargo-machete]
+ignored = ["match-template"]
+
+[lints]
+workspace = true