diff --git a/duckdb-dataframe/results/c6a.metal.json b/duckdb-dataframe/results/c6a.metal.json index 44ee1b8a1..f939a4147 100644 --- a/duckdb-dataframe/results/c6a.metal.json +++ b/duckdb-dataframe/results/c6a.metal.json @@ -1,57 +1,57 @@ { "system": "DuckDB (DataFrame)", - "date": "2025-09-07", + "date": "2025-09-16", "machine": "c6a.metal", "cluster_size": 1, "proprietary": "no", "tuned": "no", "tags": ["C++","column-oriented","embedded","Python","dataframe"], - "load_time": 94, - "data_size": 179532066816, + "load_time": 82.77, + "data_size": 176641134592, "result": [ - [0.028, 0.018, 0.017], - [0.026, 0.018, 0.018], - [0.025, 0.022, 0.024], - [0.036, 0.022, 0.025], - [0.22, 0.186, 0.162], - [0.163, 0.167, 0.16], - [0.031, 0.029, 0.032], - [0.035, 0.039, 0.039], - [0.206, 0.2, 0.192], - [0.253, 0.233, 0.238], - [0.074, 0.069, 0.074], - [0.084, 0.076, 0.069], - [0.143, 0.145, 0.144], - [0.286, 0.284, 0.277], - [0.182, 0.147, 0.151], - [0.164, 0.167, 0.176], - [0.401, 0.348, 0.29], - [0.286, 0.295, 0.29], - [0.388, 0.41, 0.395], - [0.024, 0.025, 0.026], - [0.552, 0.311, 0.31], - [0.311, 0.382, 0.424], - [0.597, 0.585, 0.591], - [1.251, 0.985, 1.002], - [0.064, 0.064, 0.067], - [0.068, 0.069, 0.069], - [0.071, 0.072, 0.071], - [0.276, 0.291, 0.298], - [1.511, 1.455, 1.657], - [0.038, 0.032, 0.036], - [0.216, 0.172, 0.163], - [0.209, 0.186, 0.197], - [0.588, 0.55, 0.5], - [0.62, 0.579, 0.534], - [0.554, 0.597, 0.565], - [0.203, 0.212, 0.189], - [0.276, 0.267, 0.274], - [0.348, 0.339, 0.34], - [0.261, 0.261, 0.263], - [0.396, 0.397, 0.391], - [0.033, 0.032, 0.035], - [0.035, 0.037, 0.037], - [0.045, 0.048, 0.05] + [0.034, 0.016, 0.017], + [0.035, 0.022, 0.024], + [0.029, 0.026, 0.025], + [0.066, 0.026, 0.031], + [0.865, 0.094, 0.097], + [0.171, 0.151, 0.166], + [0.031, 0.03, 0.035], + [0.022, 0.023, 0.025], + [0.11, 0.114, 0.114], + [0.14, 0.141, 0.143], + [0.055, 0.052, 0.051], + [0.07, 0.054, 0.056], + [0.154, 0.15, 0.151], + [0.199, 0.194, 0.197], + [0.147, 0.152, 0.174], + [0.108, 0.106, 0.104], + [0.228, 0.223, 0.236], + [0.245, 0.254, 0.236], + [0.331, 0.345, 0.315], + [0.053, 0.03, 0.031], + [0.328, 0.315, 0.308], + [0.329, 0.332, 0.355], + [0.613, 0.622, 0.615], + [1.396, 1.1, 1.104], + [0.086, 0.08, 0.08], + [0.075, 0.076, 0.075], + [0.087, 0.085, 0.082], + [0.308, 0.322, 0.308], + [1.471, 1.36, 1.39], + [0.037, 0.037, 0.033], + [0.14, 0.136, 0.119], + [0.186, 0.141, 0.139], + [0.46, 0.43, 0.421], + [0.57, 0.563, 0.615], + [0.589, 0.547, 0.551], + [0.132, 0.127, 0.131], + [0.313, 0.31, 0.308], + [0.371, 0.363, 0.358], + [0.277, 0.32, 0.319], + [0.455, 0.421, 0.446], + [0.038, 0.039, 0.034], + [0.035, 0.036, 0.038], + [0.046, 0.067, 0.052] ] } diff --git a/duckdb-memory/benchmark.sh b/duckdb-memory/benchmark.sh index 23a387902..c7e009c53 100755 --- a/duckdb-memory/benchmark.sh +++ b/duckdb-memory/benchmark.sh @@ -9,10 +9,7 @@ source myenv/bin/activate pip install duckdb psutil # Load the data - -sudo apt-get install -y pigz -wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' -pigz -d -f hits.tsv.gz +wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.parquet' # Run the queries diff --git a/duckdb-memory/create.sql b/duckdb-memory/create.sql index 4d23eaac6..9608c6b17 100644 --- a/duckdb-memory/create.sql +++ b/duckdb-memory/create.sql @@ -1,3 +1,5 @@ +ATTACH ':memory:' AS compressed_mem (COMPRESS); +USE compressed_mem; CREATE TABLE hits ( WatchID BIGINT NOT NULL, diff --git a/duckdb-memory/load.sql b/duckdb-memory/load.sql new file mode 100644 index 000000000..24891835f --- /dev/null +++ b/duckdb-memory/load.sql @@ -0,0 +1,7 @@ +INSERT INTO hits +SELECT * REPLACE ( + make_date(EventDate) AS EventDate, + epoch_ms(EventTime * 1000) AS EventTime, + epoch_ms(ClientEventTime * 1000) AS ClientEventTime, + epoch_ms(LocalEventTime * 1000) AS LocalEventTime) +FROM read_parquet('hits.parquet', binary_as_string=True); diff --git a/duckdb-memory/memory.py b/duckdb-memory/memory.py index 30966808d..ffe43331d 100755 --- a/duckdb-memory/memory.py +++ b/duckdb-memory/memory.py @@ -13,4 +13,4 @@ con.execute("SET preserve_insertion_order = false;") con.execute(open("create.sql").read()) -con.execute("COPY hits FROM 'hits.csv';") +con.execute(open("load.sql").read()) diff --git a/duckdb-memory/query.py b/duckdb-memory/query.py index f3922ba2b..b22f1d650 100755 --- a/duckdb-memory/query.py +++ b/duckdb-memory/query.py @@ -17,7 +17,7 @@ print("Will load the data") start = timeit.default_timer() con.execute(open("create.sql").read()) -con.execute("COPY hits FROM 'hits.tsv' (QUOTE '');") +con.execute(open("load.sql").read()) end = timeit.default_timer() print(round(end - start, 3)) diff --git a/duckdb-memory/results/c6a.4xlarge.json b/duckdb-memory/results/c6a.4xlarge.json index b7c56e6c6..8eeaa01fc 100644 --- a/duckdb-memory/results/c6a.4xlarge.json +++ b/duckdb-memory/results/c6a.4xlarge.json @@ -6,52 +6,52 @@ "proprietary": "no", "tuned": "no", "tags": ["C++","column-oriented","embedded"], - "load_time": 429, - "data_size": 27956441088, + "load_time": 281.689, + "data_size": 28136767488, "result": [ - [0.032, 0.013, 0.006], - [0.67, 0.016, 0.016], - [0.667, 0.051, 0.061], - [1.702, 0.047, 0.089], - [0.778, 0.544, 0.498], - [2.859, 0.72, 0.52], - [1.022, 0.022, 0.022], - [0.079, 0.013, 0.013], - [1.815, 1.078, 0.814], - [1.086, 1.082, 0.937], - [1.38, 0.11, 0.107], - [0.646, 0.145, 0.14], - [0.739, 0.468, 0.427], - [1.379, 0.874, 1.345], - [1.035, 0.503, 0.437], - [0.595, 0.463, 0.446], - [1.32, 1.227, 1.079], - [0.771, 0.807, 0.821], - [4.499, 2.536, 2.253], - [0.096, 0.023, 0.023], - [26.487, 10.098, 6.269], - [5.939, 6.146, 4.001], - [42.878, 38.645, 34.709], - [6.553, 1.854, 1.948], - [0.505, 0.074, 0.035], - [0.238, 0.229, 0.09], - [0.032, 0.027, 0.034], - [15.547, 13.13, 9.468], - [28.847, 17.557, 13.645], - [0.818, 0.031, 0.019], - [6.765, 2.383, 1.185], - [5.801, 2.327, 2.681], - [14.69, 3.848, 2.552], - [27.652, 23.772, 19.437], - [15.76, 16.019, 12.449], - [1.041, 0.622, 0.585], - [0.064, 0.023, 0.033], - [0.121, 0.026, 0.017], - [0.023, 0.009, 0.01], - [0.085, 0.039, 0.049], - [0.034, 0.007, 0.007], - [0.028, 0.007, 0.007], - [0.024, 0.009, 0.01] + [0.032,0.003,0.003], + [0.009,0.01,0.005], + [0.03,0.022,0.022], + [0.041,0.034,0.033], + [0.371,0.364,0.366], + [0.374,0.335,0.32], + [0.006,0.005,0.005], + [0.018,0.006,0.006], + [0.4,0.402,0.401], + [0.554,0.554,0.552], + [0.082,0.081,0.079], + [0.085,0.088,0.088], + [0.366,0.351,0.35], + [0.701,0.693,0.674], + [0.395,0.392,0.388], + [0.37,0.36,0.356], + [0.862,0.817,0.832], + [0.633,0.638,0.615], + [1.584,1.57,1.563], + [0.008,0.012,0.007], + [0.546,0.521,0.501], + [0.499,0.498,0.501], + [0.766,0.677,0.682], + [0.084,0.08,0.08], + [0.02,0.02,0.02], + [0.075,0.07,0.069], + [0.021,0.015,0.015], + [0.409,0.407,0.405], + [6.503,6.472,6.45], + [0.033,0.03,0.032], + [0.301,0.3,0.299], + [0.39,0.369,0.374], + [1.826,1.753,1.744], + [1.805,1.777,1.779], + [1.959,1.955,1.934], + [0.476,0.505,0.538], + [0.075,0.027,0.042], + [0.038,0.015,0.027], + [0.016,0.035,0.046], + [0.078,0.11,0.062], + [0.028,0.006,0.009], + [0.008,0.009,0.007], + [0.021,0.02,0.013] ] } diff --git a/duckdb-memory/results/c6a.metal.json b/duckdb-memory/results/c6a.metal.json index 233df2757..e4a429575 100644 --- a/duckdb-memory/results/c6a.metal.json +++ b/duckdb-memory/results/c6a.metal.json @@ -1,57 +1,57 @@ { "system": "DuckDB (memory)", - "date": "2025-08-31", + "date": "2025-09-16", "machine": "c6a.metal", "cluster_size": 1, "proprietary": "no", "tuned": "no", "tags": ["C++","column-oriented","embedded"], - "load_time": 27, - "data_size": 118522880000, + "load_time": 15.244, + "data_size": 109925945344, "result": [ - [0.089, 0.018, 0.017], - [0.018, 0.01, 0.008], - [0.023, 0.019, 0.019], - [0.021, 0.022, 0.02], - [0.162, 0.149, 0.155], - [0.187, 0.145, 0.15], - [0.042, 0.018, 0.018], - [0.018, 0.012, 0.01], - [0.212, 0.184, 0.186], - [0.251, 0.233, 0.239], - [0.073, 0.073, 0.066], - [0.068, 0.071, 0.062], - [0.137, 0.126, 0.124], - [0.26, 0.271, 0.265], - [0.139, 0.154, 0.213], - [0.257, 0.159, 0.16], - [0.394, 0.35, 0.351], - [0.414, 0.303, 0.322], - [0.466, 0.386, 0.375], - [0.034, 0.033, 0.045], - [0.241, 0.094, 0.091], - [0.12, 0.118, 0.1], - [0.477, 0.214, 0.19], - [0.194, 0.115, 0.121], - [0.049, 0.049, 0.056], - [0.041, 0.04, 0.041], - [0.104, 0.07, 0.045], - [0.11, 0.103, 0.115], - [2.42, 1.384, 1.328], - [0.114, 0.033, 0.071], - [0.345, 0.134, 0.152], - [0.322, 0.314, 0.188], - [0.622, 0.642, 0.646], - [0.597, 0.509, 0.479], - [0.494, 0.484, 0.478], - [0.195, 0.175, 0.19], - [0.059, 0.046, 0.046], - [0.061, 0.042, 0.046], - [0.03, 0.04, 0.026], - [0.105, 0.043, 0.05], - [0.034, 0.032, 0.02], - [0.035, 0.01, 0.046], - [0.034, 0.013, 0.042] -] + [0.012,0.027,0.004], + [0.005,0.011,0.035], + [0.096,0.008,0.006], + [0.099,0.007,0.006], + [0.177,0.248,0.183], + [0.208,0.193,0.26], + [0.009,0.004,0.004], + [0.033,0.088,0.006], + [0.208,0.268,0.178], + [0.217,0.307,0.307], + [0.086,0.111,0.055], + [0.059,0.068,0.074], + [0.103,0.119,0.121], + [0.25,0.173,0.283], + [0.122,0.133,0.142], + [0.099,0.099,0.1], + [0.168,0.169,0.169], + [0.171,0.185,0.184], + [0.317,0.31,0.306], + [0.006,0.005,0.004], + [0.104,0.077,0.073], + [0.074,0.072,0.074], + [0.098,0.091,0.095], + [0.074,0.07,0.063], + [0.011,0.013,0.013], + [0.031,0.034,0.032], + [0.011,0.011,0.012], + [0.073,0.075,0.074], + [1.195,1.103,1.06], + [0.024,0.019,0.023], + [0.094,0.09,0.082], + [0.106,0.096,0.097], + [0.471,0.436,0.445], + [0.366,0.323,0.346], + [0.351,0.366,0.351], + [0.105,0.098,0.107], + [0.022,0.021,0.023], + [0.015,0.006,0.011], + [0.016,0.016,0.012], + [0.04,0.045,0.038], + [0.007,0.005,0.008], + [0.006,0.014,0.006], + [0.008,0.01,0.019] + ] } diff --git a/duckdb-parquet-partitioned/results/c6a.4xlarge.json b/duckdb-parquet-partitioned/results/c6a.4xlarge.json index 4374ead4e..588a943a3 100644 --- a/duckdb-parquet-partitioned/results/c6a.4xlarge.json +++ b/duckdb-parquet-partitioned/results/c6a.4xlarge.json @@ -9,49 +9,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.173, 0.002, 0.001], - [0.129, 0.02, 0.02], - [0.228, 0.053, 0.052], - [0.846, 0.048, 0.046], - [1.704, 0.345, 0.343], - [1.754, 0.477, 0.471], - [0.126, 0.043, 0.042], - [0.07, 0.022, 0.022], - [1.174, 0.48, 0.483], - [1.874, 0.593, 0.596], - [0.829, 0.105, 0.105], - [1.686, 0.131, 0.122], - [1.868, 0.48, 0.474], - [3.114, 0.772, 0.773], - [1.337, 0.526, 0.524], - [1, 0.42, 0.416], - [3.199, 1.011, 1.012], - [2.842, 0.794, 0.811], - [5.361, 1.802, 1.806], - [0.126, 0.019, 0.018], - [11.241, 0.855, 0.858], - [11.986, 0.807, 0.806], - [22.281, 1.694, 1.695], - [4.39, 0.749, 0.844], - [0.187, 0.108, 0.11], - [1.63, 0.258, 0.252], - [0.458, 0.08, 0.079], - [10.909, 0.713, 0.73], - [11.251, 9.212, 9.244], - [0.265, 0.483, 0.427], - [3.099, 0.532, 0.583], - [6.788, 0.614, 0.632], - [5.628, 1.987, 2.02], - [10.087, 2.23, 2.229], - [10.076, 2.293, 2.305], - [0.759, 0.643, 0.645], - [0.171, 0.097, 0.084], - [0.123, 0.082, 0.094], - [0.122, 0.048, 0.049], - [0.366, 0.188, 0.17], - [0.075, 0.021, 0.019], - [0.07, 0.019, 0.018], - [0.078, 0.033, 0.041] + [0.089,0.002,0.001], + [0.075,0.021,0.021], + [0.151,0.053,0.052], + [0.612,0.048,0.047], + [1.053,0.290,0.288], + [0.981,0.470,0.463], + [0.084,0.044,0.043], + [0.080,0.022,0.023], + [0.836,0.413,0.417], + [1.162,0.523,0.525], + [0.551,0.105,0.104], + [1.048,0.125,0.121], + [1.420,0.489,0.495], + [2.601,0.821,0.822], + [0.995,0.540,0.533], + [0.638,0.381,0.378], + [2.589,0.981,1.010], + [2.334,0.725,0.723], + [4.817,1.820,1.854], + [0.153,0.022,0.020], + [10.012,0.862,0.856], + [11.201,0.799,0.801], + [19.869,1.655,1.635], + [3.773,0.407,0.395], + [0.196,0.100,0.106], + [1.095,0.241,0.246], + [0.159,0.077,0.080], + [10.042,0.736,0.718], + [8.992,8.638,8.658], + [0.148,0.052,0.050], + [2.391,0.499,0.504], + [6.177,0.597,0.605], + [5.468,1.868,1.878], + [10.079,2.192,2.200], + [10.088,2.238,2.240], + [0.721,0.594,0.600], + [0.184,0.096,0.103], + [0.122,0.078,0.077], + [0.120,0.047,0.048], + [0.358,0.183,0.193], + [0.078,0.020,0.020], + [0.075,0.020,0.019], + [0.083,0.033,0.031] ] } diff --git a/duckdb-parquet-partitioned/results/c6a.metal.json b/duckdb-parquet-partitioned/results/c6a.metal.json index 4ad5a3c23..c65ed4a42 100644 --- a/duckdb-parquet-partitioned/results/c6a.metal.json +++ b/duckdb-parquet-partitioned/results/c6a.metal.json @@ -1,6 +1,6 @@ { "system": "DuckDB (Parquet, partitioned)", - "date": "2025-08-31", + "date": "2025-09-16", "machine": "c6a.metal", "cluster_size": 1, "proprietary": "no", @@ -9,49 +9,49 @@ "load_time": 0, "data_size": 14737666736, "result": [ - [0.11, 0.002, 0.001], - [0.069, 0.019, 0.018], - [0.103, 0.021, 0.022], - [0.65, 0.023, 0.021], - [1.325, 0.149, 0.144], - [1.509, 0.2, 0.202], - [0.07, 0.019, 0.018], - [0.086, 0.021, 0.022], - [1.243, 0.206, 0.247], - [1.649, 0.225, 0.207], - [1.017, 0.047, 0.043], - [1.327, 0.053, 0.048], - [1.677, 0.211, 0.184], - [2.859, 0.379, 0.396], - [1.23, 0.195, 0.204], - [1.066, 0.287, 0.17], - [2.776, 0.364, 0.389], - [2.593, 0.394, 0.399], - [4.376, 0.576, 0.558], - [0.127, 0.017, 0.016], - [10.186, 0.233, 0.205], - [11.706, 0.265, 0.227], - [20.007, 0.77, 0.518], - [7.166, 0.322, 0.324], - [0.365, 0.326, 0.085], - [1.672, 0.089, 0.09], - [0.612, 0.389, 0.055], - [10.28, 0.229, 0.196], - [9.226, 1.984, 1.796], - [0.085, 0.026, 0.026], - [2.614, 0.206, 0.178], - [6.363, 0.23, 0.238], - [5.08, 1.085, 0.72], - [9.784, 1.334, 0.784], - [9.927, 0.935, 0.687], - [0.373, 0.386, 0.274], - [0.163, 0.098, 0.096], - [0.136, 0.078, 0.077], - [0.141, 0.046, 0.046], - [0.322, 0.185, 0.18], - [0.106, 0.021, 0.019], - [0.105, 0.023, 0.02], - [0.107, 0.032, 0.032] + [0.113,0.002,0.001], + [0.089,0.022,0.021], + [0.112,0.026,0.024], + [0.313,0.028,0.026], + [0.971,0.101,0.093], + [0.939,0.179,0.177], + [0.082,0.020,0.020], + [0.108,0.028,0.027], + [0.607,0.135,0.115], + [1.037,0.143,0.144], + [0.392,0.056,0.048], + [0.880,0.055,0.050], + [1.185,0.195,0.179], + [2.083,0.243,0.209], + [0.832,0.211,0.189], + [0.569,0.119,0.105], + [2.075,0.306,0.223], + [2.079,0.242,0.284], + [3.927,0.364,0.351], + [0.119,0.019,0.021], + [9.635,0.213,0.202], + [10.934,0.252,0.174], + [19.563,0.605,0.321], + [3.659,2.772,0.332], + [0.190,0.083,0.083], + [0.892,0.078,0.069], + [0.193,0.325,0.056], + [9.914,0.217,0.187], + [8.596,1.523,1.558], + [0.104,0.032,0.027], + [2.117,0.142,0.132], + [5.671,0.205,0.166], + [4.357,0.509,0.457], + [9.688,0.545,0.522], + [9.498,0.724,0.522], + [0.422,0.169,0.337], + [0.196,0.103,0.101], + [0.142,0.076,0.077], + [0.152,0.050,0.051], + [0.336,0.208,0.193], + [0.115,0.024,0.027], + [0.117,0.029,0.028], + [0.111,0.032,0.031] ] } diff --git a/duckdb-parquet/results/c6a.4xlarge.json b/duckdb-parquet/results/c6a.4xlarge.json index f506364a7..a36d646ba 100644 --- a/duckdb-parquet/results/c6a.4xlarge.json +++ b/duckdb-parquet/results/c6a.4xlarge.json @@ -1,6 +1,6 @@ { "system": "DuckDB (Parquet, single)", - "date": "2025-08-31", + "date": "2025-09-16", "machine": "c6a.4xlarge", "cluster_size": 1, "proprietary": "no", @@ -9,49 +9,49 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.07, 0.014, 0.013], - [0.112, 0.032, 0.03], - [0.162, 0.057, 0.056], - [0.662, 0.057, 0.057], - [1.609, 0.35, 0.353], - [1.455, 0.485, 0.481], - [0.117, 0.053, 0.051], - [0.116, 0.032, 0.033], - [1.015, 0.457, 0.468], - [1.667, 0.567, 0.581], - [0.559, 0.115, 0.114], - [0.92, 0.137, 0.138], - [1.286, 0.478, 0.483], - [2.438, 0.766, 0.766], - [0.962, 0.53, 0.526], - [0.588, 0.412, 0.418], - [2.417, 0.979, 0.99], - [2.211, 0.781, 0.783], - [4.481, 1.688, 1.696], - [0.181, 0.034, 0.033], - [9.992, 0.88, 0.893], - [11.149, 0.827, 0.811], - [19.695, 1.706, 1.712], - [2.726, 0.487, 0.436], - [0.24, 0.14, 0.132], - [0.941, 0.263, 0.267], - [0.229, 0.119, 0.115], - [9.868, 0.762, 0.742], - [9.523, 9.235, 9.217], - [0.148, 0.054, 0.054], - [2.303, 0.538, 0.537], - [5.888, 0.613, 0.617], - [5.326, 1.904, 1.963], - [10.002, 2.194, 2.167], - [10.015, 2.209, 2.241], - [0.669, 0.53, 0.537], - [0.237, 0.123, 0.122], - [0.19, 0.102, 0.117], - [0.181, 0.07, 0.066], - [0.442, 0.231, 0.234], - [0.152, 0.038, 0.035], - [0.12, 0.039, 0.035], - [0.132, 0.048, 0.049] + [0.080,0.017,0.014], + [0.119,0.035,0.035], + [0.176,0.063,0.063], + [0.436,0.058,0.058], + [1.092,0.286,0.289], + [0.946,0.497,0.491], + [0.119,0.051,0.047], + [0.118,0.037,0.034], + [0.797,0.408,0.403], + [1.109,0.508,0.516], + [0.511,0.119,0.118], + [0.943,0.140,0.136], + [1.281,0.483,0.496], + [2.459,0.813,0.813], + [0.964,0.545,0.540], + [0.744,0.347,0.349], + [2.440,0.939,0.942], + [2.207,0.729,0.756], + [4.571,1.738,1.772], + [0.215,0.037,0.037], + [9.918,0.896,0.906], + [11.142,0.811,0.819], + [20.084,1.688,1.676], + [2.671,0.420,0.428], + [0.257,0.138,0.138], + [0.964,0.257,0.259], + [0.228,0.117,0.117], + [9.877,0.751,0.757], + [9.282,8.933,8.840], + [0.169,0.059,0.059], + [2.293,0.520,0.522], + [5.891,0.619,0.615], + [5.307,1.820,1.871], + [10.055,2.215,2.205], + [10.076,2.240,2.258], + [0.646,0.460,0.470], + [0.246,0.128,0.128], + [0.190,0.104,0.103], + [0.191,0.072,0.071], + [0.453,0.241,0.239], + [0.155,0.040,0.040], + [0.136,0.041,0.039], + [0.145,0.049,0.055] ] } diff --git a/duckdb-parquet/results/c6a.metal.json b/duckdb-parquet/results/c6a.metal.json index 4c4be99b2..d2d30a405 100644 --- a/duckdb-parquet/results/c6a.metal.json +++ b/duckdb-parquet/results/c6a.metal.json @@ -1,6 +1,6 @@ { "system": "DuckDB (Parquet, single)", - "date": "2025-08-28", + "date": "2025-09-16", "machine": "c6a.metal", "cluster_size": 1, "proprietary": "no", @@ -9,48 +9,48 @@ "load_time": 0, "data_size": 14779976446, "result": [ - [0.067, 0.015, 0.013], - [0.112, 0.025, 0.022], - [0.138, 0.03, 0.031], - [0.279, 0.035, 0.033], - [0.968, 0.201, 0.18], - [0.757, 0.234, 0.221], - [0.107, 0.028, 0.024], - [0.119, 0.026, 0.024], - [0.659, 0.257, 0.215], - [0.953, 0.272, 0.242], - [0.357, 0.076, 0.069], - [0.78, 0.087, 0.075], - [1.137, 0.214, 0.192], - [2.158, 0.388, 0.376], - [0.843, 0.226, 0.188], - [0.415, 0.304, 0.232], - [2.153, 0.396, 0.596], - [2.154, 0.523, 0.468], - [3.948, 0.667, 0.654], - [0.181, 0.03, 0.028], - [9.581, 0.288, 0.282], - [10.958, 0.255, 0.233], - [19.827, 0.687, 0.376], - [10.845, 0.437, 0.457], - [2.081, 0.106, 0.102], - [0.948, 0.096, 0.102], - [2.421, 0.083, 0.08], - [9.775, 0.253, 0.244], - [8.379, 1.848, 1.883], - [0.149, 0.037, 0.034], - [2.072, 0.222, 0.201], - [5.465, 0.292, 0.258], - [4.261, 0.779, 0.617], - [9.955, 0.981, 0.791], - [10.033, 1.277, 0.837], - [0.387, 0.359, 0.261], - [0.246, 0.124, 0.12], - [0.197, 0.106, 0.1], - [0.197, 0.068, 0.068], - [0.393, 0.217, 0.219], - [0.157, 0.037, 0.039], - [0.137, 0.035, 0.034], - [0.129, 0.053, 0.045] + [0.073,0.013,0.013], + [0.151,0.032,0.027], + [0.136,0.033,0.040], + [0.287,0.041,0.037], + [0.884,0.154,0.185], + [0.781,0.230,0.217], + [0.115,0.050,0.033], + [0.119,0.040,0.028], + [0.572,0.143,0.135], + [0.945,0.161,0.165], + [0.361,0.066,0.058], + [0.808,0.105,0.065], + [1.061,0.216,0.208], + [1.999,0.341,0.394], + [0.817,0.251,0.215], + [0.446,0.118,0.117], + [2.002,0.242,0.245], + [1.991,0.243,0.263], + [3.766,0.401,0.393], + [0.168,0.033,0.032], + [9.652,0.213,0.215], + [10.966,0.283,0.192], + [19.854,0.440,0.305], + [10.978,0.785,0.395], + [2.094,0.107,0.105], + [0.950,0.104,0.112], + [2.424,0.105,0.084], + [9.711,0.205,0.187], + [8.417,1.579,1.511], + [0.156,0.037,0.038], + [2.036,0.239,0.170], + [5.457,0.222,0.176], + [4.230,0.543,0.493], + [9.613,0.527,0.513], + [9.597,0.523,0.480], + [0.633,0.148,0.129], + [0.243,0.126,0.124], + [0.197,0.101,0.099], + [0.216,0.068,0.074], + [0.429,0.235,0.230], + [0.196,0.039,0.036], + [0.166,0.039,0.037], + [0.178,0.049,0.045] ] } diff --git a/duckdb/benchmark.sh b/duckdb/benchmark.sh index 52e1a586d..7042b968f 100755 --- a/duckdb/benchmark.sh +++ b/duckdb/benchmark.sh @@ -6,12 +6,10 @@ curl https://install.duckdb.org | sh export PATH=$HOME'/.duckdb/cli/latest':$PATH # Load the data -sudo apt-get install -y pigz -wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' -pigz -d -f hits.tsv.gz +wget --continue --progress=dot:giga 'https://datasets.clickhouse.com/hits_compatible/hits.parquet' echo -n "Load time: " -command time -f '%e' duckdb hits.db -f create.sql -c "COPY hits FROM 'hits.tsv' (QUOTE '')" +command time -f '%e' duckdb hits.db -storage_version latest -f create.sql -f load.sql # Run the queries diff --git a/duckdb/load.sql b/duckdb/load.sql new file mode 100644 index 000000000..24891835f --- /dev/null +++ b/duckdb/load.sql @@ -0,0 +1,7 @@ +INSERT INTO hits +SELECT * REPLACE ( + make_date(EventDate) AS EventDate, + epoch_ms(EventTime * 1000) AS EventTime, + epoch_ms(ClientEventTime * 1000) AS ClientEventTime, + epoch_ms(LocalEventTime * 1000) AS LocalEventTime) +FROM read_parquet('hits.parquet', binary_as_string=True); diff --git a/duckdb/results/c6a.4xlarge.json b/duckdb/results/c6a.4xlarge.json index ebb27a1ba..738ef07df 100644 --- a/duckdb/results/c6a.4xlarge.json +++ b/duckdb/results/c6a.4xlarge.json @@ -1,57 +1,57 @@ { "system": "DuckDB", - "date": "2025-08-31", + "date": "2025-09-16", "machine": "c6a.4xlarge", "cluster_size": 1, "proprietary": "no", "tuned": "no", "tags": ["C++","column-oriented","embedded"], - "load_time": 380, - "data_size": 23597953024, + "load_time": 110.78, + "data_size": 20549742592, "result": [ - [0.027, 0.001, 0], - [0.119, 0.004, 0.003], - [0.961, 0.018, 0.019], - [1.059, 0.031, 0.03], - [1.16, 0.321, 0.325], - [1.704, 0.4, 0.403], - [0.093, 0.01, 0.005], - [0.117, 0.005, 0.004], - [1.781, 0.41, 0.41], - [2.712, 0.57, 0.569], - [1.564, 0.134, 0.134], - [2.049, 0.144, 0.142], - [2.014, 0.366, 0.36], - [3.27, 0.624, 0.612], - [2.04, 0.379, 0.383], - [0.606, 0.386, 0.39], - [3.232, 0.849, 0.843], - [3.008, 0.613, 0.623], - [5.937, 1.444, 1.446], - [0.127, 0.004, 0.003], - [18.539, 0.596, 0.598], - [20.528, 0.581, 0.586], - [34.375, 0.908, 0.934], - [2.075, 0.536, 0.114], - [0.165, 0.025, 0.024], - [0.143, 0.035, 0.035], - [0.446, 0.024, 0.023], - [19.005, 0.421, 0.46], - [13.45, 8.924, 8.912], - [0.153, 0.025, 0.024], - [5.058, 0.341, 0.348], - [7.847, 0.397, 0.419], - [6.239, 1.757, 1.776], - [18.42, 1.821, 1.842], - [18.422, 1.903, 1.936], - [0.642, 0.461, 0.465], - [0.122, 0.029, 0.027], - [0.092, 0.016, 0.015], - [0.094, 0.016, 0.015], - [0.178, 0.06, 0.15], - [0.097, 0.005, 0.004], - [0.077, 0.006, 0.005], - [0.076, 0.011, 0.01] + [0.026,0.000,0.000], + [0.170,0.003,0.003], + [1.021,0.018,0.019], + [1.130,0.032,0.031], + [1.150,0.243,0.246], + [1.213,0.282,0.275], + [0.107,0.005,0.004], + [0.152,0.004,0.004], + [1.851,0.335,0.339], + [2.747,0.495,0.500], + [1.329,0.066,0.065], + [2.160,0.075,0.073], + [1.532,0.306,0.310], + [2.671,0.622,0.612], + [1.749,0.351,0.351], + [0.530,0.300,0.307], + [2.620,0.737,0.748], + [2.386,0.503,0.503], + [5.381,1.386,1.401], + [0.143,0.004,0.003], + [11.305,0.477,0.474], + [12.437,0.493,0.482], + [14.806,2.351,1.428], + [1.005,0.093,0.094], + [0.155,0.020,0.020], + [1.642,0.066,0.066], + [0.253,0.017,0.016], + [11.837,0.387,0.389], + [8.817,6.559,6.562], + [0.192,0.024,0.023], + [4.869,0.260,0.256], + [7.168,0.330,0.330], + [6.132,1.655,1.694], + [11.221,1.629,1.643], + [11.255,1.796,1.831], + [0.607,0.393,0.393], + [0.110,0.029,0.026], + [0.077,0.007,0.007], + [0.097,0.011,0.012], + [0.175,0.064,0.058], + [0.097,0.006,0.004], + [0.079,0.006,0.006], + [0.081,0.012,0.010] ] } diff --git a/duckdb/results/c6a.metal.json b/duckdb/results/c6a.metal.json index dc77badf3..8d864cd07 100644 --- a/duckdb/results/c6a.metal.json +++ b/duckdb/results/c6a.metal.json @@ -1,56 +1,56 @@ { "system": "DuckDB", - "date": "2025-08-30", + "date": "2025-09-16", "machine": "c6a.metal", "cluster_size": 1, "proprietary": "no", "tuned": "no", "tags": ["C++","column-oriented","embedded"], - "load_time": 157, - "data_size": 23675023360, + "load_time": 102.80, + "data_size": 20557869056, "result": [ - [0.03, 0, 0], - [0.134, 0.005, 0.005], - [2.069, 0.017, 0.009], - [1.641, 0.018, 0.01], - [1.752, 0.17, 0.163], - [2.69, 0.184, 0.172], - [0.112, 0.013, 0.007], - [0.501, 0.008, 0.006], - [2.803, 0.43, 0.252], - [4.271, 0.248, 0.219], - [2.688, 0.071, 0.061], - [3.27, 0.072, 0.057], - [3.072, 0.161, 0.132], - [4.757, 0.295, 0.263], - [3.131, 0.156, 0.13], - [1.138, 0.366, 0.159], - [4.658, 0.352, 0.347], - [4.685, 0.435, 0.444], - [8.063, 0.506, 0.477], - [0.326, 0.005, 0.005], - [23.439, 0.386, 0.108], - [25.916, 0.219, 0.108], - [41.207, 0.209, 0.184], - [4.529, 0.055, 0.056], - [0.887, 0.027, 0.015], - [1.672, 0.131, 0.043], - [1.317, 0.02, 0.123], - [23.813, 0.173, 0.097], - [18.09, 1.215, 1.185], - [0.366, 0.023, 0.024], - [7.472, 0.132, 0.113], - [10.647, 0.267, 0.146], - [7.644, 0.784, 0.649], - [23.038, 1.113, 0.691], - [22.949, 0.68, 0.599], - [0.738, 0.381, 0.292], - [0.111, 0.024, 0.024], - [0.08, 0.012, 0.016], - [0.089, 0.016, 0.013], - [0.158, 0.06, 0.048], - [0.074, 0.005, 0.005], - [0.065, 0.007, 0.006], - [0.066, 0.008, 0.01] + [0.031,0.000,0.001], + [0.122,0.003,0.006], + [2.167,0.007,0.004], + [1.696,0.007,0.006], + [1.755,0.083,0.082], + [2.180,0.105,0.097], + [0.081,0.003,0.002], + [0.586,0.005,0.005], + [2.865,0.107,0.104], + [4.453,0.127,0.127], + [2.397,0.032,0.031], + [3.208,0.034,0.031], + [2.333,0.092,0.084], + [3.972,0.165,0.151], + [2.935,0.133,0.114], + [1.107,0.091,0.090], + [4.125,0.173,0.163], + [3.955,0.179,0.369], + [7.451,0.328,0.317], + [0.353,0.007,0.006], + [16.283,0.272,0.086], + [18.131,0.166,0.078], + [16.737,0.265,0.136], + [2.760,0.692,0.189], + [0.545,0.036,0.017], + [2.577,0.026,0.024], + [1.159,0.049,0.009], + [16.720,0.185,0.070], + [12.607,1.049,1.055], + [0.420,0.011,0.011], + [7.473,0.081,0.075], + [10.388,0.098,0.091], + [8.060,0.452,0.453], + [15.800,0.402,0.317], + [15.934,0.437,0.372], + [0.734,0.104,0.098], + [0.094,0.025,0.025], + [0.061,0.008,0.008], + [0.079,0.015,0.012], + [0.161,0.053,0.050], + [0.091,0.007,0.006], + [0.071,0.008,0.007], + [0.072,0.012,0.011] ] }