pg_duckdb: Bring pg_duckdb-indexed scripts up to date

Author: JelteF

pg_duckdb-indexed/benchmark.sh

@@ -1,38 +1,73 @@
 #!/bin/bash
 
-set -ex
+set -eux
 
 #sudo apt-get update
 #sudo apt-get install -y docker.io
 #sudo apt-get install -y postgresql-client
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
 gzip -d hits.tsv.gz
-sudo chmod 777 *
 
-sudo docker run -d --name pgduck -p 5432:5432 -e POSTGRES_PASSWORD=duckdb -v ./:/tmp/files pgduckdb/pgduckdb:16-main
+memory=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
+threads=$(nproc)
+cpus=$(($threads / 2))
+# Shared buffers is set to 25% of memory in AWS RDS by default. We do the same.
+# https://docs.aws.amazon.com/prescriptive-guidance/latest/tuning-postgresql-parameters/shared-buffers.html
+shared_buffers=$(($memory / 4))
+# Effective cache size does not need to be perfect, but it should be somewhat
+# close to the total memory minus what is expected to be used for queries.
+# https://www.cybertec-postgresql.com/en/effective_cache_size-what-it-means-in-postgresql/
+effective_cache_size=$(($memory - ($memory / 4)))
+# By default, max_worker_processes is set to in postgres. We want to be able to
+# use all the threads for parallel workers so we increase it. We also add a
+# small buffer of 15 for any other background workers that might be created.
+max_worker_processes=$(($threads + 15))
+# We also give DuckDB 25% of the memory to work with.
+duckdb_memory=$(($memory / 4))
+# Below we make sure to configure the rest of the parallel worker settings to
+# match the number of cpu cores:
+# https://www.crunchydata.com/blog/postgres-tuning-and-performance-for-analytics-data
+#
+# We also increase work_mem because we are doing an analytics workload to allow
+# some more memory for sorting, aggregations, etc.
+#
+# It's necessary to increase max_wal_size to make the dataload not take very
+# long. With the default value it's constantly checkpointing, and the PG logs
+# warn you about that and tell you to increase max_wal_size.
+
+sudo docker run -d --name pgduck -p 5432:5432 -e POSTGRES_PASSWORD=duckdb pgduckdb/pgduckdb:17-v0.3.1
+
+sleep 2
 
 sudo docker exec -it pgduck bash -c "
-    sed -i -e '
-        s/shared_buffers = 128MB/shared_buffers = 8GB/;
-        s/#max_parallel_workers = 8/max_parallel_workers = 16/;
-        s/#max_parallel_workers_per_gather = 2/max_parallel_workers_per_gather = 8/;
-        s/max_wal_size = 1GB/max_wal_size = 32GB/;
-    ' /var/lib/postgresql/data/postgresql.conf
+cat >> /var/lib/postgresql/data/postgresql.conf <<'EOF'
+shared_buffers=${shared_buffers}kB
+max_worker_processes=${max_worker_processes}
+max_parallel_workers=${threads}
+max_parallel_maintenance_workers=${cpus}
+max_parallel_workers_per_gather=${cpus}
+duckdb.max_workers_per_postgres_scan=${cpus}
+max_wal_size=32GB
+work_mem=64MB
+effective_cache_size=${effective_cache_size}kB
+duckdb.max_memory=${duckdb_memory}kB
+EOF
 "
 
 docker restart pgduck
 
+export PGUSER=postgres
+export PGPASSWORD=duckdb
+
 sleep 5
-docker exec -i pgduck psql -U postgres -c 'CREATE DATABASE test;'
-docker exec -i pgduck psql -U postgres -d test -c 'CREATE EXTENSION IF NOT EXISTS pg_duckdb;'
-docker exec -i pgduck psql -U postgres -d test -f /tmp/files/create.sql
-time docker exec -i pgduck split /tmp/files/hits.tsv --verbose  -n r/$(( $(nproc)/2 )) --filter='psql -U postgres -d test -t -c "\\copy hits (WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID) FROM STDIN"'
-docker exec -i pgduck psql -U postgres -d test -f /tmp/files/index.sql
-docker exec -i pgduck du -bcs /var/lib/postgresql/data
+psql -t <create.sql
+time ./load.sh
 
-docker exec -i pgduck psql -U postgres -d test -c "ALTER DATABASE test SET duckdb.force_execution = true;"
+psql -c "ALTER DATABASE postgres SET duckdb.force_execution = true;"
 ./run.sh 2>&1 | tee log.txt
 
+docker exec -i pgduck du -bcs /var/lib/postgresql/data
+
 cat log.txt | grep -oP 'Time: \d+\.\d+ ms' | sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |
     awk '{ if (i % 3 == 0) { printf "[" }; printf $1 / 1000; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'

pg_duckdb-indexed/index.sql

@@ -17,7 +17,7 @@ CREATE INDEX regionuser on hits (RegionID,UserID);
 CREATE INDEX mobile2 on hits (mobilephonemodel) WHERE mobilephonemodel <> ''::text;
 CREATE INDEX search2 on hits (searchphrase) WHERE searchphrase <> ''::text;
 
-CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
 CREATE INDEX trgm_idx_title ON hits USING gin (title gin_trgm_ops);
 CREATE INDEX trgm_idx_url ON hits USING gin (url gin_trgm_ops);
 

pg_duckdb-indexed/load.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eux
+
+threads=$(nproc)
+cpus=$(($threads / 2))
+
+export PGUSER=postgres
+export PGPASSWORD=duckdb
+
+# Using COPY with explicit column mapping to ensure correct alignment.
+time split hits.tsv --verbose -n r/$cpus --filter='psql -t -c "\\copy hits (WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID) FROM STDIN"'
+
+psql -t -c 'CREATE EXTENSION pg_trgm;'
+time psql -t <index.sql
+
+time psql -t -c 'VACUUM ANALYZE hits'

pg_duckdb-indexed/run.sh

@@ -2,12 +2,16 @@
 
 TRIES=3
 
+export PGUSER=postgres
+export PGPASSWORD=duckdb
+
 cat queries.sql | while read -r query; do
     sync
     echo 3 | sudo tee /proc/sys/vm/drop_caches
 
     echo "$query"
-    for i in $(seq 1 $TRIES); do
-        echo -e "\\\timing\n$query" | docker exec -i pgduck psql -U postgres -d test -t | grep 'Time'
-    done
-done
+    (
+        echo '\timing'
+        yes "$query" | head -n $TRIES
+    ) | psql --no-psqlrc --tuples-only | grep 'Time'
+done