Run gzip with -f

Author: rschu1ze

aurora-mysql/README.md

@@ -8,7 +8,7 @@ pip install --break-system-packages chdb==2.2.0b1
 
 # Load the data
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 ./load.py
 
 # Run the queries

aurora-postgresql/README.md

@@ -7,7 +7,7 @@ sudo apt-get install -y postgresql-client
 sudo docker run -d --name citus -p 5432:5432 -e POSTGRES_PASSWORD=mypass citusdata/citus:11.0
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 echo "*:*:*:*:mypass" > .pgpass
 chmod 400 .pgpass

bigquery/README.md

@@ -41,7 +41,7 @@ clickhouse-client < create"$SUFFIX".sql
 if [ ! -f hits.tsv ]
 then
     wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d hits.tsv.gz
+    gzip -d -f hits.tsv.gz
 fi
 
 clickhouse-client --time --query "INSERT INTO hits FORMAT TSV" < hits.tsv

bytehouse/NOTES.md

@@ -105,7 +105,7 @@ elif [[ $1 == 'test' ]]; then
 	chmod +x /home/gpadmin/run.sh
 	chown gpadmin:gpadmin /home/gpadmin/*
 	if [[ $2 != 'no_dl' ]]; then sudo -iu gpadmin wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'; fi
-	if [[ $2 != 'no_dl' ]]; then sudo -iu gpadmin gzip -d hits.tsv.gz; fi
+	if [[ $2 != 'no_dl' ]]; then sudo -iu gpadmin gzip -d -f hits.tsv.gz; fi
 	sudo -iu gpadmin chmod 777 ~ hits.tsv
 	sudo -iu gpadmin psql -d postgres -f /home/gpadmin/create.sql
 	sudo -iu gpadmin nohup gpfdist &

bytehouse/README.md

@@ -35,7 +35,7 @@ do
 done
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' -O /tmp/hits.tsv.gz
-gzip -d /tmp/hits.tsv.gz
+gzip -d -f /tmp/hits.tsv.gz
 chmod 444 /tmp/hits.tsv
 
 psql -U crate -h localhost --no-password -t < $CREATE_FILE

chdb/benchmark.sh

@@ -89,7 +89,7 @@ mysql -h 127.0.0.1 -P9030 -uroot hits <"$ROOT"/create.sql
 # Download data
 if [[ ! -f hits.tsv.gz ]] && [[ ! -f hits.tsv ]]; then
     wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d hits.tsv.gz
+    gzip -d -f hits.tsv.gz
 fi
 
 # Load data

citus/benchmark.sh

@@ -27,7 +27,7 @@ echo "druid.query.groupBy.maxMergingDictionarySize=5000000000" >> apache-druid-$
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 ./apache-druid-${VERSION}/bin/post-index-task --file ingest.json --url http://localhost:8081
 

clickhouse/benchmark.sh

@@ -9,7 +9,7 @@ pip install --break-system-packages duckdb==1.1.3 psutil
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 
 # Run the queries
 

cloudberry/benchmark.sh

@@ -38,7 +38,7 @@ curl -k -X PUT "https://localhost:9200/hits?pretty" -u "elastic:${PASSWORD}"  -H
 
 # Download and unzip dataset
 wget https://datasets.clickhouse.com/hits_compatible/hits.json.gz
-gzip -d hits.json.gz 
+gzip -d -f hits.json.gz 
 
 # Prepare Elasticsearch for large bulk insert. To do the large upload, you have to break up JSON file into smaller files to prevent 'curl' from OOM while doing it, and adjust ELasticsearch HTTP upload size minimum. This creates roughly 250M files (note it takes a while)
 split -l 10000000 hits.json hits_

cratedb/benchmark.sh

@@ -64,7 +64,7 @@ sudo chmod 777 /gpmaster /gpdata1 /gpdata2 /gpdata3 /gpdata4 /gpdata5 /gpdata6 /
 gpinitsystem -ac gpinitsystem_singlenode
 export MASTER_DATA_DIRECTORY=/gpmaster/gpsne-1/
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 chmod 777 ~ hits.tsv
 psql -d postgres -f create.sql
 nohup gpfdist &

doris/benchmark.sh

@@ -28,7 +28,7 @@ sudo systemctl enable heavydb
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 chmod 777 ~ hits.csv
 
 sudo bash -c "echo 'allowed-import-paths = [\"/home/ubuntu/\"]' > /var/lib/heavyai/heavy.conf_"

druid/benchmark.sh

@@ -5,7 +5,7 @@ sudo apt-get install -y python3-pip
 pip install --break-system-packages tableauhyperapi
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 
 ./load.py
 

duckdb-memory/benchmark.sh

@@ -14,7 +14,7 @@ sudo docker run -it --rm --network host mysql:5 mysql --host 127.0.0.1 --port 50
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 # ERROR 2 (HY000) at line 1: Wrong data or column definition. Row: 93557187, field: 100.
 head -n 90000000 hits.tsv > hits90m.tsv

elasticsearch/benchmark.sh

@@ -15,7 +15,7 @@ sudo apt-get install -y g++ capnproto libclang-14-dev
 cargo build --features "enable_rocksdb" --features "enable_lz4" --release
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 
 target/release/repl --load hits.csv --db-path db
 

greenplum/benchmark.sh

@@ -17,7 +17,7 @@ mysql --password="${PASSWORD}" --host 127.0.0.1 test < create.sql
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 time mysql --password="${PASSWORD}" --host 127.0.0.1 test -e "
     LOAD DATA LOCAL INFILE 'hits.tsv' INTO TABLE hits

heavyai/benchmark.sh

@@ -15,7 +15,7 @@ sudo service mariadb restart
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 sudo mariadb -e "CREATE DATABASE test"
 sudo mariadb test < create.sql

hyper/benchmark.sh

@@ -23,7 +23,7 @@ sudo apt-get install -y expect
 ./query.expect "$(cat create.sql)"
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 chmod 777 ~ hits.tsv
 
 ./query.expect "COPY INTO hits FROM '$(pwd)/hits.tsv' USING DELIMITERS '\t'"

infobright/benchmark.sh

@@ -56,7 +56,7 @@ time mongosh --quiet --eval 'db.hits.createIndex({"ClientIP": 1, "WatchID": 1, "
 #################################
 # Load data and import
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 # Use mongo import to load data into mongo. By default numInsertionWorkers is 1 so change to half of VM where it would be run
 #time mongoimport --collection hits --type tsv hits.tsv --fieldFile=create.txt --columnsHaveTypes --numInsertionWorkers=8

locustdb/benchmark.sh

@@ -10,7 +10,7 @@ sudo service mysql restart
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 sudo mysql -e "CREATE DATABASE test"
 sudo mysql test < create.sql

mariadb-columnstore/benchmark.sh

@@ -10,7 +10,7 @@ sudo service mysql restart
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 sudo mysql -e "CREATE DATABASE test"
 sudo mysql test < create.sql

mariadb/benchmark.sh

@@ -11,7 +11,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential
 echo "Download dataset."
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
 echo "Unpack dataset."
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 mkdir data
 mv hits.csv data
 

monetdb/benchmark.sh

@@ -7,7 +7,7 @@ set -eux
 #sudo apt-get install -y postgresql-client
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 memory=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
 threads=$(nproc)

mongodb/benchmark.sh

@@ -7,7 +7,7 @@ set -eux
 #sudo apt-get install -y postgresql-client
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 memory=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
 threads=$(nproc)

mysql-myisam/benchmark.sh

@@ -45,7 +45,7 @@ fi
 psql -h 127.0.0.1 -U postgres -t < create/"$CREATE_FILE".sql
 
 #get and unpack hits.tsv
-sudo docker exec pgpro_tam bash -c "cd /tmp && wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' && gzip -d hits.tsv.gz"
+sudo docker exec pgpro_tam bash -c "cd /tmp && wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz' && gzip -d -f hits.tsv.gz"
 
 #insert data to table
 if [ "$1" == "parquet_fd_parall" ] ; then

mysql/benchmark.sh

@@ -18,7 +18,7 @@ sleep 30
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 # Pinot was unable to load data as a single file wihout any errors returned. We have to split the data
 split -d --additional-suffix .tsv --verbose -n l/100 hits.tsv parts

oxla/benchmark.sh

@@ -52,7 +52,7 @@ EOF
 sudo systemctl restart postgresql@$PGVERSION-main
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 sudo -u postgres psql -t -c 'CREATE DATABASE test'
 sudo -u postgres psql test -t <create.sql

pg_duckdb-indexed/benchmark.sh

@@ -52,7 +52,7 @@ EOF
 sudo systemctl restart postgresql@$PGVERSION-main
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 sudo -u postgres psql -t -c 'CREATE DATABASE test'
 sudo -u postgres psql test -t <create.sql

pg_duckdb/benchmark.sh

@@ -18,7 +18,7 @@ questdb/bin/questdb.sh start
 # Import the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 
 curl -G --data-urlencode "query=$(cat create.sql)" 'http://localhost:9000/exec'
 

pgpro_tam/benchmark.sh

@@ -90,7 +90,7 @@ mysql -h 127.0.0.1 -P9030 -uroot hits <"$ROOT"/create.sql
 # Download data
 if [[ ! -f hits.tsv.gz ]] && [[ ! -f hits.tsv ]]; then
     wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-    gzip -d hits.tsv.gz
+    gzip -d -f hits.tsv.gz
 fi
 
 # Load data

pinot/benchmark.sh

@@ -15,7 +15,7 @@ cd ..
 
 echo "Download and unzip dataset"
 wget --continue https://datasets.clickhouse.com/hits_compatible/hits.json.gz
-gzip -d hits.json.gz
+gzip -d -f hits.json.gz
 
 # Add the _index line and fix the UserID from string to num and preprocesses the dataset for loading
 python3 fix_hits.py

postgresql-indexed/benchmark.sh

@@ -22,7 +22,7 @@ sudo docker exec -it memsql-ciab memsql -p"${ROOT_PASSWORD}"
 # Load the data
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 sudo docker cp hits.tsv memsql-ciab:/
 
 sudo docker exec -it memsql-ciab memsql -p"${ROOT_PASSWORD}" -e "CREATE DATABASE test"

postgresql/benchmark.sh

@@ -6,7 +6,7 @@ sudo apt-get install -y sqlite3
 sqlite3 mydb < create.sql
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.csv.gz'
-gzip -d hits.csv.gz
+gzip -d -f hits.csv.gz
 
 time sqlite3 mydb '.import --csv hits.csv hits'
 wc -c mydb

questdb/benchmark.sh

@@ -39,7 +39,7 @@ sleep 30
 # Prepare Data
 cd ../
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 
 # Create Table
 mysql -h 127.0.0.1 -P9030 -uroot -e "CREATE DATABASE hits"

selectdb/benchmark.sh

@@ -7,7 +7,7 @@ sudo apt-get update
 sudo apt-get install -y postgresql-client
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 chmod 777 ~ hits.tsv
 
 psql "host=$HOSTNAME port=5432 dbname=csdb user=csuser password=$PASSWORD sslmode=require" < create.sql

siglens/benchmark.sh

@@ -7,7 +7,7 @@ sudo apt-get update
 sudo apt-get install -y postgresql-client
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 chmod 777 ~ hits.tsv
 
 psql postgresql://postgres:$PASSWORD@$HOSTNAME:5432 -t -c 'CREATE DATABASE test'

singlestore/benchmark.sh

@@ -17,7 +17,7 @@ sudo -u postgres psql -c "CREATE DATABASE nocolumnstore"
 sudo -u postgres psql nocolumnstore -c "CREATE EXTENSION timescaledb WITH VERSION '2.17.2';"
 
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 sudo chmod og+rX ~
 chmod 777 hits.tsv
 

sqlite/benchmark.sh

@@ -18,7 +18,7 @@ sudo -u postgres psql test -c "CREATE EXTENSION timescaledb WITH VERSION '2.17.2
 
 # Import the data
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 sudo chmod og+rX ~
 chmod 777 hits.tsv
 

starrocks/benchmark.sh

@@ -9,7 +9,7 @@ sudo apt-get install -y postgresql-client gzip
 
 rm -rf hits.tsv
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
-gzip -d hits.tsv.gz
+gzip -d -f hits.tsv.gz
 chmod 777 hits.tsv
 
 rm -rf umbra-25-01-23.tar.xz umbra