feat: support customized initialization SQL statements (#271)

* fix: refactor the postgresql and duckdb config management

* feat: support customized initialization SQL statements (#268)

README.md

@@ -19,6 +19,7 @@
   - [Installation](#installation)
   - [Usage](#usage)
   - [Replicating Data](#replicating-data)
+  - [Initialize MyDuck Server with Custom SQLs](#initialize-myduck-server-with-custom-sqls)
   - [Connecting to Cloud MySQL & Postgres](#connecting-to-cloud-mysql--postgres)
   - [HTAP Setup](#htap-setup)
   - [Query Parquet Files](#query-parquet-files)
@@ -146,6 +147,19 @@ docker run -d --name myduck \
 > [!NOTE]
 > To replicate from a server running on the host machine, use `host.docker.internal` as the hostname instead of `localhost` or `127.0.0.1`. On Linux, you must also add `--add-host=host.docker.internal:host-gateway` to the `docker run` command.
 
+### Initialize MyDuck Server with Custom SQLs
+
+To initialize MyDuck Server by executing custom SQL statements once it’s ready, you can place the SQL statements in a file with the .sql extension and mount the file to either `/docker-entrypoint-initdb.d/mysql/` or `/docker-entrypoint-initdb.d/postgres/` in the Docker container. The choice between these directories depends on the protocol used for executing the SQL statements.
+
+For instance:
+```bash
+# Execute init.sql on MySQL protocol
+docker run -d -p 13306:3306 -p 15432:5432 --name=myduck -v ./init.sql:/docker-entrypoint-initdb.d/mysql/init.sql apecloud/myduckserver:latest
+
+# Execute init.sql on PostgreSQL protocol
+docker run -d -p 13306:3306 -p 15432:5432 --name=myduck -v ./init.sql:/docker-entrypoint-initdb.d/postgres/init.sql apecloud/myduckserver:latest
+```
+
 ### Connecting to Cloud MySQL & Postgres
 
 MyDuck Server supports setting up replicas from common cloud-based MySQL & Postgres offerings. For more information, please refer to the [replica setup guide](docs/tutorial/replica-setup-rds.md).

catalog/provider.go

@@ -58,7 +58,6 @@ func NewDBProvider(dataDir, dbFile string) (*DatabaseProvider, error) {
 	}
 
 	storage := stdsql.OpenDB(connector)
-	configuration.Init(storage)
 
 	bootQueries := []string{
 		"INSTALL arrow",

docker/Dockerfile

@@ -82,10 +82,11 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
     fi && \
     rm -f /usr/local/mysqlsh.arm64
 
-# Setup user and working directory
+# Setup user and working directory, and create init directories
 RUN useradd --create-home --user-group --shell /bin/bash admin \
     && echo 'admin:admin' | chpasswd \
-    && echo 'admin ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+    && echo 'admin ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+    && mkdir -p /docker-entrypoint-initdb.d/mysql /docker-entrypoint-initdb.d/postgres
 
 USER admin
 WORKDIR /home/admin

docker/entrypoint.sh

@@ -5,6 +5,7 @@ export LOG_PATH="${HOME}/log"
 export MYSQL_REPLICA_SETUP_PATH="${HOME}/replica-setup-mysql"
 export POSTGRES_REPLICA_SETUP_PATH="${HOME}/replica-setup-postgres"
 export PID_FILE="${LOG_PATH}/myduck.pid"
+export INIT_SQLS_DIR="/docker-entrypoint-initdb.d"
 
 parse_dsn() {
     # Check if SOURCE_DSN is set
@@ -126,15 +127,10 @@ run_replica_setup() {
 
 run_server_in_background() {
       cd "$DATA_PATH" || { echo "Error: Could not change directory to ${DATA_PATH}"; exit 1; }
-      nohup myduckserver $LOG_LEVEL >> "${LOG_PATH}"/server.log 2>&1 &
+      nohup myduckserver $LOG_LEVEL $PROFILER_PORT | tee -a "${LOG_PATH}"/server.log 2>&1 &
       echo "$!" > "${PID_FILE}"
 }
 
-run_server_in_foreground() {
-    cd "$DATA_PATH" || { echo "Error: Could not change directory to ${DATA_PATH}"; exit 1; }
-    myduckserver $LOG_LEVEL
-}
-
 wait_for_my_duck_server_ready() {
     local host="127.0.0.1"
     local user="root"
@@ -180,6 +176,28 @@ check_process_alive() {
     fi
 }
 
+execute_init_sqls() {
+    local host="127.0.0.1"
+    local mysql_user="root"
+    local mysql_port="3306"
+    local postgres_user="postgres"
+    local postgres_port="5432"
+    if [ -d "$INIT_SQLS_DIR/mysql" ] && [ "$(find "$INIT_SQLS_DIR/mysql" -maxdepth 1 -name '*.sql' -type f | head -n 1)" ]; then
+        echo "Executing init SQL scripts from $INIT_SQLS_DIR/mysql..."
+        for file in "$INIT_SQLS_DIR/mysql"/*.sql; do
+            echo "Executing $file..."
+            mysqlsh --sql --host "$host" --port "$mysql_port" --user "$mysql_user" --no-password --file="$file"
+        done
+    fi
+    if [ -d "$INIT_SQLS_DIR/postgres" ] && [ "$(find "$INIT_SQLS_DIR/postgres" -maxdepth 1 -name '*.sql' -type f | head -n 1)" ]; then
+        echo "Executing init SQL scripts from $INIT_SQLS_DIR/postgres..."
+        for file in "$INIT_SQLS_DIR/postgres"/*.sql; do
+            echo "Executing $file..."
+            psql -h "$host" -p "$postgres_port" -U "$postgres_user" -f "$file"
+        done
+    fi
+}
+
 # Handle the setup_mode
 setup() {
     # Setup signal handlers
@@ -189,19 +207,26 @@ setup() {
         export LOG_LEVEL="-loglevel $LOG_LEVEL"
     fi
 
+    if [ -n "$PROFILER_PORT" ]; then
+        export PROFILER_PORT="-profiler-port $PROFILER_PORT"
+    fi
+
     # Ensure required directories exist
     mkdir -p "${DATA_PATH}" "${LOG_PATH}"
 
     case "$SETUP_MODE" in
         "" | "SERVER")
             echo "Starting MyDuck Server in SERVER mode..."
-            run_server_in_foreground
+            run_server_in_background
+            wait_for_my_duck_server_ready
+            execute_init_sqls
             ;;
         "REPLICA")
             echo "Starting MyDuck Server in REPLICA mode..."
             parse_dsn
             run_server_in_background
             wait_for_my_duck_server_ready
+            execute_init_sqls
             run_replica_setup
             ;;
         *)
@@ -213,7 +238,7 @@ setup() {
 
 setup
 
-while [[ "$SETUP_MODE" == "REPLICA" ]]; do
+while true; do
     # Check if the processes have started
     if ! check_process_alive "$PID_FILE" "MyDuck Server"; then
         echo "CRITICAL: MyDuck Server process died unexpectedly."

main.go

@@ -23,8 +23,8 @@ import (
 	"github.com/apecloud/myduckserver/catalog"
 	"github.com/apecloud/myduckserver/myfunc"
 	"github.com/apecloud/myduckserver/pgserver"
-	"github.com/apecloud/myduckserver/pgserver/config"
 	"github.com/apecloud/myduckserver/pgserver/logrepl"
+	"github.com/apecloud/myduckserver/pgserver/pgconfig"
 	"github.com/apecloud/myduckserver/plugin"
 	"github.com/apecloud/myduckserver/replica"
 	"github.com/apecloud/myduckserver/transpiler"
@@ -188,7 +188,7 @@ func main() {
 		}
 
 		// Load the configuration for the Postgres server.
-		config.Init()
+		pgconfig.Init()
 		go pgServer.Start()
 	}
 

pgserver/pg_catalog_handler.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/apecloud/myduckserver/adapter"
 	"github.com/apecloud/myduckserver/catalog"
-	duckConfig "github.com/apecloud/myduckserver/configuration"
+	"github.com/apecloud/myduckserver/pgserver/pgconfig"
 	"github.com/cockroachdb/cockroachdb-parser/pkg/sql/sem/tree"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/jackc/pgx/v5/pgproto3"
@@ -199,7 +199,7 @@ func isPgCurrentSetting(query ConvertedQuery) bool {
 	if len(matches) != 3 {
 		return false
 	}
-	if duckConfig.IsValidConfig(matches[2]) {
+	if !pgconfig.IsValidPostgresConfigParameter(matches[2]) {
 		// This is a configuration of DuckDB, it should be bypassed to DuckDB
 		return false
 	}
@@ -290,7 +290,7 @@ var pgCatalogHandlers = map[string]PGCatalogHandler{
 					// Route it to the engine directly.
 					return false, nil
 				}
-				if duckConfig.IsValidConfig(key) {
+				if !pgconfig.IsValidPostgresConfigParameter(key) {
 					// This is a configuration of DuckDB, it should be bypassed to DuckDB
 					return false, nil
 				}
@@ -315,7 +315,7 @@ var pgCatalogHandlers = map[string]PGCatalogHandler{
 				// Route it to the engine directly.
 				return false, nil
 			}
-			if duckConfig.IsValidConfig(key) {
+			if !pgconfig.IsValidPostgresConfigParameter(key) {
 				// This is a configuration of DuckDB, it should be bypassed to DuckDB
 				return false, nil
 			}
@@ -344,7 +344,7 @@ var pgCatalogHandlers = map[string]PGCatalogHandler{
 					return false, fmt.Errorf("error: invalid reset statement: %v", stmt)
 				}
 				key := strings.ToLower(stmt.Name)
-				if duckConfig.IsValidConfig(key) {
+				if !pgconfig.IsValidPostgresConfigParameter(key) {
 					return false, nil
 				}
 				return true, nil
@@ -357,7 +357,7 @@ var pgCatalogHandlers = map[string]PGCatalogHandler{
 				return false, fmt.Errorf("error: invalid reset statement: %v", query.String)
 			}
 			key := strings.ToLower(resetVar.Name)
-			if duckConfig.IsValidConfig(key) {
+			if !pgconfig.IsValidPostgresConfigParameter(key) {
 				// This is a configuration of DuckDB, it should be bypassed to DuckDB
 				return false, nil
 			}

pgserver/config/parameters.go → pgserver/pgconfig/parameters.go

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // Copied from github.com/dolthub/doltgresql/server/config/parameters.go
-package config
+package pgconfig
 
 import (
 	"fmt"

pgserver/config/parameters_list.go → pgserver/pgconfig/parameters_list.go

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // Copied from github.com/dolthub/doltgresql/server/config/parameters_list.go
-package config
+package pgconfig
 
 import (
 	"math"

pgtest/server.go

@@ -10,7 +10,7 @@ import (
 	"github.com/apecloud/myduckserver/backend"
 	"github.com/apecloud/myduckserver/catalog"
 	"github.com/apecloud/myduckserver/pgserver"
-	pgConfig "github.com/apecloud/myduckserver/pgserver/config"
+	pgConfig "github.com/apecloud/myduckserver/pgserver/pgconfig"
 	sqle "github.com/dolthub/go-mysql-server"
 	"github.com/dolthub/go-mysql-server/memory"
 	"github.com/dolthub/go-mysql-server/server"