Merge pull request #101 from apecloud/99-launch-slave-mysql

feat: introduce tool for setting up MyDuckServer as a MySQL replica instance

backend/handler.go

@@ -17,16 +17,19 @@ package backend
 import (
 	"context"
 	"fmt"
-
 	"github.com/dolthub/go-mysql-server/server"
 	"github.com/dolthub/vitess/go/mysql"
+	"regexp"
 )
 
 type MyHandler struct {
 	*server.Handler
 	pool *ConnectionPool
 }
 
+// Precompile regex for performance
+var autoIncrementRegex = regexp.MustCompile(`AUTO_INCREMENT=\d+`)
+
 func (h *MyHandler) ConnectionClosed(c *mysql.Conn) {
 	h.pool.CloseConn(c.ConnectionID)
 	h.Handler.ConnectionClosed(c)
@@ -48,6 +51,8 @@ func (h *MyHandler) ComQuery(
 	query string,
 	callback mysql.ResultSpoolFn,
 ) error {
+	query = autoIncrementRegex.ReplaceAllString(query, "")
+
 	return h.Handler.ComQuery(ctx, c, query, callback)
 }
 

devtools/replica-setup/README.md

@@ -0,0 +1,35 @@
+# MyDuckServer MySQL Replica Setup
+
+This guide walks you through creating a MySQL replica using MyDuckServer. Follow the steps outlined here to seamlessly integrate MyDuckServer as a replica of an existing MySQL instance.
+
+## Prerequisites
+
+Before you begin, ensure that:
+
+- **MyDuckServer** is installed and running on your server.
+- You have the necessary **MySQL credentials** (host, port, user, password) to set up replication.
+
+## Getting Started
+
+To create a MySQL replica in MyDuckServer, run the provided `create_replica.sh` script. You will need to supply the MySQL instance connection details as parameters.
+
+### Usage
+
+```bash
+bash create_replica.sh --mysql_host <mysql_host> --mysql_port <mysql_port> --mysql_user <mysql_user> --mysql_password <mysql_password>
+```
+
+### Parameters
+
+- **`--mysql_host`**: The hostname or IP address of the MySQL instance.
+- **`--mysql_port`**: The port on which the MySQL instance is running.
+- **`--mysql_user`**: The MySQL user that has the appropriate privileges for replication.
+- **`--mysql_password`**: The password for the provided MySQL user.
+
+## Example
+
+```bash
+bash create_replica.sh --mysql_host 192.168.1.100 --mysql_port 3306 --mysql_user root --mysql_password mypassword
+```
+
+This command sets up MyDuckServer as a replica of the MySQL instance running at `192.168.1.100` on port `3306` with the user `root` and password `mypassword`.

devtools/replica-setup/create_replica.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Function to display usage
+usage() {
+    echo "Usage: $0 --mysql_host <host> --mysql_port <port> --mysql_user <user> --mysql_password <password>"
+    exit 1
+}
+
+# Parse input parameters
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --mysql_host)
+        MYSQL_HOST="$2"
+        shift # past argument
+        shift # past value
+        ;;
+        --mysql_port)
+        MYSQL_PORT="$2"
+        shift
+        shift
+        ;;
+        --mysql_user)
+        MYSQL_USER="$2"
+        shift
+        shift
+        ;;
+        --mysql_password)
+        MYSQL_PASSWORD="$2"
+        shift
+        shift
+        ;;
+        *)
+        echo "Unknown parameter: $1"
+        usage
+        ;;
+    esac
+done
+
+# Check if all parameters are set
+if [[ -z "$MYSQL_HOST" || -z "$MYSQL_PORT" || -z "$MYSQL_USER" || -z "$MYSQL_PASSWORD" ]]; then
+    echo "Error: All parameters are required."
+    usage
+fi
+
+# Step 1: Check if mysqlsh exists, if not, call install_mysql_shell.sh
+if ! command -v mysqlsh &> /dev/null; then
+    echo "mysqlsh not found, attempting to install..."
+    bash install_mysql_shell.sh
+    if [[ $? -ne 0 ]]; then
+        echo "Failed to install MySQL Shell. Exiting."
+        exit 1
+    fi
+else
+    echo "mysqlsh is already installed."
+fi
+
+# Step 2: Check if Replica of MyDuckServer has already been started
+REPLICA_STATUS=$(mysql -h127.0.0.1 -uroot -P3306 -e "SHOW REPLICA STATUS\G")
+SOURCE_HOST=$(echo "$REPLICA_STATUS" | awk '/Source_Host/ {print $2}')
+
+# Check if Source_Host is not null or empty
+if [[ -n "$SOURCE_HOST" ]]; then
+  echo "Replica has already been started. Source Host: $SOURCE_HOST"
+  exit 1
+else
+  echo "No replica has been started. Proceeding with setup."
+fi
+
+# Step 3: Call start_snapshot.sh with MySQL parameters
+echo "Starting snapshot..."
+source start_snapshot.sh
+if [[ $? -ne 0 ]]; then
+    echo "Failed to start snapshot. Exiting."
+    exit 1
+fi
+
+# Step 4: Call start_delta.sh with MySQL parameters
+echo "Starting delta..."
+source start_delta.sh
+if [[ $? -ne 0 ]]; then
+    echo "Failed to start delta. Exiting."
+    exit 1
+fi
+
+echo "All steps completed successfully."

devtools/replica-setup/install_mysql_shell.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Detect system architecture (x86_64, arm64, etc.)
+ARCH=$(uname -m)
+
+# Detect OS platform (Linux or Darwin for macOS)
+OS=$(uname -s)
+
+# Function to install MySQL Shell on Debian-like Linux systems using apt
+install_mysql_shell_debian() {
+    echo "Installing MySQL Shell on Debian-like system..."
+
+    # Add MySQL APT repository
+    sudo apt-get update
+    sudo apt-get install -y lsb-release wget
+    wget https://dev.mysql.com/get/mysql-apt-config_0.8.22-1_all.deb
+    sudo dpkg -i mysql-apt-config_0.8.22-1_all.deb
+
+    # Install MySQL Shell
+    sudo apt-get update
+    sudo apt-get install -y mysql-shell
+
+    # Clean up the package file
+    rm -f mysql-apt-config_0.8.22-1_all.deb
+}
+
+# Function to install MySQL Shell on Linux using RPM
+install_mysql_shell_linux() {
+    echo "Installing MySQL Shell on Linux (RPM-based)..."
+
+    # Set the base URL for MySQL Shell downloads
+    BASE_URL="https://dev.mysql.com/get/Downloads/MySQL-Shell"
+
+    # Dynamically fetch the latest MySQL Shell version
+    LATEST_VERSION=$(curl -s https://dev.mysql.com/downloads/shell/ | grep -oP '(?<=MySQL Shell )\d+\.\d+\.\d+' | head -n 1)
+
+    # Determine architecture for RPM download
+    if [[ "$ARCH" == "x86_64" ]]; then
+        PACKAGE_NAME="mysql-shell-${LATEST_VERSION}-1.el9.x86_64.rpm"
+    elif [[ "$ARCH" == "aarch64" || "$ARCH" == "arm64" ]]; then
+        PACKAGE_NAME="mysql-shell-${LATEST_VERSION}-1.el9.aarch64.rpm"
+    else
+        echo "Unsupported architecture: $ARCH"
+        exit 1
+    fi
+
+    # Download the RPM package
+    DOWNLOAD_URL="${BASE_URL}/${PACKAGE_NAME}"
+    echo "Downloading MySQL Shell version ${LATEST_VERSION} from $DOWNLOAD_URL..."
+    curl -O "$DOWNLOAD_URL"
+
+    # Install the package
+    echo "Installing MySQL Shell..."
+    sudo dnf install -y ./$PACKAGE_NAME
+
+    # Cleanup the RPM package after installation
+    rm -f ./$PACKAGE_NAME
+}
+
+# Function to install MySQL Shell on macOS
+install_mysql_shell_macos() {
+    echo "Installing MySQL Shell on macOS..."
+
+    # macOS-specific installation method (Homebrew)
+    if [[ "$ARCH" == "x86_64" || "$ARCH" == "arm64" ]]; then
+        # Check if Homebrew is installed
+        if ! command -v brew &> /dev/null; then
+            echo "Homebrew not found. Installing Homebrew..."
+            /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+        fi
+
+        # Install MySQL Shell using Homebrew
+        echo "Installing MySQL Shell using Homebrew..."
+        brew install mysql-shell
+    else
+        echo "Unsupported architecture: $ARCH"
+        exit 1
+    fi
+}
+
+# Main function
+main() {
+    # Determine the platform and install MySQL Shell accordingly
+    if [[ "$OS" == "Linux" ]]; then
+        if command -v apt-get &> /dev/null; then
+            # Debian-like Linux system (e.g., Ubuntu, Debian)
+            install_mysql_shell_debian
+        else
+            # Other Linux system (e.g., RPM-based systems)
+            install_mysql_shell_linux
+        fi
+    elif [[ "$OS" == "Darwin" ]]; then
+        install_mysql_shell_macos
+    else
+        echo "Unsupported operating system: $OS"
+        exit 1
+    fi
+
+    # Verify installation
+    mysqlsh --version
+}
+
+# Run the main function
+main

devtools/replica-setup/start_delta.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Use the EXECUTED_GTID_SET variable from the previous steps
+if [ -z "$EXECUTED_GTID_SET" ]; then
+  echo "Executed_GTID_set is empty, exiting."
+  exit 1
+fi
+
+# Connect to MySQL and execute the replication configuration commands
+mysql -h127.0.0.1 -uroot -P3306 <<EOF
+SET global gtid_purged = "${EXECUTED_GTID_SET}";
+CHANGE REPLICATION SOURCE TO SOURCE_HOST='${MYSQL_HOST}',
+  SOURCE_PORT=${MYSQL_PORT},
+  SOURCE_USER='${MYSQL_USER}',
+  SOURCE_PASSWORD='${MYSQL_PASSWORD}'
+;
+START REPLICA;
+EOF
+
+# Check if the commands were successful
+if [ $? -ne 0 ]; then
+  echo "Failed to start replication. Exiting."
+  exit 1
+else
+  echo "Replication started successfully."
+fi

devtools/replica-setup/start_snapshot.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# 1st step: Run MySQL commands to create 'admin' user and set local_infile
+echo "Creating admin user and setting local_infile..."
+mysql -h127.0.0.1 -uroot -P3306 <<EOF
+CREATE USER 'admin'@'%' IDENTIFIED BY 'admin';
+GRANT ALL PRIVILEGES ON *.* TO 'admin'@'%';
+SET GLOBAL local_infile = 1;
+EOF
+
+if [[ $? -ne 0 ]]; then
+    echo "Failed to create admin user or set local_infile. Exiting."
+    exit 1
+fi
+
+# 2nd step: Get the core count based on cgroup information
+
+# Function to extract core count from cgroup v1 and v2
+get_core_count() {
+    if [[ -f /sys/fs/cgroup/cpu/cpu.cfs_quota_us && -f /sys/fs/cgroup/cpu/cpu.cfs_period_us ]]; then
+        # CGroup v1
+        local quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us)
+        local period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us)
+        if [[ $quota -gt 0 && $period -gt 0 ]]; then
+            echo $(( quota / period ))
+        else
+            # Use available CPU count as a fallback
+            nproc
+        fi
+    elif [[ -f /sys/fs/cgroup/cpu.max ]]; then
+        # CGroup v2
+        local max=$(cat /sys/fs/cgroup/cpu.max | cut -d' ' -f1)
+        local period=$(cat /sys/fs/cgroup/cpu.max | cut -d' ' -f2)
+        if [[ $max != "max" && $period -gt 0 ]]; then
+            echo $(( max / period ))
+        else
+            # Use available CPU count as a fallback
+            nproc
+        fi
+    else
+        # Use available CPU count if cgroup info is unavailable
+        nproc
+    fi
+}
+
+CORE_COUNT=$(get_core_count)
+THREAD_COUNT=$(( 2 * CORE_COUNT ))
+
+echo "Detected core count: $CORE_COUNT"
+echo "Thread count set to: $THREAD_COUNT"
+
+# 3rd step: Execute mysqlsh command
+echo "Starting snapshot copy with mysqlsh..."
+# Run mysqlsh command and capture the output
+output=$(mysqlsh -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASSWORD} -- util copy-instance 'mysql://admin:[email protected]:3306' --exclude-users root --ignore-existing-objects true --handle-grant-errors ignore --threads $THREAD_COUNT --bytesPerChunk 256M)
+
+# Extract the EXECUTED_GTID_SET using grep and awk
+EXECUTED_GTID_SET=$(echo "$output" | grep -i "EXECUTED_GTID_SET" | awk '{print $2}')
+
+# Check if EXECUTED_GTID_SET is empty
+if [ -z "$EXECUTED_GTID_SET" ]; then
+  echo "EXECUTED_GTID_SET is empty, exiting."
+  exit 1
+fi
+
+# If not empty, print the extracted GTID set
+echo "EXECUTED_GTID_SET: $EXECUTED_GTID_SET"
+
+echo "Snapshot completed successfully."