From 0d691f2e2790859480454aa37f49adf7a531efeb Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Tue, 10 Dec 2024 20:02:10 +0800 Subject: [PATCH] feat: skip index creation during loading MySQL snapshot (#278) --- backend/loaddata.go | 10 +++++++++ catalog/database.go | 27 +++++++++++++++++++----- catalog/table.go | 4 ++-- configuration/env.go | 2 +- devtools/replica-setup-mysql/prepare.sh | 1 + devtools/replica-setup-mysql/snapshot.sh | 19 +++++++++++++++-- replica/options.go | 9 ++++++++ 7 files changed, 62 insertions(+), 10 deletions(-) diff --git a/backend/loaddata.go b/backend/loaddata.go index 5012779..b9b9a91 100644 --- a/backend/loaddata.go +++ b/backend/loaddata.go @@ -117,6 +117,16 @@ func (db *DuckBuilder) executeLoadData(ctx *sql.Context, insert *plan.InsertInto b.Grow(256) keyless := sql.IsKeyless(dst.Schema()) + // TODO(fan): This is an ugly hack for MySQL Shell's utilities to work properly. + if !keyless { + if t, ok := dst.(*catalog.Table); ok { + // Replicated tables do not have physical primary keys. + // Their logical primary keys are fake and should not be used in INSERT INTO statements. + // https://github.com/apecloud/myduckserver/issues/272 + keyless = t.ExtraTableInfo().Replicated + } + } + b.WriteString("INSERT") if load.IsIgnore && !keyless { b.WriteString(" OR IGNORE") diff --git a/catalog/database.go b/catalog/database.go index a35ee1f..500e12d 100644 --- a/catalog/database.go +++ b/catalog/database.go @@ -170,11 +170,11 @@ func (d *Database) createAllTable(ctx *sql.Context, name string, schema sql.Prim primaryKeys = append(primaryKeys, schema.Schema[pkord].Name) } + withoutIndex := isIndexCreationDisabled(ctx) + // https://github.com/apecloud/myduckserver/issues/272 - if !(mycontext.IsReplicationQuery(ctx) && configuration.IsReplicationWithoutIndex()) { - if len(primaryKeys) > 0 { - b.WriteString(fmt.Sprintf(", PRIMARY KEY (%s)", strings.Join(primaryKeys, ", "))) - } + if len(primaryKeys) > 0 && !withoutIndex { + b.WriteString(fmt.Sprintf(", PRIMARY KEY (%s)", strings.Join(primaryKeys, ", "))) } b.WriteString(")") @@ -183,7 +183,7 @@ func (d *Database) createAllTable(ctx *sql.Context, name string, schema sql.Prim b.WriteString(fmt.Sprintf( "; COMMENT ON TABLE %s IS '%s'", fullTableName, - NewCommentWithMeta(comment, ExtraTableInfo{schema.PkOrdinals}).Encode(), + NewCommentWithMeta(comment, ExtraTableInfo{schema.PkOrdinals, withoutIndex}).Encode(), )) // Add column comments @@ -205,6 +205,23 @@ func (d *Database) createAllTable(ctx *sql.Context, name string, schema sql.Prim return nil } +func isIndexCreationDisabled(ctx *sql.Context) bool { + if !configuration.IsReplicationWithoutIndex() { + return false + } + if mycontext.IsReplicationQuery(ctx) { + return true + } + _, vv, ok := sql.SystemVariables.GetGlobal("replica_is_loading_snapshot") + if !ok { + return false + } + if b, ok := vv.(int8); ok { + return b != 0 + } + return false +} + // CreateTable implements sql.TableCreator. func (d *Database) CreateTable(ctx *sql.Context, name string, schema sql.PrimaryKeySchema, collation sql.CollationID, comment string) error { d.mu.Lock() diff --git a/catalog/table.go b/catalog/table.go index 75d9715..3cc4936 100644 --- a/catalog/table.go +++ b/catalog/table.go @@ -8,7 +8,6 @@ import ( "github.com/apecloud/myduckserver/adapter" "github.com/apecloud/myduckserver/configuration" - "github.com/apecloud/myduckserver/mycontext" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" "github.com/marcboeker/go-duckdb" @@ -25,6 +24,7 @@ type Table struct { type ExtraTableInfo struct { PkOrdinals []int + Replicated bool } type ColumnInfo struct { @@ -362,7 +362,7 @@ func (t *Table) CreateIndex(ctx *sql.Context, indexDef sql.IndexDef) error { defer t.mu.Unlock() // https://github.com/apecloud/myduckserver/issues/272 - if mycontext.IsReplicationQuery(ctx) && configuration.IsReplicationWithoutIndex() { + if isIndexCreationDisabled(ctx) { return nil } diff --git a/configuration/env.go b/configuration/env.go index ad334ef..c5f38c5 100644 --- a/configuration/env.go +++ b/configuration/env.go @@ -11,7 +11,7 @@ const ( func IsReplicationWithoutIndex() bool { switch strings.ToLower(os.Getenv(replicationWithoutIndex)) { - case "", "t", "1", "true": + case "", "y", "t", "1", "on", "yes", "true": return true } return false diff --git a/devtools/replica-setup-mysql/prepare.sh b/devtools/replica-setup-mysql/prepare.sh index ca7a634..1fdfaf6 100644 --- a/devtools/replica-setup-mysql/prepare.sh +++ b/devtools/replica-setup-mysql/prepare.sh @@ -17,4 +17,5 @@ echo "Setting local_infile and server_id..." mysqlsh --sql --host=${MYDUCK_HOST} --port=${MYDUCK_PORT} --user=root --no-password <