Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## Unreleased

### Fixed

* Fixed typo in flag name `--on-module-hash-mistmatch` → `--on-module-hash-mismatch`. The old flag name is still supported for backward compatibility but is deprecated and will be removed in a future version. A deprecation warning will be logged if the old flag is used.

## v4.12.0

### DatabaseChanges mode improvements
Expand Down
21 changes: 19 additions & 2 deletions cmd/substreams-sink-sql/common_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,38 @@ import (
"context"
"time"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/streamingfast/bstream"
"github.com/streamingfast/cli"
"github.com/streamingfast/cli/sflags"
"github.com/streamingfast/shutter"
sink "github.com/streamingfast/substreams-sink"
pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
"go.uber.org/zap"
)

var (
onModuleHashMistmatchFlag = "on-module-hash-mistmatch"
onModuleHashMismatchFlag = "on-module-hash-mismatch" // new, correct spelling
onModuleHashMistmatchFlagDeprecated = "on-module-hash-mistmatch" // old, typo (deprecated)
)

var supportedOutputTypes = "sf.substreams.sink.database.v1.DatabaseChanges,sf.substreams.database.v1.DatabaseChanges"

// AddCommonSinkerFlags adds the flags common to all command that needs to create a sinker,
// namely the `run` and `generate-csv` commands.
func AddCommonSinkerFlags(flags *pflag.FlagSet) {
flags.String(onModuleHashMistmatchFlag, "error", cli.FlagDescription(`
flags.String(onModuleHashMismatchFlag, "error", cli.FlagDescription(`
What to do when the module hash in the manifest does not match the one in the database, can be 'error', 'warn' or 'ignore'

- If 'error' is used (default), it will exit with an error explaining the problem and how to fix it.
- If 'warn' is used, it does the same as 'ignore' but it will log a warning message when it happens.
- If 'ignore' is set, we pick the cursor at the highest block number and use it as the starting point. Subsequent
updates to the cursor will overwrite the module hash in the database.
`))
// Register deprecated flag for backward compatibility
flags.String(onModuleHashMistmatchFlagDeprecated, "error", "(deprecated) use --on-module-hash-mismatch instead")
flags.Lookup(onModuleHashMistmatchFlagDeprecated).Deprecated = "use --on-module-hash-mismatch instead"
}

func AddCommonDatabaseChangesFlags(flags *pflag.FlagSet) {
Expand Down Expand Up @@ -81,3 +87,14 @@ func (a *cliApplication) WaitForTermination(logger *zap.Logger, unreadyPeriodAft
logger.Info("run terminated gracefully")
return nil
}

// resolveOnModuleHashMismatchFlag resolves the on-module-hash-mismatch flag with deprecation support.
func resolveOnModuleHashMismatchFlag(cmd *cobra.Command) string {
// Use deprecated value if set by provider
if value, provided := sflags.MustGetStringProvided(cmd, onModuleHashMistmatchFlagDeprecated); provided {
return value
}

// Deprecated flag wasn't explicitly set, return default from correct flag
return sflags.MustGetString(cmd, onModuleHashMismatchFlag)
}
2 changes: 1 addition & 1 deletion cmd/substreams-sink-sql/generate_csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func generateCsvE(cmd *cobra.Command, args []string) error {
historyTableName,
sflags.MustGetString(cmd, "clickhouse-cluster"),
0, 0, 0,
sflags.MustGetString(cmd, onModuleHashMistmatchFlag),
resolveOnModuleHashMismatchFlag(cmd),
&handleReorgs,
zlog, tracer,
)
Expand Down
2 changes: 1 addition & 1 deletion cmd/substreams-sink-sql/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func sinkRunE(cmd *cobra.Command, args []string) error {
BatchBlockFlushInterval: batchBlockFlushInterval,
BatchRowFlushInterval: batchRowFlushInterval,
LiveBlockFlushInterval: liveBlockFlushInterval,
OnModuleHashMismatch: sflags.MustGetString(cmd, onModuleHashMistmatchFlag),
OnModuleHashMismatch: resolveOnModuleHashMismatchFlag(cmd),
HandleReorgs: handleReorgs,
FlushRetryCount: flushRetryCount,
FlushRetryDelay: flushRetryDelay,
Expand Down
2 changes: 1 addition & 1 deletion cmd/substreams-sink-sql/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func sinkSetupE(cmd *cobra.Command, args []string) error {
CursorTableName: sflags.MustGetString(cmd, "cursors-table"),
HistoryTableName: sflags.MustGetString(cmd, "history-table"),
ClickhouseCluster: sflags.MustGetString(cmd, "clickhouse-cluster"),
OnModuleHashMismatch: sflags.MustGetString(cmd, onModuleHashMistmatchFlag),
OnModuleHashMismatch: resolveOnModuleHashMismatchFlag(cmd),
SystemTablesOnly: sflags.MustGetBool(cmd, "system-tables-only"),
IgnoreDuplicateTableErrors: sflags.MustGetBool(cmd, "ignore-duplicate-table-errors"),
Postgraphile: sflags.MustGetBool(cmd, "postgraphile"),
Expand Down
2 changes: 1 addition & 1 deletion cmd/substreams-sink-sql/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ func toolsCreateLoader(cmd *cobra.Command) (*db2.Loader, error) {
historyTableName,
sflags.MustGetString(cmd, "clickhouse-cluster"),
0, 0, 0,
sflags.MustGetString(cmd, onModuleHashMistmatchFlag),
resolveOnModuleHashMismatchFlag(cmd),
&handleReorgs,
zlog, tracer,
)
Expand Down
6 changes: 3 additions & 3 deletions db_changes/db/cursor.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func (l *Loader) GetAllCursors(ctx context.Context) (out map[string]*sink.Cursor
return out, nil
}

func (l *Loader) GetCursor(ctx context.Context, outputModuleHash string) (cursor *sink.Cursor, mistmatchDetected bool, err error) {
func (l *Loader) GetCursor(ctx context.Context, outputModuleHash string) (cursor *sink.Cursor, mismatchDetected bool, err error) {
cursors, err := l.GetAllCursors(ctx)
if err != nil {
return nil, false, fmt.Errorf("get cursor: %w", err)
Expand All @@ -71,15 +71,15 @@ func (l *Loader) GetCursor(ctx context.Context, outputModuleHash string) (cursor

case OnModuleHashMismatchWarn:
l.logger.Warn(
fmt.Sprintf("cursor module hash mismatch, continuing using cursor at highest block %s, this warning can be made silent by using '--on-module-hash-mistmatch=ignore'", activeCursor.Block()),
fmt.Sprintf("cursor module hash mismatch, continuing using cursor at highest block %s, this warning can be made silent by using '--on-module-hash-mismatch=ignore'", activeCursor.Block()),
zap.String("expected_module_hash", outputModuleHash),
zap.String("actual_module_hash", actualOutputModuleHash),
)

return activeCursor, true, err

case OnModuleHashMismatchError:
return nil, true, fmt.Errorf("cursor module hash mismatch, refusing to continue because flag '--on-module-hash-mistmatch=error' (defaults) is set, you can change to 'warn' or 'ignore': your module's hash is %q but cursor with highest block (%d) module hash is actually %q in the database",
return nil, true, fmt.Errorf("cursor module hash mismatch, refusing to continue because flag '--on-module-hash-mismatch=error' (defaults) is set, you can change to 'warn' or 'ignore': your module's hash is %q but cursor with highest block (%d) module hash is actually %q in the database",
outputModuleHash,
activeCursor.Block().Num(),
actualOutputModuleHash,
Expand Down
Loading