diff --git a/book/src/user_manual/getting_started.md b/book/src/user_manual/getting_started.md index 809f9b2..2bcf57c 100644 --- a/book/src/user_manual/getting_started.md +++ b/book/src/user_manual/getting_started.md @@ -67,6 +67,7 @@ The `config.yaml` file (shown here with default values) is structured as follows general: dataFolder: "~/.khedra/data" # See note 1 + skipMainnetProbe: false # See note 7 chains: mainnet: # Blockchain name (see notes 2, 3, and 4) @@ -133,6 +134,10 @@ logging: 6. When a `scraper` or `monitor` is "catching up" to a chain, the `sleep` value is ignored. +7. The `skipMainnetProbe` value defaults to `false`. When `false`, khedra verifies on startup that the configured mainnet RPC is reachable and returns `chainId == 1`; if it isn't, the installation wizard is presented. Set this to `true` for headless / air-gapped / container / CI deployments (e.g. isolated devnets via [kurtosis `ethereum-package`](https://github.com/ethpandaops/ethereum-package)) where no public mainnet node is available. Even with `skipMainnetProbe: true` the `mainnet` chain block itself must be present, with a non-empty `rpcs` entry and `chainId: 1`, because downstream code reads `mainnet.rpcs[0]` for env wiring. + + The corresponding environment variable override is `TB_KHEDRA_GENERAL_SKIPMAINNETPROBE=true`. + --- ## Using Environment Variables diff --git a/config.yaml.example b/config.yaml.example index d7814ca..f34bab3 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -3,6 +3,7 @@ general: dataFolder: "~/.khedra/data" # See note 1 + skipMainnetProbe: false # See note 7 chains: mainnet: # Blockchain name (see notes 2, 3, and 4) @@ -76,3 +77,5 @@ logging: # 5. The `services` section is required. At least one service must be enabled. # # 6. When a `scraper` or `monitor` is "catching up" to a chain, the `sleep` value is ignored. +# +# 7. The `skipMainnetProbe` value defaults to `false`. When `false`, khedra verifies on startup that the configured mainnet RPC is reachable and returns `chainId == 1`; if it isn't, the installation wizard is presented. Set this to `true` for headless / air-gapped / container / CI deployments (e.g. isolated devnets) where no mainnet node is available. Even with `skipMainnetProbe: true`, the `mainnet` chain block itself must be present and have a non-empty `rpcs` entry and `chainId: 1`, because downstream code reads `mainnet.RPCs[0]` for env wiring. diff --git a/pkg/install/validation.go b/pkg/install/validation.go index d8d5560..d7f2e0d 100644 --- a/pkg/install/validation.go +++ b/pkg/install/validation.go @@ -84,6 +84,7 @@ func ValidateChains(d *Draft) []FieldError { if d == nil { return []FieldError{{Field: "", Code: "internal", Message: "nil draft"}} } + skipProbe := d.Config.General.SkipMainnetProbe hasMainnet := false for name, ch := range d.Config.Chains { // name validation (always) @@ -92,14 +93,21 @@ func ValidateChains(d *Draft) []FieldError { } if name == "mainnet" || ch.ChainID == 1 { hasMainnet = true - if ch.Enabled { - if len(ch.RPCs) == 0 || strings.TrimSpace(firstRPC(ch.RPCs)) == "" { - out = append(out, FieldError{Field: "chains.mainnet.rpc", Code: "mainnet_missing_rpc", Message: "mainnet enabled but RPC missing"}) - } else if !validRPCScheme(firstRPC(ch.RPCs)) { - out = append(out, FieldError{Field: "chains.mainnet.rpc", Code: "invalid_rpc_scheme", Message: "mainnet RPC must start with http(s)://"}) + if ch.ChainID == 0 { + out = append(out, FieldError{Field: "chains.mainnet.chainId", Code: "mainnet_chain_id", Message: "mainnet chainId must be non-zero"}) + } + rpc := firstRPC(ch.RPCs) + if len(ch.RPCs) == 0 || strings.TrimSpace(rpc) == "" { + msg := "mainnet RPC is required" + if ch.Enabled { + msg = "mainnet enabled but RPC missing" } - } else { - out = append(out, FieldError{Field: "chains.mainnet.rpc", Code: "require_mainnet", Message: "mainnet must be enabled"}) + out = append(out, FieldError{Field: "chains.mainnet.rpc", Code: "mainnet_missing_rpc", Message: msg}) + } else if !validRPCScheme(rpc) { + out = append(out, FieldError{Field: "chains.mainnet.rpc", Code: "invalid_rpc_scheme", Message: "mainnet RPC must start with http(s)://"}) + } + if !ch.Enabled && !skipProbe { + out = append(out, FieldError{Field: "chains.mainnet.rpc", Code: "require_mainnet", Message: "mainnet must be enabled (or set general.skipMainnetProbe: true)"}) } } else if ch.Enabled { // non-mainnet enabled if len(ch.RPCs) == 0 || strings.TrimSpace(firstRPC(ch.RPCs)) == "" { diff --git a/pkg/install/validation_test.go b/pkg/install/validation_test.go index 8602a09..43317d2 100644 --- a/pkg/install/validation_test.go +++ b/pkg/install/validation_test.go @@ -53,6 +53,47 @@ func TestValidateChains_MainnetEnabledNoRPC(t *testing.T) { } } +func TestValidateChains_MainnetDisabled_RequireMainnetByDefault(t *testing.T) { + d := newDraft() + ch := d.Config.Chains["mainnet"] + ch.Enabled = false + ch.RPCs = []string{"http://localhost:8545"} + d.Config.Chains["mainnet"] = ch + ferrs := ValidateDraftPhase(d, "step:chains") + if !hasCode(ferrs, "require_mainnet") { + t.Fatalf("expected require_mainnet when mainnet disabled without skipMainnetProbe: %+v", ferrs) + } +} + +func TestValidateChains_MainnetDisabled_SkipProbeAllowed(t *testing.T) { + d := newDraft() + d.Config.General.SkipMainnetProbe = true + ch := d.Config.Chains["mainnet"] + ch.Enabled = false + ch.RPCs = []string{"http://localhost:8545"} + d.Config.Chains["mainnet"] = ch + ferrs := ValidateDraftPhase(d, "step:chains") + if hasCode(ferrs, "require_mainnet") { + t.Fatalf("unexpected require_mainnet with skipMainnetProbe=true: %+v", ferrs) + } + if len(filterCodes(ferrs, "mainnet_missing_rpc")) > 0 { + t.Fatalf("unexpected mainnet_missing_rpc: %+v", ferrs) + } +} + +func TestValidateChains_MainnetDisabled_SkipProbe_RequiresRPC(t *testing.T) { + d := newDraft() + d.Config.General.SkipMainnetProbe = true + ch := d.Config.Chains["mainnet"] + ch.Enabled = false + ch.RPCs = []string{} + d.Config.Chains["mainnet"] = ch + ferrs := ValidateDraftPhase(d, "step:chains") + if !hasCode(ferrs, "mainnet_missing_rpc") { + t.Fatalf("expected mainnet_missing_rpc, got %+v", ferrs) + } +} + func TestValidateChains_InvalidRPCScheme(t *testing.T) { d := newDraft() ch := d.Config.Chains["mainnet"] diff --git a/pkg/install/validity.go b/pkg/install/validity.go index cfb82cc..768b764 100644 --- a/pkg/install/validity.go +++ b/pkg/install/validity.go @@ -8,8 +8,9 @@ import ( coreFile "github.com/TrueBlocks/trueblocks-chifra/v6/pkg/file" "github.com/TrueBlocks/trueblocks-chifra/v6/pkg/logger" "github.com/TrueBlocks/trueblocks-chifra/v6/pkg/rpc" - "github.com/TrueBlocks/trueblocks-khedra/v6/pkg/types" yamlv2 "gopkg.in/yaml.v2" + + "github.com/TrueBlocks/trueblocks-khedra/v6/pkg/types" ) // Cache for mainnet accessibility checks @@ -65,8 +66,11 @@ func checkMainnetAccessible(rpcUrl string) bool { } // Configured returns true only if a config file exists AND contains a mainnet -// configuration with a reachable RPC endpoint that returns chainId == 1. -// Mainnet may be disabled for processing but its RPC must always be accessible. +// configuration. By default mainnet RPC must be reachable and return +// chainId == 1. When general.skipMainnetProbe is true, the reachability probe +// is skipped so the daemon can run in hermetic / air-gapped environments +// (CI, containers, isolated devnets) where no mainnet node is available; +// structural checks (RPC non-empty, chainId non-zero) still apply. // Deep RPC reachability probing is performed here but cached for performance; // this determines whether to present the installation wizard. func Configured() bool { @@ -87,19 +91,22 @@ func Configured() bool { } // Must have a mainnet key if main, ok := cfg.Chains["mainnet"]; ok { - // Mainnet RPC is always required and must return valid data (chainId == 1) if len(main.RPCs) == 0 || strings.TrimSpace(main.RPCs[0]) == "" { logger.Info("mainnet RPC missing but is required") return false } - // Verify mainnet RPC is reachable and returns chainId == 1 (with caching) - if !checkMainnetAccessible(main.RPCs[0]) { + if main.ChainID == 0 { + logger.Info("mainnet chainId zero", "cfgFile", fn, "chain", main, "rawLen", len(b)) return false } - if main.ChainID == 0 { - logger.Info("mainnet chainId zero", "cfgFile", fn, "chain", main, "rawLen", len(b)) + if cfg.General.SkipMainnetProbe { + return true + } + + // Verify mainnet RPC is reachable and returns chainId == 1 (with caching) + if !checkMainnetAccessible(main.RPCs[0]) { return false } return true diff --git a/pkg/install/validity_test.go b/pkg/install/validity_test.go new file mode 100644 index 0000000..7131cda --- /dev/null +++ b/pkg/install/validity_test.go @@ -0,0 +1,160 @@ +package install + +import ( + "os" + "path/filepath" + "testing" +) + +// writeConfig writes a config YAML to a temp file and points +// types.GetConfigFnNoCreate at it via KHEDRA_TEST_CONFIG_FN. +func writeConfig(t *testing.T, yaml string) func() { + t.Helper() + dir := t.TempDir() + fn := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(fn, []byte(yaml), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + prev, hadPrev := os.LookupEnv("KHEDRA_TEST_CONFIG_FN") + if err := os.Setenv("KHEDRA_TEST_CONFIG_FN", fn); err != nil { + t.Fatalf("setenv: %v", err) + } + return func() { + if hadPrev { + _ = os.Setenv("KHEDRA_TEST_CONFIG_FN", prev) + } else { + _ = os.Unsetenv("KHEDRA_TEST_CONFIG_FN") + } + } +} + +// resetAccessibilityCache clears the in-memory cache so test runs don't +// observe stale probe results from prior tests. +func resetAccessibilityCache() { + accessibilityCacheMu.Lock() + defer accessibilityCacheMu.Unlock() + accessibilityCache = map[string]accessibilityCacheEntry{} +} + +// When general.skipMainnetProbe is true the daemon must be able to come up +// without a reachable mainnet RPC — see issue #22 (kurtosis / air-gapped). +func TestConfigured_SkipMainnetProbe_SkipsReachabilityProbe(t *testing.T) { + resetAccessibilityCache() + // Use an obviously unreachable endpoint: a TEST-NET-1 address on a closed + // port. If the probe runs, Configured() will return false. + yaml := `general: + dataFolder: "/tmp/khedra-test" + strategy: "download" + detail: "index" + skipMainnetProbe: true +chains: + mainnet: + rpcs: + - "http://192.0.2.1:1/unreachable" + chainId: 1 + enabled: false + kurtosis: + rpcs: + - "http://localhost:8545" + chainId: 3151908 + enabled: true +` + restore := writeConfig(t, yaml) + defer restore() + + if !Configured() { + t.Fatalf("Configured() = false; expected true when skipMainnetProbe=true even with unreachable mainnet RPC") + } +} + +// Default behavior: skipMainnetProbe omitted (== false) → probe must run. +func TestConfigured_SkipMainnetProbeOmitted_RequiresReachableProbe(t *testing.T) { + resetAccessibilityCache() + yaml := `general: + dataFolder: "/tmp/khedra-test" + strategy: "download" + detail: "index" +chains: + mainnet: + rpcs: + - "http://192.0.2.1:1/unreachable" + chainId: 1 + enabled: true +` + restore := writeConfig(t, yaml) + defer restore() + + if Configured() { + t.Fatalf("Configured() = true; expected false when skipMainnetProbe is omitted and mainnet RPC is unreachable") + } +} + +// skipMainnetProbe explicitly false → probe must run. +func TestConfigured_SkipMainnetProbeFalse_RequiresReachableProbe(t *testing.T) { + resetAccessibilityCache() + yaml := `general: + dataFolder: "/tmp/khedra-test" + strategy: "download" + detail: "index" + skipMainnetProbe: false +chains: + mainnet: + rpcs: + - "http://192.0.2.1:1/unreachable" + chainId: 1 + enabled: true +` + restore := writeConfig(t, yaml) + defer restore() + + if Configured() { + t.Fatalf("Configured() = true; expected false when skipMainnetProbe=false and mainnet RPC is unreachable") + } +} + +// skipMainnetProbe=true with no RPCs at all is still invalid — downstream code +// reads main.RPCs[0] (env wiring in action_daemon.go), so the structural check +// must remain. +func TestConfigured_SkipMainnetProbe_StillRequiresRPC(t *testing.T) { + resetAccessibilityCache() + yaml := `general: + dataFolder: "/tmp/khedra-test" + strategy: "download" + detail: "index" + skipMainnetProbe: true +chains: + mainnet: + rpcs: [] + chainId: 1 + enabled: false +` + restore := writeConfig(t, yaml) + defer restore() + + if Configured() { + t.Fatalf("Configured() = true; expected false when mainnet has no RPCs even with skipMainnetProbe=true") + } +} + +// skipMainnetProbe=true with chainId == 0 is still rejected — malformed YAML. +func TestConfigured_SkipMainnetProbe_StillRequiresChainID(t *testing.T) { + resetAccessibilityCache() + yaml := `general: + dataFolder: "/tmp/khedra-test" + strategy: "download" + detail: "index" + skipMainnetProbe: true +chains: + mainnet: + rpcs: + - "http://localhost:8545" + chainId: 0 + enabled: false +` + restore := writeConfig(t, yaml) + defer restore() + + if Configured() { + t.Fatalf("Configured() = true; expected false when mainnet has chainId=0 even with skipMainnetProbe=true") + } +} diff --git a/pkg/types/apply_env.go b/pkg/types/apply_env.go index a3f5563..6c86661 100644 --- a/pkg/types/apply_env.go +++ b/pkg/types/apply_env.go @@ -18,9 +18,10 @@ const ( PrefixServices = "TB_KHEDRA_SERVICES_" // General Keys - KeyDataFolder = "TB_KHEDRA_GENERAL_DATAFOLDER" - KeyStrategy = "TB_KHEDRA_GENERAL_STRATEGY" - KeyDetail = "TB_KHEDRA_GENERAL_DETAIL" + KeyDataFolder = "TB_KHEDRA_GENERAL_DATAFOLDER" + KeyStrategy = "TB_KHEDRA_GENERAL_STRATEGY" + KeyDetail = "TB_KHEDRA_GENERAL_DETAIL" + KeySkipMainnetProbe = "TB_KHEDRA_GENERAL_SKIPMAINNETPROBE" // Logging Keys KeyLoggingFolder = "TB_KHEDRA_LOGGING_FOLDER" @@ -160,6 +161,12 @@ func applyEnv(keys []string, receiver *Config) error { receiver.General.Strategy = envValue case key == KeyDetail: receiver.General.Detail = envValue + case key == KeySkipMainnetProbe: + skip, err := strconv.ParseBool(envValue) + if err := validateValueParsing(key, err); err != nil { + return err + } + receiver.General.SkipMainnetProbe = skip // Logging settings case key == KeyLoggingFolder: diff --git a/pkg/types/config.go b/pkg/types/config.go index 7d4cc53..82ffc39 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -199,6 +199,7 @@ general: dataFolder: "{{ .General.DataFolder }}" strategy: "{{ .General.Strategy }}" detail: "{{ .General.Detail }}" + skipMainnetProbe: {{ .General.SkipMainnetProbe }} chains: {{- range $key, $value := .Chains }} diff --git a/pkg/types/general.go b/pkg/types/general.go index 0c62f13..8076e7d 100644 --- a/pkg/types/general.go +++ b/pkg/types/general.go @@ -10,9 +10,10 @@ import ( // General represents configuration for data storage, ensuring the data folder is specified, // validated for existence, and serialized for YAML-based configuration management. type General struct { - DataFolder string `koanf:"dataFolder" yaml:"dataFolder" json:"dataFolder,omitempty" validate:"required,folder_exists"` - Strategy string `koanf:"strategy" yaml:"strategy" json:"strategy,omitempty" validate:"oneof=download scratch"` - Detail string `koanf:"detail" yaml:"detail" json:"detail,omitempty" validate:"oneof=index bloom"` + DataFolder string `koanf:"dataFolder" yaml:"dataFolder" json:"dataFolder,omitempty" validate:"required,folder_exists"` + Strategy string `koanf:"strategy" yaml:"strategy" json:"strategy,omitempty" validate:"oneof=download scratch"` + Detail string `koanf:"detail" yaml:"detail" json:"detail,omitempty" validate:"oneof=index bloom"` + SkipMainnetProbe bool `koanf:"skipMainnetProbe" yaml:"skipMainnetProbe" json:"skipMainnetProbe,omitempty"` } func NewGeneral() General { diff --git a/pkg/types/get_env_keys_test.go b/pkg/types/get_env_keys_test.go index 109c897..df82e6e 100644 --- a/pkg/types/get_env_keys_test.go +++ b/pkg/types/get_env_keys_test.go @@ -18,6 +18,7 @@ func TestGetEnvironmentKeys(t *testing.T) { "TB_KHEDRA_GENERAL_DATAFOLDER", "TB_KHEDRA_GENERAL_STRATEGY", "TB_KHEDRA_GENERAL_DETAIL", + "TB_KHEDRA_GENERAL_SKIPMAINNETPROBE", "TB_KHEDRA_CHAINS_MAINNET_ENABLED", "TB_KHEDRA_CHAINS_MAINNET_RPCS", "TB_KHEDRA_CHAINS_MAINNET_CHAINID",