diff --git a/README.md b/README.md index 9e826e6c..d1e3c268 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,8 @@ All configuration comes from environment variables (see `.env.example`): | `RPC_URL` | `https://soroban-testnet.stellar.org` | Stellar RPC endpoint (JSON-RPC 2.0). Point at a provider URL for mainnet. | | `RPC_URLS` | unset | Comma-separated, priority-ordered list of Stellar RPC endpoints. When set, `RPC_URL` is ignored and the multi-provider failover client is used. List order is priority: index 0 is tried first. | | `RPC_RATE_LIMIT_RPS` | `10` | Per-provider request rate limit (`requests/second`) applied to each RPC endpoint independently. Only used when `RPC_URLS` is set. | +| `HORIZON_URL` | `https://horizon-testnet.stellar.org` | Stellar Horizon REST endpoint used by `sorotrail backfill` only. Live ingestion does not touch Horizon. | +| `BACKFILL_RATE_RPS` | `10` | Pace against Horizon when backfilling. 10 req/s is the public-instance cap; private deployments can lift this. | | `DATABASE_URL` | — (required) | Postgres connection string. | | `POLL_INTERVAL` | `5s` | Sleep between polls once caught up. | | `HTTP_ADDR` | `:8080` | API listen address. | @@ -155,6 +157,7 @@ All configuration comes from environment variables (see `.env.example`): | `AUDIT_MAX_REPAIR_ATTEMPTS` | `3` | Repair iterations before a finding is kept open as `unrecoverable`. | | `AUDIT_FINDING_MAX_LEDGERS` | `100` | Largest range a single finding is allowed to span. | | `API_MAX_LIMIT` | `500` | Maximum page size accepted for list endpoints (`/events`, `/subscriptions/{id}/deliveries`). Values above this are rejected with 400. | +| `API_KEY` | empty | Required to use the runtime `/watched-contracts` surface; empty means every request there is rejected with 503. This is a placeholder until #17 (real auth) lands — at that point `API_KEY` will be replaced. | | `RATE_LIMIT_RPS` | unset | Per-client HTTP request rate limit (`requests/second`). Both `RATE_LIMIT_RPS` and `RATE_LIMIT_BURST` must be set together; otherwise no rate limiting is applied. | | `RATE_LIMIT_BURST` | unset | Maximum instantaneous burst size for the rate limiter. Pairs with `RATE_LIMIT_RPS`. | | `RATE_LIMIT_TRUSTED_PROXY` | `false` | Honor `X-Forwarded-For` for client IP detection. Must only be enabled behind a proxy you trust to strip/rewrite the header — clients control `X-Forwarded-For` themselves, so enabling it on an Internet-facing surface lets any caller pick their own rate-limit key. | @@ -643,6 +646,42 @@ Fetch a single event by its ID (the TOID-based identifier from the RPC). Shell curl -s localhost:8080/events/0001099511627776-0000000001 +### `GET /contracts` + +Lists every contract the indexer has seen, with cached token metadata +(name, symbol, decimals) when available. Metadata is `null` for contracts +that haven't been enriched yet or that don't implement the SEP-41 token +interface. + +```sh +curl -s localhost:8080/contracts +``` + +```json +{ + "contracts": [ + {"contract_id":"CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"}, + {"contract_id":"CCW67...","name":"USD Coin","symbol":"USDC","decimals":6} + ] +} +``` + +### `GET /contracts/{id}/stats` + +```sh +curl -s localhost:8080/contracts/CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC/stats +``` + +```json +{ + "contract_id":"CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC", + "name":"USD Coin", + "symbol":"USDC", + "decimals":6, + "event_count":1423 +} +``` + GET /contracts/{id}/events Convenience wrapper for GET /events?contract_id={id}; accepts the same remaining query parameters. @@ -1396,6 +1435,45 @@ be defined with a foreign key to `events(id)` and `ON DELETE CASCADE`. The pruner therefore deletes only from `events` and lets Postgres cascade the dependent rows; today's single-table DELETE is correct because no such table exists yet, and the contract is “derived tables ride along”. +## Contract metadata enrichment + +SoroTrail runs a background worker that enriches contracts with +human-readable token metadata (name, symbol, decimals) by calling the +SEP-41 token interface via RPC simulation. + +**How it works:** + +1. The worker periodically scans the events table for contracts that don't + yet have cached metadata (or whose cached metadata has exceeded the + TTL). +2. For each candidate, it simulates calling `name()`, `symbol()`, and + `decimals()` on the contract via the RPC's `simulateTransaction` + endpoint. +3. Results are stored in the `contract_meta` table with a `fetched_at` + timestamp. +4. Contracts that don't implement the SEP-41 token interface (the + simulation traps or returns an error) are negatively cached so they're + never re-probed. +5. Metadata surfaces in `/contracts` and `/contracts/{id}/stats` responses; + fields are `null` when unknown. + +**Design principles:** + +- **Never blocks ingestion** — the enrichment worker runs on its own + goroutine with its own poll interval. +- **Rate-limited** — simulation calls go through the same RPC client with + the same ~10 req/s limiter, so enrichment doesn't overwhelm the endpoint. +- **Failures are data, not errors** — failed RPC calls are logged at debug + level and the contract is retried on the next pass. +- **Negative caching is permanent** — non-token contracts are cached with + `is_token=false` and never re-fetched, so contracts that emit events but + aren't tokens don't cause wasteful RPC calls. +- **TTL-based refresh** — token metadata is re-fetched after + `CONTRACT_META_TTL` (default 24h) so name/symbol changes (e.g. a token + upgrade) are eventually picked up. + +Disable enrichment entirely with `CONTRACT_META_ENABLED=false`. + ## Caching Stored events are immutable — a row written by ingest is never rewritten diff --git a/cmd/sorotrail/backfill.go b/cmd/sorotrail/backfill.go index 16a88ad6..0036be5c 100644 --- a/cmd/sorotrail/backfill.go +++ b/cmd/sorotrail/backfill.go @@ -119,7 +119,6 @@ flags: minInterval := time.Duration(float64(time.Second) / rpsFinal) st := store.NewPostgres(pool, int64(cfg.PartitionLedgerSpan)) - tokenProc := ingester.NewTokenBalanceProcessor(st, log) hClient := horizon.NewHTTPClient(hURL, minInterval) b := backfill.New(hClient, st, decode.XDRDecoder{}, log, backfill.Options{ @@ -155,20 +154,6 @@ flags: return err } - if !*dryRun && sum.Completed && sum.Extracted > 0 { - // Process token balances for newly backfilled events. We query the - // store for events in the backfilled range and feed them through - // the token balance processor so holders are up-to-date. - log.Info("processing token balances for backfilled events", - "contract_id", *contractID, - "from_ledger", *fromLedger, - "to_ledger", sum.ThroughLedger, - "extracted", sum.Extracted) - if err := processBackfillTokenBalances(ctx, st, tokenProc, *contractID, *fromLedger, sum.ThroughLedger, cfg.DefaultNetworkName()); err != nil { - log.Warn("token balance processing for backfilled events", "error", err) - } - } - printBackfillSummary(sum, *dryRun) if !sum.Completed { return errInterrupted @@ -291,41 +276,6 @@ func contains(haystack, needle string) bool { return false } -// processBackfillTokenBalances reads events for the backfilled range and feeds -// them through the token balance processor so holder balances stay accurate. -func processBackfillTokenBalances(ctx context.Context, st store.Store, proc *ingester.TokenBalanceProcessor, contractID string, fromLedger, toLedger int64, network string) error { - if toLedger <= 0 { - return nil - } - // Query events in batches to avoid loading everything into memory. - // We use the store's QueryEvents with order=asc to page through. - var cursor string - limit := 500 - for { - events, next, err := st.QueryEvents(ctx, store.EventFilter{ - ContractID: contractID, - Network: network, - FromLedger: fromLedger, - ToLedger: toLedger, - Limit: limit, - Order: "asc", - Cursor: cursor, - }) - if err != nil { - return fmt.Errorf("querying backfilled events: %w", err) - } - if len(events) == 0 { - break - } - proc.NotifyEvents(ctx, events) - if next == "" || len(events) < limit { - break - } - cursor = next - } - return nil -} - // printBackfillSummary mirrors replay's printReplaySummary shape so // operator scripts see consistent backfill output. func printBackfillSummary(s backfill.Summary, dryRun bool) { diff --git a/cmd/sorotrail/main.go b/cmd/sorotrail/main.go index e607b69e..3c9c694a 100644 --- a/cmd/sorotrail/main.go +++ b/cmd/sorotrail/main.go @@ -20,7 +20,6 @@ import ( "time" "github.com/jackc/pgx/v5/pgxpool" - _ "modernc.org/sqlite" "github.com/sorotrail/sorotrail/internal/api" "github.com/sorotrail/sorotrail/internal/api/graphql" @@ -33,18 +32,10 @@ import ( "github.com/sorotrail/sorotrail/internal/rpc" "github.com/sorotrail/sorotrail/internal/spec" "github.com/sorotrail/sorotrail/internal/store" + "github.com/sorotrail/sorotrail/internal/telemetry" "github.com/sorotrail/sorotrail/internal/webhook" ) -// compositeNotifier fans out to multiple EventNotifiers. -type compositeNotifier []ingester.EventNotifier - -func (n compositeNotifier) NotifyEvents(ctx context.Context, events []store.Event) { - for _, notifier := range n { - notifier.NotifyEvents(ctx, events) - } -} - var errInterrupted = errors.New("interrupted") func main() { @@ -59,6 +50,8 @@ func main() { } } +// dispatch routes to a subcommand, defaulting to the indexer so existing +// deployments (and the Dockerfile entrypoint) keep working unchanged. func dispatch(args []string) error { if len(args) == 0 { return run() @@ -188,54 +181,19 @@ func run() error { st = pg } - // Tag any events that have empty network with the default network. - // This handles the upgrade path for single-network deployments. - if pool != nil { - defaultNetwork := cfg.DefaultNetworkName() - if defaultNetwork == "" { - networks := cfg.NetworksOrDefault() - if len(networks) > 0 { - defaultNetwork = networks[0].Name - } - } - if defaultNetwork != "" { - if _, err := pool.Exec(ctx, `UPDATE events SET network = $1 WHERE network = '' OR network IS NULL`, defaultNetwork); err != nil { - log.Warn("tagging legacy events with default network", "error", err) - } - if _, err := pool.Exec(ctx, `INSERT INTO ingestion_state (network, last_ingested_ledger, last_cursor, updated_at) - SELECT $1, last_ingested_ledger, last_cursor, updated_at FROM ingestion_state WHERE network = '' OR network IS NULL - ON CONFLICT (network) DO NOTHING`, defaultNetwork); err != nil { - log.Warn("migrating ingestion state", "error", err) - } - if _, err := pool.Exec(ctx, `INSERT INTO audit_state (network, verified_through_ledger, updated_at) - SELECT $1, verified_through_ledger, updated_at FROM audit_state WHERE network = '' OR network IS NULL - ON CONFLICT (network) DO NOTHING`, defaultNetwork); err != nil { - log.Warn("migrating audit state", "error", err) - } - } - } - for _, id := range cfg.WatchedContracts { - // In multi-network mode, watched contracts apply to all networks. - for _, n := range cfg.NetworksOrDefault() { - if err := st.AddWatchedContract(ctx, id); err != nil { - return err - } - _ = n // preserve for per-network contract lists + if err := st.AddWatchedContract(ctx, id); err != nil { + return err } } // Shared broadcaster for live event streaming across all networks. bcast := broadcast.New(broadcast.DefaultBufferSize) - // Build per-network components. - networks := cfg.NetworksOrDefault() - type networkIngester struct { - ing *ingester.Ingester - auditor *audit.Auditor - rpc rpc.Client - } - ingesters := make([]networkIngester, 0, len(networks)) + rpcClient := rpc.NewHTTPClient(cfg.RPCURL) + wh := webhook.NewNotifier(st, log) + + // Wire the spec cache and enricher for spec-decoded event views. specCache := spec.NewCache(st) specFetcher := spec.NewFetcher(rpcClient) specEnricher := spec.NewEnricher(specFetcher, specCache, log) @@ -340,10 +298,6 @@ func run() error { limiter.Start(ctx) defer limiter.Stop() - // Wire spec enricher for the API using the first network's fetcher. - firstSpecFetcher := spec.NewFetcher(ingesters[0].rpc) - firstSpecEnricher := spec.NewEnricher(firstSpecFetcher, specCache, log) - // Guarded store for API-originated reads with timeout and slow-query logging. apiStore := store.NewGuardedStore(st, store.GuardedStoreOptions{ Timeout: cfg.APIQueryTimeout, @@ -429,7 +383,7 @@ func run() error { if ingesterEnabled { remaining++ // + ingester go func() { - log.Info("ingester starting", "rpc_url", cfg.RPCURL, "poll_interval", cfg.PollInterval) + log.Info("ingester starting", "rpc_urls", rpcURLsForLog(cfg), "poll_interval", cfg.PollInterval) if err := ing.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { errCh <- fmt.Errorf("ingester: %w", err) } else { @@ -446,13 +400,27 @@ func run() error { } }() - // Count expected goroutines that send to errCh. - remaining := 2 + len(ingesters) // http server + webhook + ingesters - for _, ni := range ingesters { - if ni.auditor != nil { - remaining++ - } + // The auditor runs alongside ingestion and reports into the same + // error channel when enabled. + if aud != nil { + remaining++ // + auditor + go func() { + log.Info("auditor starting", + "budget_share", cfg.AuditBudgetShare, + "batch_ledgers", cfg.AuditBatchLedgers, + "lag_threshold", cfg.AuditLagThreshold, + "max_repair_attempts", cfg.AuditMaxRepair) + if err := aud.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + errCh <- fmt.Errorf("auditor: %w", err) + } else { + errCh <- nil + } + }() } + + // The pruner goroutine always runs; without a retention policy it + // returns immediately and reports nil so shutdown accounting holds. + remaining++ // + pruner go func() { if cfg.RetentionEnabled() { log.Info("pruner starting", @@ -470,9 +438,6 @@ func run() error { }() var firstErr error - if aud != nil { - remaining++ - } select { case <-ctx.Done(): log.Info("shutdown signal received") @@ -483,7 +448,6 @@ func run() error { shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout) defer cancel() - broker.Shutdown() if err := server.Shutdown(shutdownCtx); err != nil { log.Error("http shutdown", "error", err) } @@ -513,7 +477,7 @@ func bootstrapAdminKey(ctx context.Context, ts store.TenantStore, key string, lo } prefix, digest, ok := api.ParseAPIKeyForBootstrap(key) if !ok { - return fmt.Errorf("MULTI_TENANT_BOOTSTRAP_KEY is not a valid key; " + + return fmt.Errorf("MULTI_TENANT_BOOTSTRAP_KEY is not a valid key; "+ "generate one with `sorotrail help` format st_<12 chars>_") } tenant, err := ts.GetTenantByName(ctx, "default") diff --git a/go.mod b/go.mod index 0b19510a..4c4f3662 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,13 @@ require ( github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa github.com/jackc/pgx/v5 v5.10.0 github.com/prometheus/client_golang v1.24.1 + github.com/stellar/go v0.0.0-20251210100531-aab2ea4aca88 github.com/stellar/go-stellar-sdk v0.6.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.43.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 + go.opentelemetry.io/otel/sdk v1.44.0 golang.org/x/time v0.15.0 ) @@ -32,7 +35,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -40,11 +43,10 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.19.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -64,65 +66,45 @@ require ( github.com/rogpeppe/go-internal v1.15.0 // indirect github.com/shirou/gopsutil/v4 v4.26.5 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 // indirect + github.com/stellar/go-xdr v0.0.0-20260529210834-0bf8f4956364 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect + github.com/vektah/gqlparser/v2 v2.5.36 github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - github.com/urfave/cli/v3 v3.10.1 - github.com/vektah/gqlparser/v2 v2.5.36 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 + go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 + golang.org/x/crypto v0.54.0 // indirect golang.org/x/sync v0.22.0 - golang.org/x/time v0.15.0 + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.18.1 ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect - github.com/cpuguy83/dockercfg v0.3.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.19.1 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-isatty v0.0.21 // indirect - github.com/moby/term v0.5.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.15.0 // indirect - github.com/stellar/go-xdr v0.0.0-20260529210834-0bf8f4956364 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/mod v0.38.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/tools v0.48.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/grpc v1.74.2 // indirect google.golang.org/protobuf v1.36.11 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/cc/v3 v3.36.3 // indirect diff --git a/go.sum b/go.sum index dc726e44..3aa26c51 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/99designs/gqlgen v0.17.94 h1:+3EUDVgX/8gDyDL+7NUqCo4cy2ylylwW0GvR1dGiEsA= -github.com/99designs/gqlgen v0.17.94/go.mod h1:o+XaAMpPA/AX4rqeiK03tZUb/5T+WCgpRDD4aujgdas= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -16,8 +14,6 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -41,12 +37,6 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= -github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -54,6 +44,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -67,19 +59,21 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= -github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -93,17 +87,12 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -114,31 +103,13 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= -github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= -github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= -github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= -github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= -github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= -github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= -github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= -github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -180,16 +151,6 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= -github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= -github.com/shirou/gopsutil/v4 v4.26.5 h1:RPcBXkpz7kOj9PqGFQOlBPZHsyaPvPVQc098y9RmCNM= -github.com/shirou/gopsutil/v4 v4.26.5/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/stellar/go v0.0.0-20251210100531-aab2ea4aca88 h1:T7CDnX+NSQlu9pxLlxZN0qt6SeUoQ6lxwZjY+Y9Ky54= -github.com/stellar/go v0.0.0-20251210100531-aab2ea4aca88/go.mod h1:pcoYvfcsyFzzSut3RBWF9Ts8g4Z7SWbkb8Hitu7k4BU= -github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE= -github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -203,6 +164,12 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/shirou/gopsutil/v4 v4.26.5 h1:RPcBXkpz7kOj9PqGFQOlBPZHsyaPvPVQc098y9RmCNM= +github.com/shirou/gopsutil/v4 v4.26.5/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stellar/go v0.0.0-20251210100531-aab2ea4aca88 h1:T7CDnX+NSQlu9pxLlxZN0qt6SeUoQ6lxwZjY+Y9Ky54= +github.com/stellar/go v0.0.0-20251210100531-aab2ea4aca88/go.mod h1:pcoYvfcsyFzzSut3RBWF9Ts8g4Z7SWbkb8Hitu7k4BU= github.com/stellar/go-stellar-sdk v0.6.0 h1:NM2oqZJQup0QxnJMq6C8s4iIIhU6rHFX0rlsF3wh/Ho= github.com/stellar/go-stellar-sdk v0.6.0/go.mod h1:IkcqcrE9UQi7n/1y+MxKB+7qzdjG1T2kGOD7Ss8dqjw= github.com/stellar/go-xdr v0.0.0-20260529210834-0bf8f4956364 h1:gOKrfuWdZ92LFlv0TAwgZ7OsWKeBsOMDlGLyFgduI1w= @@ -222,59 +189,13 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= -github.com/xdrpp/goxdr v0.1.1 h1:E1B2c6E8eYhOVyd7yEpOyopzTPirUeF6mVOfXfGyJyc= -github.com/xdrpp/goxdr v0.1.1/go.mod h1:dXo1scL/l6s7iME1gxHWo2XCppbHEKZS7m/KyYWkNzA= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -github.com/urfave/cli/v3 v3.10.1 h1:7Kx9H50hrHbRbyxgO1KP6/BcbiGRz0uYh5YyQ30JEEY= -github.com/urfave/cli/v3 v3.10.1/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= github.com/vektah/gqlparser/v2 v2.5.36 h1:CN9mKVHgMkc+XftdOWIhb4HEL8wKSYkFAqhf8booa7s= github.com/vektah/gqlparser/v2 v2.5.36/go.mod h1:cAJ9qwVgPaUkWv6Gn8vn0mqOE0Ui5Pn56wNy5396XWo= github.com/xdrpp/goxdr v0.1.1 h1:E1B2c6E8eYhOVyd7yEpOyopzTPirUeF6mVOfXfGyJyc= github.com/xdrpp/goxdr v0.1.1/go.mod h1:dXo1scL/l6s7iME1gxHWo2XCppbHEKZS7m/KyYWkNzA= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= @@ -304,6 +225,8 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= @@ -318,11 +241,16 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= @@ -338,6 +266,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -350,8 +284,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= -pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= @@ -403,3 +335,5 @@ modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/api/api_integration_test.go b/internal/api/api_integration_test.go index 0651c80e..24f02032 100644 --- a/internal/api/api_integration_test.go +++ b/internal/api/api_integration_test.go @@ -23,10 +23,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/khaylebfortune/sorotrail/internal/api" - "github.com/khaylebfortune/sorotrail/internal/rpc" - "github.com/khaylebfortune/sorotrail/internal/store" - "github.com/khaylebfortune/sorotrail/internal/testdb" + "github.com/sorotrail/sorotrail/internal/api" + "github.com/sorotrail/sorotrail/internal/rpc" + "github.com/sorotrail/sorotrail/internal/store" + "github.com/sorotrail/sorotrail/internal/testdb" ) const ( diff --git a/internal/api/api_test.go b/internal/api/api_test.go index db3fa97f..9b621f5c 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "log/slog" "net/http" @@ -33,6 +34,7 @@ type stubStore struct { eventByID map[string]store.Event lastFilter store.EventFilter nextCursor string + queryErr error totalCount int64 countEventsErr error @@ -87,20 +89,25 @@ type stubStore struct { deadLettersResult []store.DeadLetter deadLettersCursor string deadLettersErr error -} - - // Watched contract fields - watchedList []store.WatchedContract - watchedListErr error - added []string - removed []string - addErr error - removeErr error - ingestionState *store.IngestionState - ingestionStateEr error + contractCursors map[string]store.ContractCursor +} - pingErr error +func (s *stubStore) QueryEvents(_ context.Context, f store.EventFilter) ([]store.Event, string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.lastFilter = f + if s.queryErr != nil { + return nil, "", s.queryErr + } + n := f.Limit + if n <= 0 { + n = 50 + } + if n > len(s.events) { + n = len(s.events) + } + return s.events[:n], s.nextCursor, nil } func (s *stubStore) CountEvents(_ context.Context, f store.EventFilter) (int64, error) { @@ -119,40 +126,8 @@ func (s *stubStore) AggregateEvents(_ context.Context, f store.EventFilter, buck func (s *stubStore) ReplaceEventsInRange(context.Context, []store.Event, int64, int64) error { return nil } -func (s *stubStore) GetEvent(ctx context.Context, id string) (store.Event, error) { - s.mu.Lock() - defer s.mu.Unlock() - if s.eventByID != nil { - if e, ok := s.eventByID[id]; ok { - return e, nil - } - return store.Event{}, store.ErrNotFound - } - if s.event.ID != "" { - return s.event, nil - } - return store.Event{}, store.ErrNotFound -} -func (s *stubStore) EventExists(_ context.Context, id string) (bool, error) { - s.mu.Lock() - defer s.mu.Unlock() - s.existsCalls++ - s.lastExistsID = id - if s.eventByID != nil { - _, ok := s.eventByID[id] - return ok, nil - } - return s.exists, s.existsErr -} - -// GetIngestionState backs the list-cache frontier lookup. Tests stage -// LastIngestedLedger to drive the boundary decisions (just-below, at, -// and above the frontier). -func (s *stubStore) GetIngestionState(_ context.Context, network string) (store.IngestionState, error) { - if s.ingestionState != nil { - return *s.ingestionState, s.ingestionStateEr - } - return s.ingestion, s.ingestionErr +func (s *stubStore) LedgerRangeCensus(context.Context, int64, int64, bool) ([]store.LedgerCensus, error) { + return nil, nil } func (s *stubStore) SaveIngestionState(ctx context.Context, state store.IngestionState) error { return nil @@ -184,6 +159,10 @@ func (s *stubStore) DeleteEventsBefore(context.Context, int64, time.Time, int) ( return 0, nil } +func (s *stubStore) DeleteEventsBeforeLedger(context.Context, int64) (int64, error) { + return 0, nil +} + func (s *stubStore) GetEvent(context.Context, string, store.Scope) (store.Event, error) { return s.event, s.eventErr } @@ -209,13 +188,6 @@ func (s *stubStore) EventExists(_ context.Context, id string, _ store.Scope) (bo return s.exists, s.existsErr } -func (s *stubStore) GetContractSpec(context.Context, string) ([]byte, error) { - return nil, store.ErrNotFound -} -func (s *stubStore) SetContractSpec(context.Context, string, string, []byte) error { - return nil -} - // GetIngestionState backs the list-cache frontier lookup. Tests stage // LastIngestedLedger to drive the boundary decisions (just-below, at, // and above the frontier). @@ -223,14 +195,7 @@ func (s *stubStore) GetIngestionState(context.Context) (store.IngestionState, er if s.ingestionState != nil { return *s.ingestionState, s.ingestionStateEr } - n := f.Limit - if n <= 0 { - n = 50 - } - if n > len(s.events) { - n = len(s.events) - } - return s.events[:n], s.nextCursor, nil + return s.ingestion, s.ingestionErr } // MigrationVersion backs /readyz's schema check. Tests that need a dirty @@ -314,26 +279,49 @@ func (s *stubStore) CountAddressEvents(_ context.Context, address string) (int64 _ = address return s.addressCount, s.addressCountErr } - -func (s *stubStore) UpsertTokenBalances(ctx context.Context, network string, state store.TokenBalanceState, updates []store.TokenBalanceUpdate) error { +func (s *stubStore) GetAddressSummary(context.Context, string) (store.AddressSummary, error) { + return store.AddressSummary{}, nil +} +func (s *stubStore) UpsertAddressRefs(context.Context, []store.AddressRef) error { return nil } - -func (s *stubStore) GetTokenBalances(ctx context.Context, contractID, network, minBalance string, cursor string, limit int) ([]store.TokenBalance, string, error) { - return nil, "", nil +func (s *stubStore) UpsertEvents(context.Context, []store.Event) (int64, error) { + return 0, nil } -func (s *stubStore) GetTokenBalanceState(ctx context.Context, network, contractID string) (store.TokenBalanceState, error) { - return store.TokenBalanceState{}, store.ErrNotFound +func (s *stubStore) GetContractCursor(_ context.Context, contractID string) (store.ContractCursor, error) { + s.mu.Lock() + defer s.mu.Unlock() + if c, ok := s.contractCursors[contractID]; ok { + return c, nil + } + return store.ContractCursor{}, store.ErrNotFound } - -func (s *stubStore) UpsertTokenBalanceState(ctx context.Context, state store.TokenBalanceState) error { +func (s *stubStore) SaveContractCursor(context.Context, store.ContractCursor) error { + return nil +} +func (s *stubStore) DeleteContractCursor(context.Context, string) error { return nil } +func (s *stubStore) ListContractCursors(context.Context) ([]store.ContractCursor, error) { + return nil, nil +} -func (s *stubStore) GetEarliestLedger(ctx context.Context, network, contractID string) (int64, error) { +func (s *stubStore) ListContractIDs(context.Context) ([]string, error) { + return nil, nil +} +func (s *stubStore) GetContractMeta(context.Context, string) (store.ContractMeta, error) { + return store.ContractMeta{}, store.ErrNotFound +} +func (s *stubStore) UpsertContractMeta(context.Context, store.ContractMeta) error { + return nil +} +func (s *stubStore) CountContractEvents(context.Context, string) (int64, error) { return 0, nil } +func (s *stubStore) ListContractsNeedingRefresh(context.Context, time.Time) ([]string, error) { + return nil, nil +} type stubRPC struct { rpc.Client @@ -350,7 +338,7 @@ func newTestServerWithKey(st *stubStore, rc *stubRPC, apiKey string) *Server { if rc == nil { rc = &stubRPC{health: rpc.Health{Status: "healthy"}} } - return New(st, rc, slog.New(slog.NewTextHandler(io.Discard, nil)), apiKey, 17280) + return New(st, rc, slog.New(slog.NewTextHandler(io.Discard, nil)), apiKey) } func newTestServer(st *stubStore, rc *stubRPC) *Server { @@ -363,6 +351,13 @@ func doGet(t *testing.T, s *Server, path string) (*http.Response, []byte) { return doGetWithHeader(t, s, path, "", "") } +// doGetWithAuth performs a GET against the test server with an api-key +// header, for the API_KEY-gated admin endpoints. +func doGetWithAuth(t *testing.T, s *Server, path, apiKey string) (*http.Response, []byte) { + t.Helper() + return doGetWithHeader(t, s, path, "X-Api-Key", apiKey) +} + func doGetWithHeader(t *testing.T, s *Server, path, key, value string) (*http.Response, []byte) { t.Helper() srv := httptest.NewServer(s.Router()) @@ -1146,7 +1141,6 @@ func TestReadyz(t *testing.T) { } func TestListEvents_FieldsProjection(t *testing.T) { - now := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC) st := &stubStore{ eventByID: map[string]store.Event{ "ev-1": {ID: "ev-1", ContractID: "C1", Ledger: 100}, @@ -1199,21 +1193,6 @@ func TestListEvents(t *testing.T) { }) } -func TestStats(t *testing.T) { - st := &stubStore{stats: store.Stats{TotalEvents: 42, LastIngestedLedger: 999}} - s := newTestServer(st, nil) - handler := s.Router() - - rr := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/stats", nil) - handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusOK, rr.Code) - var got store.Stats - require.NoError(t, json.NewDecoder(rr.Body).Decode(&got)) - assert.Equal(t, int64(42), got.TotalEvents) -} - func TestStats(t *testing.T) { t.Run("includes store and freshness fields", func(t *testing.T) { st := &stubStore{stats: store.Stats{ @@ -1669,7 +1648,7 @@ func TestRequestID(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var buf bytes.Buffer log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})) - s := New(&stubStore{}, nil, log, "test-key", 17280) + s := New(&stubStore{}, nil, log, "test-key") srv := httptest.NewServer(s.Router()) defer srv.Close() diff --git a/internal/api/broker.go b/internal/api/broker.go index 5813493f..f5defdf8 100644 --- a/internal/api/broker.go +++ b/internal/api/broker.go @@ -8,7 +8,7 @@ import ( "encoding/json" "sync" - "github.com/khaylebfortune/sorotrail/internal/store" + "github.com/sorotrail/sorotrail/internal/store" ) // subscriberBufferSize bounds the event channel per subscriber. When a @@ -127,11 +127,21 @@ func (b *Broker) Shutdown() { // eventMatches reports whether e satisfies every constraint in f. Zero-value // fields mean "unconstrained". +// containsType reports whether want appears in types. +func containsType(types []string, want string) bool { + for _, t := range types { + if t == want { + return true + } + } + return false +} + func eventMatches(f store.EventFilter, e store.Event) bool { if f.ContractID != "" && e.ContractID != f.ContractID { return false } - if f.Type != "" && e.Type != f.Type { + if len(f.Types) > 0 && !containsType(f.Types, e.Type) { return false } if f.FromLedger > 0 && e.Ledger < f.FromLedger { diff --git a/internal/api/cache_test.go b/internal/api/cache_test.go index 56bf6278..ee5e0343 100644 --- a/internal/api/cache_test.go +++ b/internal/api/cache_test.go @@ -30,26 +30,6 @@ func (e *stubEnricher) EnrichEvents(_ context.Context, events []store.Event) []s }} } -// doGetWithHeader is doGet plus a header for conditional requests. The -// 304 tests use this so the If-None-Match setup reads naturally without -// the caller constructing http.Request by hand. -func doGetWithHeader(t *testing.T, s *Server, path, header, value string) (*http.Response, []byte) { - t.Helper() - srv := httptest.NewServer(s.Router()) - defer srv.Close() - req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil) - require.NoError(t, err) - if header != "" { - req.Header.Set(header, value) - } - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - resp.Body.Close() - return resp, body -} - // assertImmutable asserts that a response carries the immutable-cache // header set: strong ETag (when expected), Vary: Accept-Encoding, // Cache-Control: public + max-age + immutable. @@ -497,7 +477,6 @@ func TestListETag_CoversEveryFilterField(t *testing.T) { {"Topic2", func(f *store.EventFilter) { f.Topic2 = json.RawMessage(`{"symbol":"transfer"}`) }}, {"Topic3", func(f *store.EventFilter) { f.Topic3 = json.RawMessage(`{"symbol":"transfer"}`) }}, {"TopicContains", func(f *store.EventFilter) { f.TopicContains = json.RawMessage(`[{"u64":7}]`) }}, - {"TopicCount", func(f *store.EventFilter) { n := 2; f.TopicCount = &n }}, {"TxHash", func(f *store.EventFilter) { f.TxHash = "abc123def" }}, {"HasValueTrue", func(f *store.EventFilter) { t := true; f.HasValue = &t }}, {"HasValueFalse", func(f *store.EventFilter) { v := false; f.HasValue = &v }}, @@ -591,7 +570,7 @@ func TestListEvents_TopicFilterCannotReuseAnothersValidator(t *testing.T) { func TestGetEvent_Decoded_Immutable(t *testing.T) { const id = "0001099511627776-0000000001" st := &stubStore{event: store.Event{ID: id, Ledger: 100}} - s := New(st, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), "test-key", 17280, &stubEnricher{}) + s := New(st, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), "test-key", &stubEnricher{}) resp, _ := doGet(t, s, "/events/"+id+"?decoded=true") require.Equal(t, http.StatusOK, resp.StatusCode) @@ -603,7 +582,7 @@ func TestGetEvent_Decoded_Immutable(t *testing.T) { func TestGetEvent_DecodedWithXDR_Immutable(t *testing.T) { const id = "0001099511627776-0000000002" st := &stubStore{event: store.Event{ID: id, Ledger: 100}} - s := New(st, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), "test-key", 17280, &stubEnricher{}) + s := New(st, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), "test-key", &stubEnricher{}) resp, _ := doGet(t, s, "/events/"+id+"?decoded=true&include_xdr=true") require.Equal(t, http.StatusOK, resp.StatusCode) diff --git a/internal/api/export_test.go b/internal/api/export_test.go index 0c5d360f..87a5bdcb 100644 --- a/internal/api/export_test.go +++ b/internal/api/export_test.go @@ -140,11 +140,11 @@ func (f *fakeExportStore) GetIngestionState(context.Context) (store.IngestionSta func (f *fakeExportStore) SaveIngestionState(context.Context, store.IngestionState) error { return nil } -func (f *fakeExportStore) GetAuditState(context.Context) (store.AuditState, error) { +func (f *fakeExportStore) GetAuditState(context.Context, string) (store.AuditState, error) { return store.AuditState{}, store.ErrNotFound } func (f *fakeExportStore) SaveAuditState(context.Context, store.AuditState) error { return nil } -func (f *fakeExportStore) SaveAuditStateIfGreater(context.Context, int64) (store.AuditState, error) { +func (f *fakeExportStore) SaveAuditStateIfGreater(context.Context, string, int64) (store.AuditState, error) { return store.AuditState{}, store.ErrNotFound } func (f *fakeExportStore) ListWatchedContracts(context.Context) ([]store.WatchedContract, error) { @@ -156,7 +156,7 @@ func (f *fakeExportStore) RecordAuditFinding(context.Context, store.AuditFinding return store.AuditFinding{}, nil } func (f *fakeExportStore) UpdateAuditFinding(context.Context, store.AuditFinding) error { return nil } -func (f *fakeExportStore) ListOpenFindingsByRange(context.Context, int64, int64) (store.AuditFinding, error) { +func (f *fakeExportStore) ListOpenFindingsByRange(context.Context, string, int64, int64) (store.AuditFinding, error) { return store.AuditFinding{}, store.ErrNotFound } func (f *fakeExportStore) CreateSubscription(context.Context, store.Subscription) (store.Subscription, error) { @@ -201,7 +201,7 @@ func (f *fakeExportStore) Ping(context.Context) error { return nil } func testServer(t *testing.T, st store.Store, maxRange int64) http.Handler { t.Helper() logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - s := New(st, nil, logger, "", 0) + s := New(st, nil, logger, "") s.SetExportMaxRange(maxRange) return s.Router() } diff --git a/internal/api/handlers.go b/internal/api/handlers.go index d97000e4..fd4567b2 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -28,17 +28,15 @@ import ( "time" - "github.com/khaylebfortune/sorotrail/internal/config" - "github.com/khaylebfortune/sorotrail/internal/metrics" - "github.com/khaylebfortune/sorotrail/internal/store" + "github.com/coder/websocket" + "github.com/go-chi/chi/v5" + "github.com/sorotrail/sorotrail/internal/api/queries" "github.com/sorotrail/sorotrail/internal/broadcast" "github.com/sorotrail/sorotrail/internal/buildinfo" - "github.com/sorotrail/sorotrail/internal/config" - + "github.com/sorotrail/sorotrail/internal/metrics" "github.com/sorotrail/sorotrail/internal/store" - ) @@ -177,38 +175,6 @@ type eventsWithXDRResponse struct { Cursor string `json:"cursor,omitempty"` } -// eventWithXDR is an event plus the raw XDR it was decoded from, returned -// when ?include_xdr=true. ValueXDR is a pointer so an event with no value -// serialises as null rather than an empty string. -type eventWithXDR struct { - - store.Event - - TopicsXDR []string `json:"topics_xdr"` - - ValueXDR *string `json:"value_xdr"` - -} - -// enrichedEventWithXDR combines the raw-XDR view with spec-decoded fields. -type enrichedEventWithXDR struct { - - eventWithXDR - - DecodedEvent *store.DecodedEventResponse `json:"decoded_event,omitempty"` - - Decoded bool `json:"decoded"` - -} - -type enrichedEventsWithXDRResponse struct { - Events []enrichedEventWithXDR `json:"events"` - // Cursor is non-empty when another page exists. - - Cursor string `json:"cursor,omitempty"` - -} - // envelopeResponse is the JSON body returned when ?envelope=true is set on // any paginated list endpoint. It normalises the response shape across all // list endpoints so clients that prefer a consistent outer wrapper don't @@ -483,9 +449,41 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { // handleMetrics serves the Prometheus /metrics endpoint. The response is // always cacheNoStore so scrapers never see a stale snapshot. +func (s *Server) handleLivez(w http.ResponseWriter, r *http.Request) { + writeCacheHeaders(w, cacheNoStore, 0, "") + writeJSON(w, http.StatusOK, healthResponse{Status: "ok", Checks: map[string]string{"process": "ok"}}) +} + +func (s *Server) handleReadyz(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + resp := healthResponse{Status: "ok", Checks: map[string]string{"database": "ok", "rpc": "ok"}} + status := http.StatusOK + + if err := s.store.Ping(ctx); err != nil { + resp.Status = "degraded" + resp.Checks["database"] = err.Error() + status = http.StatusServiceUnavailable + } + if health, err := s.rpc.GetHealth(ctx); err != nil { + resp.Status = "degraded" + resp.Checks["rpc"] = err.Error() + status = http.StatusServiceUnavailable + } else if health.Status != "healthy" { + resp.Status = "degraded" + resp.Checks["rpc"] = fmt.Sprintf("rpc reports %q", health.Status) + status = http.StatusServiceUnavailable + } + writeCacheHeaders(w, cacheNoStore, 0, "") + writeJSON(w, status, resp) +} + func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { writeCacheHeaders(w, cacheNoStore, 0, "") metrics.Handler().ServeHTTP(w, r) +} + // handleDeleteEvents is the admin-only bulk delete endpoint. It deletes all // events whose ledger is strictly less than the ?before_ledger= query parameter. // The endpoint is protected by apiKeyAuth middleware (same as watched-contracts). @@ -510,26 +508,22 @@ func (s *Server) handleDeleteEvents(w http.ResponseWriter, r *http.Request) { return } + deleted, err := s.store.DeleteEventsBeforeLedger(r.Context(), beforeLedger) + if err != nil { + loggerFromContext(r.Context()).Error("bulk delete events", "before_ledger", beforeLedger, "error", err) + writeError(w, http.StatusInternalServerError, errors.New("deleting events failed")) + return + } + writeJSON(w, http.StatusOK, map[string]int64{"deleted": deleted}) +} func (s *Server) handleVersion(w http.ResponseWriter, r *http.Request) { - writeCacheHeaders(w, cacheNoStore, 0, "") - writeJSON(w, http.StatusOK, versionResponse{ - - resp := healthResponse{Status: "ok", Checks: map[string]string{ - "database": "ok", - "rpc": "ok", - "schema_version": "ok", - }} - status := http.StatusOK - + Version: buildinfo.Version, Commit: buildinfo.Commit, - BuildDate: buildinfo.BuildDate, - }) - } @@ -574,6 +568,20 @@ func (s *Server) handleCountEvents(w http.ResponseWriter, r *http.Request) { filter.OrderBy = "" filter.Limit = 0 + total, err := s.store.CountEvents(r.Context(), filter) + if err != nil { + loggerFromContext(r.Context()).Error("counting events", "error", err) + writeError(w, http.StatusInternalServerError, errors.New("counting events failed")) + return + } + writeCacheHeaders(w, cacheNoCache, 0, "") + writeJSON(w, http.StatusOK, countResponse{Count: total}) +} + +// countResponse is the JSON body for GET /events/count. +type countResponse struct { + Count int64 `json:"count"` +} // bucketResponse is the JSON body for GET /events/aggregate. type bucketResponse struct { @@ -637,6 +645,10 @@ func (s *Server) handleAggregateEvents(w http.ResponseWriter, r *http.Request) { const streamBatchSize = 500 +// recentDefaultLimit is the page size applied by ?recent=N shorthand when +// the value is the bare "true" (no explicit count). +const recentDefaultLimit = 20 + func (s *Server) handleListEventsStream(w http.ResponseWriter, r *http.Request) { @@ -1102,10 +1114,12 @@ func (s *Server) handleGetEventRaw(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, errors.New("loading event failed")) return } + if len(event.RawTopicXDR) == 0 && event.RawValueXDR == "" { + writeError(w, http.StatusNotFound, fmt.Errorf("event %q has no stored raw XDR", id)) + return + } - -func (s *Server) handleGetEvent(w http.ResponseWriter, r *http.Request) { - + etag := `"` + id + `:raw"` if ifNoneMatch(r, etag) { exists, err := s.store.EventExists(r.Context(), id, scope) if err != nil { @@ -1121,6 +1135,18 @@ func (s *Server) handleGetEvent(w http.ResponseWriter, r *http.Request) { return } + writeCacheHeaders(w, cacheImmutable, immutableMaxAge, etag) + writeJSON(w, http.StatusOK, rawEventResponse{ + TopicsXDR: event.RawTopicXDR, + ValueXDR: event.RawValueXDR, + }) +} + +// rawEventResponse is the JSON body for GET /events/{id}/raw. +type rawEventResponse struct { + TopicsXDR []string `json:"topics_xdr"` + ValueXDR string `json:"value_xdr,omitempty"` +} // handleGetEventTransaction returns all sibling events from the same // transaction as the event with the given {id}. The referenced event @@ -1146,7 +1172,7 @@ func (s *Server) handleGetEventTransaction(w http.ResponseWriter, r *http.Reques } - event, err := s.store.GetEvent(r.Context(), id, scopeFrom(r.Context())) + event, err := s.store.GetEvent(r.Context(), id, scope) if errors.Is(err, store.ErrNotFound) { writeError(w, http.StatusNotFound, fmt.Errorf("event %q not found", id)) @@ -1729,7 +1755,7 @@ func (s *Server) handleAddWatchedChain(w http.ResponseWriter, r *http.Request) { } - state, err := s.store.GetIngestionState(r.Context(), s.defaultNetwork) + state, err := s.store.GetIngestionState(r.Context()) if err != nil && !errors.Is(err, store.ErrNotFound) { s.log.Error("loading ingestion state for add", "error", err) @@ -1995,7 +2021,7 @@ func (s *Server) listCachePolicy(ctx context.Context, filter store.EventFilter) return cacheNoCache, "", nil } - frontier, err := s.lastIngestedLedger(ctx, filter.Network) + frontier, err := s.lastIngestedLedger(ctx) if err != nil { return cacheNoCache, "", err @@ -2012,8 +2038,8 @@ func (s *Server) listCachePolicy(ctx context.Context, filter store.EventFilter) } // lastIngestedLedger reads the frontier from the persisted ingestion state. -func (s *Server) lastIngestedLedger(ctx context.Context, network string) (int64, error) { - state, err := s.store.GetIngestionState(ctx, network) +func (s *Server) lastIngestedLedger(ctx context.Context) (int64, error) { + state, err := s.store.GetIngestionState(ctx) if errors.Is(err, store.ErrNotFound) { return 0, nil @@ -2030,52 +2056,32 @@ func (s *Server) lastIngestedLedger(ctx context.Context, network string) (int64, } -// resolveNetwork returns the network to use for the current request. -func (s *Server) resolveNetwork(r *http.Request) (string, error) { - q := r.URL.Query().Get("network") - if q == "" { - if s.defaultNetwork != "" { - return s.defaultNetwork, nil - } - if len(s.networkNames) == 0 { - return "", nil - } - return "", fmt.Errorf("network query parameter is required when multiple networks are configured; available: %s", strings.Join(s.networkNames, ", ")) - } - if len(s.networkNames) == 0 { - return "", fmt.Errorf("unknown network %q; no networks configured", q) - } - for _, n := range s.networkNames { - if n == q { - return q, nil - } - } - return "", fmt.Errorf("unknown network %q; available: %s", q, strings.Join(s.networkNames, ", ")) -} - func listETag(f store.EventFilter) string { key := struct { - ContractID string `json:"c"` + ContractID string `json:"c"` - Types []string `json:"t"` + ContractIDPrefix string `json:"cp,omitempty"` + Types []string `json:"t"` - Topic json.RawMessage `json:"p,omitempty"` + Topic json.RawMessage `json:"p,omitempty"` - Topic0 json.RawMessage `json:"p0,omitempty"` + Topic0 json.RawMessage `json:"p0,omitempty"` - Topic1 json.RawMessage `json:"p1,omitempty"` + Topic1 json.RawMessage `json:"p1,omitempty"` - Topic2 json.RawMessage `json:"p2,omitempty"` + Topic2 json.RawMessage `json:"p2,omitempty"` - Topic3 json.RawMessage `json:"p3,omitempty"` + Topic3 json.RawMessage `json:"p3,omitempty"` - TopicContains json.RawMessage `json:"pc,omitempty"` + TopicContains json.RawMessage `json:"pc,omitempty"` - TxHash string `json:"th,omitempty"` + TxHash string `json:"th,omitempty"` - HasValue *bool `json:"hv,omitempty"` - FromLedger int64 `json:"fl"` + HasValue *bool `json:"hv,omitempty"` + TxIndex *int32 `json:"txi,omitempty"` + OpIndex *int32 `json:"opi,omitempty"` + FromLedger int64 `json:"fl"` ToLedger int64 `json:"tl"` @@ -2129,7 +2135,6 @@ func listETag(f store.EventFilter) string { FromTime: timeOrEmpty(f.FromTime), ToTime: timeOrEmpty(f.ToTime), - HasValue: f.HasValue, Cursor: f.Cursor, Limit: resolvedLimit(f.Limit), @@ -2510,7 +2515,9 @@ func filterFromQuery(r *http.Request) (store.EventFilter, error) { return f, fmt.Errorf("invalid in_successful_call %q (want true or false)", raw) } - + // order/order_by/topic/topic0..topic3/topic_contains/from_ledger/ + // to_ledger/from_time/to_time are applied by queries.BuildEventFilter + // above (via the args struct populated at the top of this function). if raw := q.Get("limit"); raw != "" { @@ -2519,9 +2526,6 @@ func filterFromQuery(r *http.Request) (store.EventFilter, error) { if err != nil || limit < 1 || limit > store.MaxQueryLimit { return f, fmt.Errorf("limit must be an integer in [1,%d]", store.MaxQueryLimit) - - if err != nil || limit < 1 || limit > maxLimit { - return f, fmt.Errorf("limit must be an integer in [1,%d]", maxLimit) } f.Limit = limit @@ -2530,6 +2534,8 @@ func filterFromQuery(r *http.Request) (store.EventFilter, error) { f.Limit = store.DefaultQueryLimit + } + // ?recent=N: shorthand for "newest N events" — sets order=desc and // limit=N (default 20). This isn't a general-purpose filter (it // conflicts with explicit order/limit/order_by), so we keep the @@ -2641,80 +2647,6 @@ func (s *Server) syncStreamScope(ctx context.Context, sub *broadcast.Subscriptio }() } -// Holders endpoint types. - -type holderResponse struct { - Address string `json:"address"` - Balance string `json:"balance"` - LastLedger int64 `json:"last_ledger"` -} - -type holdersResponse struct { - ContractID string `json:"contract_id"` - EarliestLedger int64 `json:"earliest_ledger"` - Holders []holderResponse `json:"holders"` - Cursor string `json:"cursor,omitempty"` -} - -func (s *Server) handleContractHolders(w http.ResponseWriter, r *http.Request) { - contractID := chi.URLParam(r, "id") - if !config.ValidContractID(contractID) { - writeError(w, http.StatusBadRequest, fmt.Errorf("invalid contract ID %q", contractID)) - return - } - - network, err := s.resolveNetwork(r) - if err != nil { - writeError(w, http.StatusBadRequest, err) - return - } - - minBalance := r.URL.Query().Get("min_balance") - cursor := r.URL.Query().Get("cursor") - limit := store.DefaultQueryLimit - if raw := r.URL.Query().Get("limit"); raw != "" { - parsed, err := strconv.Atoi(raw) - if err != nil || parsed < 1 || parsed > store.MaxQueryLimit { - writeError(w, http.StatusBadRequest, fmt.Errorf("limit must be an integer in [1,%d]", store.MaxQueryLimit)) - return - } - limit = parsed - } - - // Determine the earliest ledger for coverage indication. - earliestLedger, err := s.store.GetEarliestLedger(r.Context(), network, contractID) - if err != nil { - // non-fatal; surface as 0 to indicate unknown coverage - loggerFromContext(r.Context()).Warn("getting earliest ledger", "contract_id", contractID, "error", err) - } - - balances, next, err := s.store.GetTokenBalances(r.Context(), contractID, network, minBalance, cursor, limit) - if err != nil { - loggerFromContext(r.Context()).Error("querying token holders", "contract_id", contractID, "error", err) - writeError(w, http.StatusInternalServerError, errors.New("querying token holders failed")) - return - } - - holders := make([]holderResponse, len(balances)) - for i, tb := range balances { - holders[i] = holderResponse{ - Address: tb.Address, - Balance: tb.Balance, - LastLedger: tb.LastLedger, - } - } - - writeCacheHeaders(w, cacheNoCache, 0, "") - writeJSON(w, http.StatusOK, holdersResponse{ - ContractID: contractID, - EarliestLedger: earliestLedger, - Holders: holders, - Cursor: next, - }) -} - - - func (s *Server) handleEventStreamWS(w http.ResponseWriter, r *http.Request) { if s.bcast == nil { diff --git a/internal/api/handlers_contracts.go b/internal/api/handlers_contracts.go new file mode 100644 index 00000000..43140d79 --- /dev/null +++ b/internal/api/handlers_contracts.go @@ -0,0 +1,56 @@ +package api + +import ( + "errors" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + + "github.com/sorotrail/sorotrail/internal/config" +) + +// contractStatsResponse is the JSON shape for GET /contracts/{id}/stats. +type contractStatsResponse struct { + ContractID string `json:"contract_id"` + Name *string `json:"name,omitempty"` + Symbol *string `json:"symbol,omitempty"` + Decimals *int `json:"decimals,omitempty"` + EventCount int64 `json:"event_count"` +} + +// handleContractStats returns per-contract statistics with cached metadata. +func (s *Server) handleContractStats(w http.ResponseWriter, r *http.Request) { + contractID := chi.URLParam(r, "id") + if !config.ValidContractID(contractID) { + writeError(w, http.StatusBadRequest, fmt.Errorf("invalid contract ID %q", contractID)) + return + } + + // Verify the contract exists (has at least one event). + count, err := s.store.CountContractEvents(r.Context(), contractID) + if err != nil { + s.log.Error("counting events for contract", "contract_id", contractID, "error", err) + writeError(w, http.StatusInternalServerError, errors.New("loading contract stats failed")) + return + } + if count == 0 { + writeError(w, http.StatusNotFound, fmt.Errorf("contract %q not found", contractID)) + return + } + + resp := contractStatsResponse{ + ContractID: contractID, + EventCount: count, + } + + // Attach metadata if available. + if meta, err := s.store.GetContractMeta(r.Context(), contractID); err == nil && meta.HasMetadata() { + resp.Name = meta.Name + resp.Symbol = meta.Symbol + resp.Decimals = meta.Decimals + } + + writeCacheHeaders(w, cacheNoCache, 0, "") + writeJSON(w, http.StatusOK, resp) +} diff --git a/internal/api/server.go b/internal/api/server.go index d659c721..d36e13e2 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -11,13 +11,11 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - otelhttp "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/sorotrail/sorotrail/internal/audit" "github.com/sorotrail/sorotrail/internal/broadcast" + "github.com/sorotrail/sorotrail/internal/ingester" "github.com/sorotrail/sorotrail/internal/metrics" "github.com/sorotrail/sorotrail/internal/pruner" "github.com/sorotrail/sorotrail/internal/rpc" @@ -125,16 +123,17 @@ type Enricher interface { // Server holds the API's dependencies. type Server struct { - store store.Store - enableMetrics bool - rpc rpc.Client - enricher Enricher - log *slog.Logger - apiKey string - limiter *RateLimiter - recoverer *Recoverer - bcast *broadcast.Broadcaster - metrics *metrics.HTTPMetrics + store store.Store + rpc rpc.Client + log *slog.Logger + apiKey string + limiter *RateLimiter + bcast *broadcast.Broadcaster + enricher Enricher + enableMetrics bool + recoverer *Recoverer + metrics *metrics.HTTPMetrics + tracer trace.Tracer // GraphQL transport, injected by main after the server is built. // internal/api/graphql imports this package for its ServerDeps, so @@ -178,6 +177,8 @@ func (s *Server) SetCompressMinSize(n int) { // SetMetricsEnabled enables or disables the /metrics endpoint. func (s *Server) SetMetricsEnabled(enabled bool) { s.enableMetrics = enabled +} + // maxLimit is the API's upper bound for page-size parameters (limit and // recent). It is set once at startup via SetMaxLimit (driven by the // API_MAX_LIMIT env var) before any requests are served so no mutex is @@ -257,13 +258,6 @@ func (s *Server) SetRateLimiter(l *RateLimiter) { s.limiter = l } -// SetCORS configures the cross-origin allow-list (CORS_ALLOWED_ORIGINS). -// Pass nil or an empty slice to leave CORS disabled (the default — the -// router emits no CORS headers, so existing deployments are unaffected). -func (s *Server) SetCORS(allowedOrigins []string) { - s.corsOrigins = allowedOrigins -} - // WithBroadcaster attaches the live event broadcaster so streaming endpoints // can deliver events as they arrive. func (s *Server) WithBroadcaster(b *broadcast.Broadcaster) *Server { @@ -329,14 +323,13 @@ func (s *Server) router() chi.Router { // Non-list routes: health, metrics, writes — responses are always // small, so compression is just overhead with no benefit. r.Get("/health", s.handleHealth) - r.Get("/metrics", s.handleMetrics) - r.Get("/livez", s.handleLivez) - r.Get("/readyz", s.handleReadyz) - r.Get("/version", s.handleVersion) // Registered as GET, not Handle: Handle advertises every method (the // route-drift test then demands CONNECT/TRACE entries in the OpenAPI // spec), and scraping is a GET. r.Get("/metrics", s.metrics.Handler().ServeHTTP) + r.Get("/livez", s.handleLivez) + r.Get("/readyz", s.handleReadyz) + r.Get("/version", s.handleVersion) r.Get("/events", s.handleListEvents) r.Get("/events/count", s.handleCountEvents) r.Get("/events/aggregate", s.handleAggregateEvents) @@ -347,9 +340,8 @@ func (s *Server) router() chi.Router { r.Get("/contracts", s.handleListContracts) r.Get("/contracts/{id}/events", s.handleContractEvents) r.Get("/contracts/{id}/export", s.handleContractExport) - + r.Get("/contracts/{id}/stats", s.handleContractStats) r.Get("/stats", s.handleStats) - r.Handle("/metrics", promhttp.Handler()) r.Get("/events/ws", s.handleEventStreamWS) // Admin bulk delete: auth-gated endpoint to delete events by ledger range. @@ -451,7 +443,7 @@ func (s *Server) router() chi.Router { r.Get("/addresses/{address}/events", s.handleAddressEvents) r.Get("/addresses/{address}/summary", s.handleAddressSummary) - return otelhttp.NewHandler(r, "HTTP") + return r } // handleOpenAPI serves the embedded OpenAPI 3.1 specification. @@ -483,13 +475,6 @@ func (s *Server) requestLogger(next http.Handler) http.Handler { "status", ww.Status(), "duration_ms", time.Since(start).Milliseconds(), ) - if s.metrics != nil { - path := chi.RouteContext(r.Context()).RoutePattern() - if path == "" { - path = r.URL.Path - } - s.metrics.RecordHTTPRequest(path, ww.Status(), time.Since(start).Seconds()) - } }) } diff --git a/internal/api/spec.go b/internal/api/spec.go new file mode 100644 index 00000000..6b2321ef --- /dev/null +++ b/internal/api/spec.go @@ -0,0 +1,25 @@ +package api + +import _ "embed" + +//go:embed openapi.json +var openapiSpec []byte + +// swaggerUI is the minimal HTML page that renders the Swagger UI for the +// embedded OpenAPI spec. It loads swagger-ui-dist from jsdelivr CDN. +const swaggerUI = ` + + + + + SoroTrail API – Swagger UI + + + +
+ + + +` diff --git a/internal/api/subscriptions_test.go b/internal/api/subscriptions_test.go index 79737407..93ee2861 100644 --- a/internal/api/subscriptions_test.go +++ b/internal/api/subscriptions_test.go @@ -139,7 +139,7 @@ func errorEnvelope(t *testing.T, body []byte) string { // slog construction on every line. func newServerFromStub(st store.Store) *Server { log := slog.New(slog.NewTextHandler(io.Discard, nil)) - return New(st, nil, log, "test-key", 0) + return New(st, nil, log, "test-key") } // TestSubscriptions_ErrorPaths covers every 400/404 branch of the diff --git a/internal/audit/audit.go b/internal/audit/audit.go index a56deff4..2aede565 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -135,7 +135,7 @@ func (a *Auditor) PassOnce(ctx context.Context) (worked bool, err error) { if err != nil && !errors.Is(err, store.ErrNotFound) { return false, fmt.Errorf("loading audit state: %w", err) } - ing, err := a.store.GetIngestionState(ctx, a.opts.Network) + ing, err := a.store.GetIngestionState(ctx) if err != nil && !errors.Is(err, store.ErrNotFound) { return false, fmt.Errorf("loading ingestion state: %w", err) } diff --git a/internal/audit/client.go b/internal/audit/client.go index ecea887e..9acaf982 100644 --- a/internal/audit/client.go +++ b/internal/audit/client.go @@ -6,17 +6,6 @@ import ( "github.com/sorotrail/sorotrail/internal/rpc" ) -// Client is the audit-side view of the RPC. It is the same interface as -// rpc.Client (so the auditor can swap in a fake in tests) but every -// outbound call gates through rpc.Budget.WaitAudit so audit traffic -// receives at most its configured share of the total request budget. -type Client interface { - GetEvents(ctx context.Context, req rpc.GetEventsRequest) (rpc.GetEventsResponse, error) - GetLatestLedger(ctx context.Context) (rpc.LatestLedger, error) - GetHealth(ctx context.Context) (rpc.Health, error) - GetLedgerEntries(ctx context.Context, req rpc.GetLedgerEntriesRequest) (rpc.GetLedgerEntriesResponse, error) -} - // budgetedClient wraps an inner rpc.Client, accounting every call against // Budget.WaitAudit before dispatching. type budgetedClient struct { @@ -24,11 +13,11 @@ type budgetedClient struct { budget *rpc.Budget } -// NewBudgetedClient returns an audit-scoped Client that shares the same -// underlying connection as inner but reserves tokens from the audit pool -// of b on every call. A nil b is permitted (the audit becomes un-paced; -// useful only in tests). -func NewBudgetedClient(inner rpc.Client, b *rpc.Budget) Client { +// NewBudgetedClient returns an rpc.Client that shares the same underlying +// connection as inner but reserves tokens from the audit pool of b on +// every call. A nil b is permitted (the audit becomes un-paced; useful +// only in tests). +func NewBudgetedClient(inner rpc.Client, b *rpc.Budget) rpc.Client { return &budgetedClient{inner: inner, budget: b} } @@ -60,5 +49,12 @@ func (c *budgetedClient) GetLedgerEntries(ctx context.Context, req rpc.GetLedger return c.inner.GetLedgerEntries(ctx, req) } -// Compile-time check that we satisfy the audit Client interface. -var _ Client = (*budgetedClient)(nil) +func (c *budgetedClient) SimulateTransaction(ctx context.Context, req rpc.SimulateTransactionRequest) (rpc.SimulateTransactionResponse, error) { + if err := c.budget.WaitAudit(ctx); err != nil { + return rpc.SimulateTransactionResponse{}, err + } + return c.inner.SimulateTransaction(ctx, req) +} + +// Compile-time check that we satisfy the rpc.Client interface. +var _ rpc.Client = (*budgetedClient)(nil) diff --git a/internal/audit/mocks_test.go b/internal/audit/mocks_test.go index 0a9cdf95..c83c50e2 100644 --- a/internal/audit/mocks_test.go +++ b/internal/audit/mocks_test.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "sync" + "time" "github.com/sorotrail/sorotrail/internal/rpc" "github.com/sorotrail/sorotrail/internal/store" @@ -65,6 +66,13 @@ func (m *mockRPC) GetLedgerEntries(_ context.Context, _ rpc.GetLedgerEntriesRequ return rpc.GetLedgerEntriesResponse{}, nil } +func (m *mockRPC) SimulateTransaction(context.Context, rpc.SimulateTransactionRequest) (rpc.SimulateTransactionResponse, error) { + return rpc.SimulateTransactionResponse{}, nil +} + +// mockStore is an in-memory implementation of store.Store good enough +// for the auditor. It mirrors and extends the ingester test mock so we +// don't import the ingester test package. type mockStore struct { // Embedded so the mock keeps satisfying store.Store as the // interface grows; unstubbed methods panic if a test calls them. @@ -74,8 +82,9 @@ type mockStore struct { events map[string]store.Event - ingestionState *store.IngestionState - auditState *store.AuditState + ingress *store.IngestionState + audit *store.AuditState + watched []store.WatchedContract findings []store.AuditFinding nextFID int64 @@ -179,12 +188,12 @@ func (m *mockStore) LedgerRangeCensus(_ context.Context, from, to int64, idsOnly defer m.mu.Unlock() byLedger := map[int64][]string{} for _, e := range m.events { - if e.Ledger >= fromLedger && e.Ledger <= toLedger { + if e.Ledger >= from && e.Ledger <= to { byLedger[e.Ledger] = append(byLedger[e.Ledger], e.ID) } } var out []store.LedgerCensus - for l := fromLedger; l <= toLedger; l++ { + for l := from; l <= to; l++ { ids := byLedger[l] if len(ids) == 0 { continue @@ -198,19 +207,19 @@ func (m *mockStore) LedgerRangeCensus(_ context.Context, from, to int64, idsOnly return out, nil } -func (m *mockStore) GetIngestionState(_ context.Context, _ string) (store.IngestionState, error) { +func (m *mockStore) GetIngestionState(_ context.Context) (store.IngestionState, error) { m.mu.Lock() defer m.mu.Unlock() if m.ingress.LastIngestedLedger <= 0 && m.ingress.LastCursor == "" { return store.IngestionState{}, store.ErrNotFound } - return m.ingress, nil + return *m.ingress, nil } func (m *mockStore) SaveIngestionState(_ context.Context, s store.IngestionState) error { m.mu.Lock() defer m.mu.Unlock() - m.ingress = s + m.ingress = &s return nil } @@ -327,11 +336,21 @@ func (m *mockStore) MigrationVersion(context.Context) (int, bool, error) { } func (m *mockStore) Ping(context.Context) error { return nil } - -func (m *mockStore) GetContractSpec(context.Context, string) ([]byte, error) { - return nil, store.ErrNotFound +func (m *mockStore) ListContractIDs(context.Context) ([]string, error) { + return nil, nil +} +func (m *mockStore) GetContractMeta(context.Context, string) (store.ContractMeta, error) { + return store.ContractMeta{}, store.ErrNotFound +} +func (m *mockStore) UpsertContractMeta(context.Context, store.ContractMeta) error { + return nil +} +func (m *mockStore) CountContractEvents(context.Context, string) (int64, error) { + return 0, nil +} +func (m *mockStore) ListContractsNeedingRefresh(context.Context, time.Time) ([]string, error) { + return nil, nil } -func (m *mockStore) SetContractSpec(context.Context, string, string, []byte) error { return nil } func (m *mockStore) CreateSubscription(_ context.Context, sub store.Subscription) (store.Subscription, error) { sub.ID = 1 @@ -364,22 +383,6 @@ func (m *mockStore) ListDeliveryAttempts(context.Context, int64, int, store.Subs return nil, nil } -func (m *mockStore) UpsertTokenBalances(ctx context.Context, network string, state store.TokenBalanceState, updates []store.TokenBalanceUpdate) error { - return nil -} - -func (m *mockStore) GetTokenBalances(ctx context.Context, contractID, network, minBalance string, cursor string, limit int) ([]store.TokenBalance, string, error) { - return nil, "", nil -} - -func (m *mockStore) GetTokenBalanceState(ctx context.Context, network, contractID string) (store.TokenBalanceState, error) { - return store.TokenBalanceState{}, store.ErrNotFound -} - -func (m *mockStore) UpsertTokenBalanceState(ctx context.Context, state store.TokenBalanceState) error { - return nil -} - func (m *mockStore) GetEarliestLedger(ctx context.Context, network, contractID string) (int64, error) { return 0, nil } diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go index e2bba669..86b03e96 100644 --- a/internal/broadcast/broadcast.go +++ b/internal/broadcast/broadcast.go @@ -205,15 +205,6 @@ func eventMatches(ev store.Event, f store.EventFilter) bool { return false } } - if f.TopicCount != nil { - var arr []json.RawMessage - if err := json.Unmarshal(ev.Topics, &arr); err != nil { - return false - } - if len(arr) != *f.TopicCount { - return false - } - } if f.FromLedger > 0 && ev.Ledger < f.FromLedger { return false } diff --git a/internal/config/config.go b/internal/config/config.go index d9ada7c6..06e5af50 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,7 +3,6 @@ package config import ( - "encoding/json" "fmt" "log/slog" "net/url" @@ -22,7 +21,17 @@ type NetworkConfig struct { // Config holds all runtime configuration. Every field is settable via the // environment variable named in its `env` tag; see .env.example for docs. type Config struct { - RPCURL string `env:"RPC_URL"` // deprecated — use NETWORKS + // RPCURL is the single-provider RPC endpoint. When RPC_URLS is set the + // multi-provider failover client is used instead and RPC_URL is ignored. + RPCURL string `env:"RPC_URL" envDefault:"https://soroban-testnet.stellar.org"` + // RPCURLS, when set, enables the multi-provider failover client + // (internal/rpc). List order is priority: index 0 is tried first. + // RPC_URL is ignored while it is set; leaving RPC_URLS unset keeps the + // single-provider behavior unchanged. + RPCURLS []string `env:"RPC_URLS"` + // RPCRateLimitRPS caps each provider's request rate (requests/second) + // when the failover client is in use. Only read by the failover client. + RPCRateLimitRPS float64 `env:"RPC_RATE_LIMIT_RPS" envDefault:"10"` DatabaseURL string `env:"DATABASE_URL"` PollInterval time.Duration `env:"POLL_INTERVAL" envDefault:"5s"` HTTPAddr string `env:"HTTP_ADDR" envDefault:":8080"` @@ -223,6 +232,8 @@ func Load() (Config, error) { return Config{}, fmt.Errorf("parsing environment: %w", err) } cfg.WatchedContracts = cleanContractList(cfg.WatchedContracts) + cfg.RPCURLS = cleanContractList(cfg.RPCURLS) + cfg.CORSAllowedOrigins = cleanOrigins(cfg.CORSAllowedOrigins) if err := cfg.ValidateAll(); err != nil { return Config{}, err } @@ -243,12 +254,36 @@ func IsSQLite(databaseURL string) bool { return strings.HasPrefix(databaseURL, "sqlite:") } +// ParseLogLevel maps a LOG_LEVEL string to a slog.Level. Unknown or empty +// values fall back to info rather than failing startup. +func ParseLogLevel(raw string) slog.Level { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "debug": + return slog.LevelDebug + case "warn": + return slog.LevelWarn + case "error": + return slog.LevelError + default: + return slog.LevelInfo + } +} + // Validate checks the configuration for values that would fail at runtime. func (c Config) Validate() error { if c.DatabaseURL == "" { return fmt.Errorf("DATABASE_URL is required") } - if !IsSQLite(c.DatabaseURL) { + // RPC_URLS takes priority when set; RPC_URL is the single-provider + // fallback that works unchanged for existing deployments. + if len(c.RPCURLS) > 0 { + for i, raw := range c.RPCURLS { + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("RPC_URLS[%d] %q is not a valid URL", i, raw) + } + } + } else if !IsSQLite(c.DatabaseURL) { u, err := url.Parse(c.RPCURL) if err != nil || u.Scheme == "" || u.Host == "" { return fmt.Errorf("RPC_URL %q is not a valid URL", c.RPCURL) @@ -319,6 +354,9 @@ func (c Config) Validate() error { if c.AuditFindingMaxLgrs == 0 { return fmt.Errorf("AUDIT_FINDING_MAX_LEDGERS must be positive") } + if c.RPCRateLimitRPS <= 0 { + return fmt.Errorf("RPC_RATE_LIMIT_RPS must be positive") + } if c.RetentionBatchSize <= 0 { return fmt.Errorf("RETENTION_BATCH_SIZE must be positive") } @@ -424,21 +462,6 @@ func ValidContractID(s string) bool { return true } -// ValidCursor reports whether s is a valid pagination cursor. -// A cursor must be non-empty, at most 128 characters, and consist only of -// alphanumeric characters, hyphens, underscores, dots, or colons. -func ValidCursor(s string) bool { - if len(s) == 0 || len(s) > 128 { - return false - } - for _, r := range s { - if !strings.ContainsRune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.:", r) { - return false - } - } - return true -} - // ValidCursor reports whether s is a valid pagination cursor. // A cursor must be non-empty, at most 128 characters, and consist only of // alphanumeric characters, hyphens, underscores, dots, or colons. @@ -478,24 +501,6 @@ func ValidOrigin(s string) bool { return true } -// ValidCursor reports whether s is a valid pagination cursor. -// A cursor must be non-empty, at most 128 characters, and consist only of -// alphanumeric characters, hyphens, underscores, dots, or colons. -func ValidCursor(s string) bool { - if len(s) == 0 || len(s) > 128 { - return false - } - for _, r := range s { - if (r < 'a' || r > 'z') && - (r < 'A' || r > 'Z') && - (r < '0' || r > '9') && - r != '-' && r != '_' && r != '.' && r != ':' { - return false - } - } - return true -} - func cleanContractList(in []string) []string { out := make([]string, 0, len(in)) for _, s := range in { @@ -514,6 +519,19 @@ func cleanContractList(in []string) []string { // // "*" is a valid literal but documented separately as a special case the // middleware recognizes (see API CORS handler). +// cleanOrigins normalizes CORS origin entries: env/v11 splits on commas but +// preserves whitespace, and operators commonly paste origins with a trailing +// slash, so each entry is trimmed and any trailing "/" removed. +func cleanOrigins(in []string) []string { + out := make([]string, 0, len(in)) + for _, s := range in { + if s = strings.TrimSpace(s); s != "" { + out = append(out, strings.TrimSuffix(s, "/")) + } + } + return out +} + func validateCORSOrigins(in []string) error { for _, o := range in { o = strings.TrimSpace(o) @@ -523,8 +541,7 @@ func validateCORSOrigins(in []string) error { if strings.EqualFold(o, "null") { return fmt.Errorf("CORS_ALLOWED_ORIGINS entry %q is not allowed (sandboxed Origin: null is a credentialed-origin bypass)", o) } - u, err := url.Parse(o) - if err != nil || u.Scheme == "" || u.Host == "" { + if !ValidOrigin(o) { return fmt.Errorf("CORS_ALLOWED_ORIGINS entry %q is not a valid origin (want scheme://host[:port])", o) } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 573227f1..246ae0c4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,7 +1,6 @@ package config import ( - "log/slog" "os" "testing" "time" @@ -13,7 +12,8 @@ import ( const validContract = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC" var envKeys = []string{ - "RPC_URL", "DATABASE_URL", "POLL_INTERVAL", "HTTP_ADDR", + "RPC_URL", "RPC_URLS", "RPC_RATE_LIMIT_RPS", "DATABASE_URL", + "POLL_INTERVAL", "HTTP_ADDR", "WATCHED_CONTRACTS", "START_LEDGER", "RETENTION_LEDGERS", "LOG_LEVEL", "LOG_FORMAT", "API_QUERY_TIMEOUT", "API_SLOW_QUERY_THRESHOLD", "HORIZON_URL", "BACKFILL_RATE_RPS", @@ -83,50 +83,6 @@ func TestLoad(t *testing.T) { env: map[string]string{}, wantErr: "DATABASE_URL: required but empty", }, - { - name: "NETWORKS with single network", - env: map[string]string{ - "DATABASE_URL": "postgres://localhost/db", - "NETWORKS": `[{"name":"testnet","rpc_url":"https://testnet.stellar.org"}]`, - }, - check: func(t *testing.T, c Config) { - networks := c.NetworksOrDefault() - require.Len(t, networks, 1) - assert.Equal(t, "testnet", networks[0].Name) - assert.Equal(t, "https://testnet.stellar.org", networks[0].RPCURL) - }, - }, - { - name: "NETWORKS with two networks requires DEFAULT_NETWORK", - env: map[string]string{ - "DATABASE_URL": "postgres://localhost/db", - "NETWORKS": `[{"name":"testnet","rpc_url":"https://testnet.stellar.org"},{"name":"mainnet","rpc_url":"https://mainnet.stellar.org"}]`, - }, - wantErr: "DEFAULT_NETWORK is required when multiple networks are configured", - }, - { - name: "NETWORKS with two networks and DEFAULT_NETWORK works", - env: map[string]string{ - "DATABASE_URL": "postgres://localhost/db", - "NETWORKS": `[{"name":"testnet","rpc_url":"https://testnet.stellar.org"},{"name":"mainnet","rpc_url":"https://mainnet.stellar.org"}]`, - "DEFAULT_NETWORK": "testnet", - }, - check: func(t *testing.T, c Config) { - networks := c.NetworksOrDefault() - require.Len(t, networks, 2) - assert.Equal(t, "testnet", c.DefaultNetworkName()) - assert.Equal(t, []string{"testnet", "mainnet"}, c.NetworkNames()) - }, - }, - { - name: "RPC_URL and NETWORKS both set is rejected", - env: map[string]string{ - "DATABASE_URL": "postgres://localhost/db", - "RPC_URL": "https://testnet.stellar.org", - "NETWORKS": `[{"name":"testnet","rpc_url":"https://testnet.stellar.org"}]`, - }, - wantErr: "RPC_URL and NETWORKS cannot both be set", - }, { name: "watched contracts parsed and trimmed", env: map[string]string{ @@ -463,6 +419,53 @@ func TestLoad(t *testing.T) { }, wantErr: "CORS_ALLOWED_ORIGINS entry", }, + { + name: "RPC_URLS with valid URLs accepted", + env: map[string]string{ + "DATABASE_URL": "postgres://localhost/db", + "RPC_URLS": "https://rpc1.example.com,https://rpc2.example.com", + }, + check: func(t *testing.T, c Config) { + assert.Equal(t, []string{"https://rpc1.example.com", "https://rpc2.example.com"}, c.RPCURLS) + assert.Equal(t, float64(10), c.RPCRateLimitRPS) + }, + }, + { + name: "RPC_URLS invalid URL rejected", + env: map[string]string{ + "DATABASE_URL": "postgres://localhost/db", + "RPC_URLS": "https://good.example.com,not a url", + }, + wantErr: "RPC_URLS[1]", + }, + { + name: "RPC_URLS empty entries trimmed", + env: map[string]string{ + "DATABASE_URL": "postgres://localhost/db", + "RPC_URLS": "https://rpc.example.com, ,", + }, + check: func(t *testing.T, c Config) { + assert.Equal(t, []string{"https://rpc.example.com"}, c.RPCURLS) + }, + }, + { + name: "RPC_RATE_LIMIT_RPS custom value", + env: map[string]string{ + "DATABASE_URL": "postgres://localhost/db", + "RPC_RATE_LIMIT_RPS": "5", + }, + check: func(t *testing.T, c Config) { + assert.Equal(t, float64(5), c.RPCRateLimitRPS) + }, + }, + { + name: "RPC_RATE_LIMIT_RPS zero rejected", + env: map[string]string{ + "DATABASE_URL": "postgres://localhost/db", + "RPC_RATE_LIMIT_RPS": "0", + }, + wantErr: "RPC_RATE_LIMIT_RPS must be positive", + }, } for _, tt := range tests { @@ -510,20 +513,6 @@ func TestValidContractID(t *testing.T) { assert.False(t, ValidContractID(validContract[:55]+"a"), "lowercase is not base32") } -func TestParseNetworks(t *testing.T) { - networks, err := ParseNetworks(`[{"name":"test","rpc_url":"https://test.stellar.org"}]`) - require.NoError(t, err) - require.Len(t, networks, 1) - assert.Equal(t, "test", networks[0].Name) - - _, err = ParseNetworks("invalid") - require.Error(t, err) - - networks, err = ParseNetworks("") - require.NoError(t, err) - assert.Nil(t, networks) -} - func TestValidCursor(t *testing.T) { assert.True(t, ValidCursor("0001099511627776-0000000001")) assert.True(t, ValidCursor("00000000000000000102-00000")) diff --git a/internal/config/validation.go b/internal/config/validation.go index 994eb5d7..7d12c0ba 100644 --- a/internal/config/validation.go +++ b/internal/config/validation.go @@ -68,7 +68,16 @@ func (c Config) ValidateAll() error { // --- URL format --------------------------------------------------------- - if c.RPCURL == "" { + // RPC_URLS takes priority when set: the multi-provider failover client + // is used and RPC_URL may be left empty. Otherwise RPC_URL is the + // single-provider endpoint and is required to be a valid URL. + if len(c.RPCURLS) > 0 { + for i, raw := range c.RPCURLS { + if u, err := url.Parse(raw); err != nil || u.Scheme == "" || u.Host == "" { + errs = append(errs, fmt.Sprintf("RPC_URLS[%d]: %q is not a valid absolute URL (want scheme://host)", i, raw)) + } + } + } else if c.RPCURL == "" { errs = append(errs, "RPC_URL: required but empty") } else if u, err := url.Parse(c.RPCURL); err != nil || u.Scheme == "" || u.Host == "" { errs = append(errs, fmt.Sprintf("RPC_URL: %q is not a valid absolute URL (want scheme://host)", diff --git a/internal/ingester/ingester.go b/internal/ingester/ingester.go index 907b75d4..4ef0e448 100644 --- a/internal/ingester/ingester.go +++ b/internal/ingester/ingester.go @@ -13,17 +13,12 @@ import ( "github.com/prometheus/client_golang/prometheus" - "github.com/khaylebfortune/sorotrail/internal/broadcast" - "github.com/khaylebfortune/sorotrail/internal/decode" - "github.com/khaylebfortune/sorotrail/internal/metrics" - "github.com/khaylebfortune/sorotrail/internal/rpc" - "github.com/khaylebfortune/sorotrail/internal/store" - "golang.org/x/sync/errgroup" - "github.com/sorotrail/sorotrail/internal/broadcast" "github.com/sorotrail/sorotrail/internal/decode" + "github.com/sorotrail/sorotrail/internal/metrics" "github.com/sorotrail/sorotrail/internal/rpc" "github.com/sorotrail/sorotrail/internal/store" + "golang.org/x/sync/errgroup" ) // Clock abstracts time operations so tests and simulations can supply a @@ -216,7 +211,7 @@ type Ingester struct { // New wires an Ingester. func New(client rpc.Client, st store.Store, dec decode.Decoder, log *slog.Logger, opts Options) *Ingester { opts.applyDefaults() - return &Ingester{client: client, store: st, decoder: dec, log: log, metrics: obs, opts: opts} + return &Ingester{client: client, store: st, decoder: dec, log: log, opts: opts} } // WithBroadcaster attaches a live event broadcaster so ingested events are @@ -364,15 +359,6 @@ func (ing *Ingester) rescanForReorg(ctx context.Context) error { // ingestion_state row — backward-compatible behavior unchanged. // Watched mode uses per-contract cursors in the contract_cursors table. func (ing *Ingester) runOnce(ctx context.Context) (caughtUp bool, err error) { - ctx, span := ing.tracer.Start(ctx, "ingester.poll_cycle") - defer func() { - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - } - span.End() - }() - startLedger, cursor, err := ing.resolvePosition(ctx) if err != nil { return false, err @@ -382,20 +368,12 @@ func (ing *Ingester) runOnce(ctx context.Context) (caughtUp bool, err error) { return false, err } if len(batches) == 1 { - return ing.singlePageUnwatched(ctx, startLedger, cursor, batches[0]) + return ing.singlePage(ctx, startLedger, cursor, batches[0]) } return ing.windowSweepUnwatched(ctx, startLedger, batches) } func (ing *Ingester) singlePage(ctx context.Context, startLedger uint32, cursor string, filters []rpc.EventFilter) (bool, error) { - ctx, span := ing.tracer.Start(ctx, "ingester.fetch_page") - defer span.End() - span.SetAttributes( - attribute.Int64("ingester.start_ledger", int64(startLedger)), - attribute.String("ingester.cursor", cursor), - attribute.Int("ingester.filter_count", len(filters)), - ) - resp, err := ing.client.GetEvents(ctx, rpc.GetEventsRequest{ StartLedger: startLedger, Filters: filters, @@ -407,7 +385,6 @@ func (ing *Ingester) singlePage(ctx context.Context, startLedger uint32, cursor return false, err } if rpc.IsLedgerOutOfRange(err) { - span.AddEvent("retention_clamp") return false, ing.reclampToOldest(ctx, startLedger) } if err != nil { @@ -422,7 +399,6 @@ func (ing *Ingester) singlePage(ctx context.Context, startLedger uint32, cursor if state.LastCursor == "" && state.LastIngestedLedger <= 0 { state.LastIngestedLedger = int64(startLedger) - 1 } - state.Network = ing.opts.Network if err := ing.store.SaveIngestionState(ctx, state); err != nil { return false, err } @@ -511,7 +487,7 @@ func (ing *Ingester) BuildFilterBatches(ctx context.Context) ([][]rpc.EventFilte func (ing *Ingester) PageLimit() uint { return ing.opts.PageLimit } // Network returns the network this ingester is responsible for. -func (ing *Ingester) Network() string { return ing.opts.Network } +func (ing *Ingester) Network() string { return "" } func nextState(resp rpc.GetEventsResponse, pageLimit uint) (store.IngestionState, bool) { caughtUp := uint(len(resp.Events)) < pageLimit @@ -639,24 +615,10 @@ func (ing *Ingester) windowSweepWatched(ctx context.Context, batches [][]rpc.Eve } if err := g.Wait(); err != nil { if ledgerOutOfRange.Load() || rpc.IsLedgerOutOfRange(err) { - // Collect all watched contract IDs and reclamp. - watched, wErr := ing.store.ListWatchedContracts(ctx) - if wErr != nil { - return false, wErr - } - ids := make([]string, len(watched)) - for i, wc := range watched { - ids[i] = wc.ContractID - } - // reclampToOldest saves per-contract cursors. - if err := ing.reclampToOldest(ctx, start, ids...); err != nil { - return false, err - } - // Also update global ingestion state so existing callers - // (and tests) see the reclamped position. - clamped := int64(health.OldestLedger) - 1 - if err := ing.store.SaveIngestionState(ctx, store.IngestionState{LastIngestedLedger: clamped}); err != nil { - return false, err + // Reclamp to the oldest retained ledger; the next runOnce + // retries the window from there. + if rerr := ing.reclampToOldest(ctx, start); rerr != nil { + return false, rerr } return false, nil } @@ -710,6 +672,11 @@ func (ing *Ingester) sweepBatch(ctx context.Context, ledgerOutOfRange *atomic.Bo ledgerOutOfRange.Store(true) return err } + if rpc.IsFailoverReanchor(err) { + ing.log.Warn("failover re-anchor during window sweep: discarding cursor") + ing.discardCursor(ctx) + return err + } if err != nil { return fmt.Errorf("getEvents sweep [%d,%d]: %w", start, end, err) } @@ -737,9 +704,6 @@ func (ing *Ingester) persistEvents(ctx context.Context, rpcEvents []rpc.Event, l if len(rpcEvents) == 0 { return nil } - ctx, span := ing.tracer.Start(ctx, "ingester.persist_events") - defer span.End() - span.SetAttributes(attribute.Int("ingester.event_count", len(rpcEvents)), attribute.Int64("ingester.latest_ledger", int64(latestLedger))) events := make([]store.Event, 0, len(rpcEvents)) for _, re := range rpcEvents { ev, err := ing.toStoreEvent(re) @@ -791,7 +755,6 @@ func (ing *Ingester) persistEvents(ctx context.Context, rpcEvents []rpc.Event, l } ing.log.Info("ingested events", - "network", ing.opts.Network, "count", len(events), "new", inserted, "through_ledger", rpcEvents[len(rpcEvents)-1].Ledger, "latest_ledger", latestLedger) @@ -808,7 +771,7 @@ func (ing *Ingester) persistEvents(ctx context.Context, rpcEvents []rpc.Event, l } func (ing *Ingester) resolvePosition(ctx context.Context) (startLedger uint32, cursor string, err error) { - state, err := ing.store.GetIngestionState(ctx, ing.opts.Network) + state, err := ing.store.GetIngestionState(ctx) if err != nil && !errors.Is(err, store.ErrNotFound) { return 0, "", err } @@ -833,7 +796,7 @@ func (ing *Ingester) resolvePosition(ctx context.Context) (startLedger uint32, c if start < 2 { start = 2 } - ing.log.Info("cold start", "network", ing.opts.Network, "start_ledger", start, "latest_ledger", health.LatestLedger) + ing.log.Info("cold start", "start_ledger", start, "latest_ledger", health.LatestLedger) return uint32(start), "", nil } @@ -843,13 +806,36 @@ func (ing *Ingester) reclampToOldest(ctx context.Context, requested uint32) erro return fmt.Errorf("getHealth while re-clamping: %w", err) } ing.log.Warn("resume ledger fell outside RPC retention window; skipping ahead — events in the gap are lost", - "network", ing.opts.Network, "requested_ledger", requested, "oldest_retained", health.OldestLedger) + "requested_ledger", requested, "oldest_retained", health.OldestLedger) return ing.store.SaveIngestionState(ctx, store.IngestionState{ - Network: ing.opts.Network, LastIngestedLedger: int64(health.OldestLedger) - 1, }) } +// discardCursor reads the persisted ingestion state and re-saves it without +// the cursor, so the next resolvePosition falls through to the +// ledger-based path. Idempotent upserts absorb the overlap from re-scanning. +// +// In windowSweep the internal cursor is never persisted (only +// LastIngestedLedger is saved at sweep end), so this is a defensive no-op +// in that path — it guards against any future change that might persist a +// cursor mid-sweep. +func (ing *Ingester) discardCursor(ctx context.Context) { + state, err := ing.store.GetIngestionState(ctx) + if err != nil { + ing.log.Warn("discardCursor: could not read state", "error", err) + return + } + if state.LastCursor == "" { + return + } + if err := ing.store.SaveIngestionState(ctx, store.IngestionState{ + LastIngestedLedger: state.LastIngestedLedger, + }); err != nil { + ing.log.Warn("discardCursor: could not save state", "error", err) + } +} + // checkLag evaluates the ingest-lag alarm against the chain head and // the persisted ingestion state, with hysteresis so the warn fires // exactly once on crossing and an info "recovered" line fires exactly @@ -1009,7 +995,6 @@ func (ing *Ingester) toStoreEvent(re rpc.Event) (store.Event, error) { Topics: topics, Value: value, CreatedAt: createdAt, - Network: ing.opts.Network, // Keep the raw XDR so `sorotrail replay` can re-decode this event // with a future decoder. Empty when the RPC delivered JSON directly // (xdrFormat "json") — there is no XDR to keep in that case, and diff --git a/internal/ingester/ingester_test.go b/internal/ingester/ingester_test.go index a1313564..5bf03181 100644 --- a/internal/ingester/ingester_test.go +++ b/internal/ingester/ingester_test.go @@ -31,9 +31,10 @@ func TestEventsIngestedTotal_SingleSuccess(t *testing.T) { st := newMockStore() ing := newTestIngester(client, st, Options{StartLedger: 100, PageLimit: 100}) + before := testutil.ToFloat64(metrics.EventsIngested) _, err := ing.runOnce(context.Background()) require.NoError(t, err) - assert.Equal(t, uint64(3), ing.EventsIngestedTotal(), + assert.Equal(t, before+3, testutil.ToFloat64(metrics.EventsIngested), "counter must equal the number of events persisted in one successful write") } @@ -71,13 +72,14 @@ func TestEventsIngestedTotal_CumulativeMultipleWrites(t *testing.T) { st := newMockStore() ing := newTestIngester(client, st, Options{StartLedger: 100, PageLimit: 2}) + before := testutil.ToFloat64(metrics.EventsIngested) _, err := ing.runOnce(context.Background()) require.NoError(t, err) - assert.Equal(t, uint64(2), ing.EventsIngestedTotal(), "after first pass") + assert.Equal(t, before+2, testutil.ToFloat64(metrics.EventsIngested), "after first pass") _, err = ing.runOnce(context.Background()) require.NoError(t, err) - assert.Equal(t, uint64(5), ing.EventsIngestedTotal(), + assert.Equal(t, before+5, testutil.ToFloat64(metrics.EventsIngested), "counter must accumulate across multiple successful writes") } @@ -90,9 +92,10 @@ func TestEventsIngestedTotal_FailedWriteDoesNotIncrement(t *testing.T) { st.upsertErr = fmt.Errorf("database connection lost") ing := newTestIngester(client, st, Options{StartLedger: 100, PageLimit: 100}) + before := testutil.ToFloat64(metrics.EventsIngested) _, err := ing.runOnce(context.Background()) assert.Error(t, err) - assert.Equal(t, uint64(0), ing.EventsIngestedTotal(), + assert.Equal(t, before, testutil.ToFloat64(metrics.EventsIngested), "counter must not increment when the store write fails") } @@ -104,16 +107,18 @@ func TestEventsIngestedTotal_MixedSuccessAndFailure(t *testing.T) { st := newMockStore() ing := newTestIngester(client, st, Options{StartLedger: 100, PageLimit: 100}) + before := testutil.ToFloat64(metrics.EventsIngested) + // First pass succeeds. _, err := ing.runOnce(context.Background()) require.NoError(t, err) - assert.Equal(t, uint64(2), ing.EventsIngestedTotal()) + assert.Equal(t, before+2, testutil.ToFloat64(metrics.EventsIngested)) // Inject failure for second pass. st.upsertErr = fmt.Errorf("deadlock detected") _, err = ing.runOnce(context.Background()) assert.Error(t, err) - assert.Equal(t, uint64(2), ing.EventsIngestedTotal(), + assert.Equal(t, before+2, testutil.ToFloat64(metrics.EventsIngested), "failed write must not change the counter; prior successes preserved") } diff --git a/internal/ingester/integration_test.go b/internal/ingester/integration_test.go index 26ad51b7..1ad815d0 100644 --- a/internal/ingester/integration_test.go +++ b/internal/ingester/integration_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/khaylebfortune/sorotrail/internal/rpc" - "github.com/khaylebfortune/sorotrail/internal/store" - "github.com/khaylebfortune/sorotrail/internal/testdb" + "github.com/sorotrail/sorotrail/internal/rpc" + "github.com/sorotrail/sorotrail/internal/store" + "github.com/sorotrail/sorotrail/internal/testdb" ) // TestIntegration_CursorResumeAcrossRestart proves that the persisted diff --git a/internal/ingester/mocks_test.go b/internal/ingester/mocks_test.go index 403374a5..57cc9b2b 100644 --- a/internal/ingester/mocks_test.go +++ b/internal/ingester/mocks_test.go @@ -69,6 +69,10 @@ func (m *scriptedRPC) GetLedgerEntries(context.Context, rpc.GetLedgerEntriesRequ return rpc.GetLedgerEntriesResponse{}, nil } +func (m *scriptedRPC) SimulateTransaction(context.Context, rpc.SimulateTransactionRequest) (rpc.SimulateTransactionResponse, error) { + return rpc.SimulateTransactionResponse{}, nil +} + // mockRPC scripts getEvents responses in order and records the requests it // received. type mockRPC struct { @@ -120,6 +124,11 @@ func (m *mockRPC) GetLedgerEntries(context.Context, rpc.GetLedgerEntriesRequest) return rpc.GetLedgerEntriesResponse{}, nil } +func (m *mockRPC) SimulateTransaction(context.Context, rpc.SimulateTransactionRequest) (rpc.SimulateTransactionResponse, error) { + return rpc.SimulateTransactionResponse{}, nil +} + +// mockStore is an in-memory Store. type mockStore struct { // Embedded so the mock keeps satisfying store.Store as the // interface grows; unstubbed methods panic if a test calls them. @@ -134,24 +143,28 @@ type mockStore struct { // ingestErr, when set, is returned by GetIngestionState so tests can // exercise the ingester's error path. ingestErr error + // upsertErr, when set, is returned by UpsertEvents. + upsertErr error + // contractCursors backs the per-contract cursor methods. + contractCursors []store.ContractCursor } func newMockStore() *mockStore { return &mockStore{events: map[string]store.Event{}} } -func (m *mockStore) UpsertEvents(_ context.Context, events []store.Event) ([]store.Event, error) { +func (m *mockStore) UpsertEvents(_ context.Context, events []store.Event) (int64, error) { m.mu.Lock() defer m.mu.Unlock() if m.upsertErr != nil { return 0, m.upsertErr } m.upserted = append(m.upserted, events) - var inserted []store.Event + var inserted int64 for _, e := range events { if _, dup := m.events[e.ID]; !dup { m.events[e.ID] = e - inserted = append(inserted, e) + inserted++ } } return inserted, nil @@ -242,7 +255,7 @@ func (m *mockStore) ListOpenFindingsByRange(_ context.Context, _ string, _, _ in return store.AuditFinding{}, store.ErrNotFound } -func (m *mockStore) GetIngestionState(_ context.Context, _ string) (store.IngestionState, error) { +func (m *mockStore) GetIngestionState(_ context.Context) (store.IngestionState, error) { m.mu.Lock() defer m.mu.Unlock() if m.ingestErr != nil { @@ -338,7 +351,21 @@ func (m *mockStore) Ping(context.Context) error { return nil } func (m *mockStore) GetContractSpec(context.Context, string) ([]byte, error) { return nil, store.ErrNotFound } -func (m *mockStore) SetContractSpec(context.Context, string, string, []byte) error { return nil } +func (m *mockStore) ListContractIDs(context.Context) ([]string, error) { + return nil, nil +} +func (m *mockStore) GetContractMeta(context.Context, string) (store.ContractMeta, error) { + return store.ContractMeta{}, store.ErrNotFound +} +func (m *mockStore) UpsertContractMeta(context.Context, store.ContractMeta) error { + return nil +} +func (m *mockStore) CountContractEvents(context.Context, string) (int64, error) { + return 0, nil +} +func (m *mockStore) ListContractsNeedingRefresh(context.Context, time.Time) ([]string, error) { + return nil, nil +} func (m *mockStore) CreateSubscription(_ context.Context, sub store.Subscription) (store.Subscription, error) { sub.ID = 1 diff --git a/internal/ingester/token_processor.go b/internal/ingester/token_processor.go deleted file mode 100644 index 97a284bb..00000000 --- a/internal/ingester/token_processor.go +++ /dev/null @@ -1,235 +0,0 @@ -package ingester - -import ( - "context" - "errors" - "fmt" - "log/slog" - "math/big" - "sort" - - "github.com/khaylebfortune/sorotrail/internal/decode" - "github.com/khaylebfortune/sorotrail/internal/store" -) - -// TokenBalanceProcessor derives per-address token balances from SEP-41 token -// events and persists them atomically. It implements EventNotifier so it can -// be wired directly into the ingester's notification chain. -// -// Correctness under re-ingest: -// - Idempotency is guaranteed by the last_applied_event watermark: events -// whose ID is <= the stored watermark are skipped. Re-ingested events -// with IDs below the watermark do not double-count. -// - Derive-from-scratch semantics (e.g. via the decoder replay tool) can be -// built on top of NextReplayBatch + UpsertTokenBalanceState for a clean -// slate. -// -// Honest limitation: -// - Balances are derived only from events SoroTrail has stored. Contracts -// with pre-ingestion history (events that were emitted before SoroTrail -// started indexing) will show artificially low balances. Use -// `sorotrail backfill` to close the gap, or the `earliest_ledger` field -// exposed on the holders endpoint to judge coverage. -type TokenBalanceProcessor struct { - store store.Store - log *slog.Logger -} - -// NewTokenBalanceProcessor creates a processor that persists token balance -// updates derived from incoming events. -func NewTokenBalanceProcessor(st store.Store, log *slog.Logger) *TokenBalanceProcessor { - return &TokenBalanceProcessor{store: st, log: log} -} - -// NotifyEvents implements EventNotifier. It is called after events are -// persisted and must not block ingestion for long. -func (p *TokenBalanceProcessor) NotifyEvents(ctx context.Context, events []store.Event) { - if ctx.Err() != nil { - return - } - if err := p.processEvents(ctx, events); err != nil { - p.log.Error("token balance processor", "error", err) - } -} - -// processEvents extracts SEP-41 token events, groups them by contract, -// and applies balance changes atomically per contract. -func (p *TokenBalanceProcessor) processEvents(ctx context.Context, events []store.Event) error { - // Parse all token events from the batch. - var tokenEvents []*decode.TokenEvent - for _, ev := range events { - te := decode.ParseTokenEvent(ev.ContractID, ev.ID, ev.Ledger, ev.Network, ev.Topics, ev.Value) - if te == nil { - continue - } - tokenEvents = append(tokenEvents, te) - } - if len(tokenEvents) == 0 { - return nil - } - - // Group by (network, contract_id). - type contractKey struct { - network string - contractID string - } - groups := make(map[contractKey][]*decode.TokenEvent) - for _, te := range tokenEvents { - key := contractKey{network: te.Network, contractID: te.ContractID} - groups[key] = append(groups[key], te) - } - - // Process each contract group. - for key, group := range groups { - if err := p.processContractGroup(ctx, key.network, key.contractID, group); err != nil { - return fmt.Errorf("processing token balances for %s/%s: %w", key.network, key.contractID, err) - } - } - return nil -} - -// processContractGroup applies a batch of token events for a single contract. -func (p *TokenBalanceProcessor) processContractGroup(ctx context.Context, network, contractID string, events []*decode.TokenEvent) error { - if len(events) == 0 { - return nil - } - - // Sort events by event ID (which sorts by ledger) to ensure deterministic processing. - sort.Slice(events, func(i, j int) bool { - return events[i].EventID < events[j].EventID - }) - - // Determine the last event ID in this batch for the watermark. - lastEventID := events[len(events)-1].EventID - lastLedger := events[len(events)-1].Ledger - - // Get current state to skip already-applied events. - currentState, err := p.store.GetTokenBalanceState(ctx, network, contractID) - if err != nil && !errors.Is(err, store.ErrNotFound) { - return fmt.Errorf("getting token balance state: %w", err) - } - - // Filter out events that have already been applied. - var newEvents []*decode.TokenEvent - if err == nil && currentState.LastAppliedEventID != "" { - for _, te := range events { - if te.EventID > currentState.LastAppliedEventID { - newEvents = append(newEvents, te) - } - } - } else { - newEvents = events - } - - if len(newEvents) == 0 { - return nil - } - - // Collect all changes and unique addresses. - type addrChange struct { - address string - delta *big.Int // positive for credit, negative for debit - ledger int64 - } - changes := make([]addrChange, 0, len(newEvents)*2) - addrSet := make(map[string]struct{}) - - for _, te := range newEvents { - switch te.Kind { - case decode.TokenTransfer: - changes = append(changes, - addrChange{address: te.From, delta: new(big.Int).Neg(te.Amount), ledger: te.Ledger}, - addrChange{address: te.To, delta: new(big.Int).Set(te.Amount), ledger: te.Ledger}, - ) - addrSet[te.From] = struct{}{} - addrSet[te.To] = struct{}{} - - case decode.TokenMint: - changes = append(changes, addrChange{address: te.To, delta: new(big.Int).Set(te.Amount), ledger: te.Ledger}) - addrSet[te.To] = struct{}{} - - case decode.TokenBurn: - changes = append(changes, addrChange{address: te.From, delta: new(big.Int).Neg(te.Amount), ledger: te.Ledger}) - addrSet[te.From] = struct{}{} - - case decode.TokenClawback: - changes = append(changes, addrChange{address: te.From, delta: new(big.Int).Neg(te.Amount), ledger: te.Ledger}) - addrSet[te.From] = struct{}{} - } - } - - // Read all current balances for this contract in one query. - // TODO: For contracts with many holders (10k+), consider targeted queries - // or batching to avoid fetching every row. - currentBalances, err := p.readAllBalances(ctx, network, contractID) - if err != nil { - return fmt.Errorf("reading current balances: %w", err) - } - - // Apply all changes. - updateMap := make(map[string]*store.TokenBalanceUpdate) - for _, ch := range changes { - current := currentBalances[ch.address] - if current == nil { - current = new(big.Int) - } - newBalance := new(big.Int).Add(current, ch.delta) - if newBalance.Sign() < 0 { - p.log.Warn("negative balance clamped to zero", - "contract_id", contractID, - "address", ch.address, - "current", current.String(), - "delta", ch.delta.String(), - "network", network, - ) - newBalance = new(big.Int) - } - currentBalances[ch.address] = newBalance - - if existing, ok := updateMap[ch.address]; ok { - existing.Balance = newBalance - if ch.ledger > existing.LastLedger { - existing.LastLedger = ch.ledger - } - } else { - updateMap[ch.address] = &store.TokenBalanceUpdate{ - Address: ch.address, - Balance: new(big.Int).Set(newBalance), - LastLedger: ch.ledger, - } - } - } - - // Build the final update list. - updates := make([]store.TokenBalanceUpdate, 0, len(updateMap)) - for _, u := range updateMap { - updates = append(updates, *u) - } - - // Persist atomically. - state := store.TokenBalanceState{ - Network: network, - ContractID: contractID, - LastAppliedEventID: lastEventID, - LastLedger: lastLedger, - } - return p.store.UpsertTokenBalances(ctx, network, state, updates) -} - -// readAllBalances reads all current token balances for a contract into a map. -func (p *TokenBalanceProcessor) readAllBalances(ctx context.Context, network, contractID string) (map[string]*big.Int, error) { - balances, _, err := p.store.GetTokenBalances(ctx, contractID, network, "0", "", 100000) - if err != nil { - return nil, err - } - result := make(map[string]*big.Int, len(balances)) - for _, tb := range balances { - n := new(big.Int) - n, ok := n.SetString(tb.Balance, 10) - if !ok { - return nil, fmt.Errorf("parsing balance for %s: %q", tb.Address, tb.Balance) - } - result[tb.Address] = n - } - return result, nil -} diff --git a/internal/meta/worker.go b/internal/meta/worker.go new file mode 100644 index 00000000..a2c553bc --- /dev/null +++ b/internal/meta/worker.go @@ -0,0 +1,358 @@ +// Package meta runs a background worker that enriches contracts with token +// metadata (name, symbol, decimals) by calling the SEP-41 token interface +// via RPC simulation. It never blocks ingestion and rate-limits itself. +package meta + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/stellar/go/strkey" + "github.com/stellar/go/xdr" + + "github.com/sorotrail/sorotrail/internal/rpc" + "github.com/sorotrail/sorotrail/internal/store" +) + +// Worker periodically discovers contracts from ingested events and attempts +// to read SEP-41 token metadata (name, symbol, decimals) via RPC simulation. +// Results are cached in the contract_meta table; non-token contracts are +// negatively cached so they're never re-probed. +type Worker struct { + rpc rpc.Client + store store.Store + log *slog.Logger + ttl time.Duration + poll time.Duration +} + +// New creates a metadata enrichment worker. +// ttl is how long metadata is considered fresh before re-fetching. +// pollInterval is the sleep between enrichment passes. +func New(rpcClient rpc.Client, st store.Store, log *slog.Logger, ttl, pollInterval time.Duration) *Worker { + return &Worker{ + rpc: rpcClient, + store: st, + log: log, + ttl: ttl, + poll: pollInterval, + } +} + +// Run starts the enrichment loop. It blocks until ctx is canceled. +func (w *Worker) Run(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if err := w.runOnce(ctx); err != nil && !errors.Is(err, context.Canceled) { + w.log.Warn("metadata enrichment pass failed", "error", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(w.poll): + } + } +} + +// runOnce performs one enrichment pass: discover contracts needing metadata, +// then probe each one for token metadata. A small sleep between contracts +// ensures the worker doesn't overwhelm the RPC even inside the rate limiter. +func (w *Worker) runOnce(ctx context.Context) error { + var contractIDs []string + var err error + + // Use TTL-based refresh when configured, otherwise fetch all. + if w.ttl > 0 { + cutoff := time.Now().Add(-w.ttl) + contractIDs, err = w.store.ListContractsNeedingRefresh(ctx, cutoff) + } else { + contractIDs, err = w.store.ListContractIDs(ctx) + } + if err != nil { + return fmt.Errorf("listing contracts for enrichment: %w", err) + } + + if len(contractIDs) == 0 { + return nil + } + + w.log.Debug("contract metadata enrichment pass", + "candidates", len(contractIDs), + "ttl", w.ttl, + ) + + var enriched, nonToken, failed int + for _, contractID := range contractIDs { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // Pause between contracts so enrichment doesn't compete with + // ingestion for RPC budget. 500ms between calls gives ~2 req/s + // which is safe even on public endpoints. + if enriched+nonToken+failed > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + meta, err := w.fetchTokenMeta(ctx, contractID) + if err != nil { + w.log.Debug("failed to read token metadata for contract", + "contract_id", contractID, + "error", err, + ) + failed++ + // Don't write anything on transient RPC errors — if the RPC is + // down, writing IsToken=true with nil fields would mark every + // contract as "unresolved token", causing unnecessary retries. + // Contracts that trap on name() are handled inside fetchTokenMeta + // (negative cache with is_token=false). + continue + } + if !meta.IsToken { + nonToken++ + } else { + enriched++ + } + if err := w.store.UpsertContractMeta(ctx, meta); err != nil { + w.log.Warn("failed to persist contract meta", + "contract_id", contractID, + "error", err, + ) + } + } + + w.log.Info("metadata enrichment pass complete", + "candidates", len(contractIDs), + "enriched", enriched, + "non_token", nonToken, + "failed", failed, + ) + return nil +} + +// fetchTokenMeta attempts to read SEP-41 token metadata from a contract. +// It tries all three calls (name, symbol, decimals) and returns whatever +// it can get. If the first call (name) traps with a recognizable error, +// the contract is marked as non-token. +func (w *Worker) fetchTokenMeta(ctx context.Context, contractID string) (store.ContractMeta, error) { + now := time.Now() + meta := store.ContractMeta{ + ContractID: contractID, + FetchedAt: now, + } + + name, err := w.simulateContractCall(ctx, contractID, "name") + if err != nil { + if isContractTrap(err) { + // Contract doesn't have a name() function — not a token. + meta.IsToken = false + meta.FetchedAt = now + return meta, nil + } + return meta, fmt.Errorf("simulating name(): %w", err) + } + meta.IsToken = true + meta.Name = &name + + symbol, err := w.simulateContractCall(ctx, contractID, "symbol") + if err != nil { + w.log.Debug("simulating symbol() failed", "contract_id", contractID, "error", err) + } else { + meta.Symbol = &symbol + } + + decimalsRaw, err := w.simulateContractCall(ctx, contractID, "decimals") + if err != nil { + w.log.Debug("simulating decimals() failed", "contract_id", contractID, "error", err) + } else { + // Decimals is returned as a u32 ScVal. Parse it. + dec, parseErr := parseDecimals(decimalsRaw) + if parseErr != nil { + w.log.Debug("parsing decimals() result", "contract_id", contractID, "raw", decimalsRaw, "error", parseErr) + } else { + meta.Decimals = &dec + } + } + + return meta, nil +} + +// simulateContractCall builds a transaction that invokes the named function +// on a contract and returns the first result value as a string. +// The transaction uses a dummy source account since simulation doesn't +// require a real signer. +func (w *Worker) simulateContractCall(ctx context.Context, contractID, functionName string) (string, error) { + // Decode contract ID to raw bytes for the ScAddress. + contractBytes, err := decodeContractID(contractID) + if err != nil { + return "", fmt.Errorf("decoding contract ID: %w", err) + } + + // Build the InvokeHostFunction operation. + // The function is invoked with no arguments (name, symbol, decimals are + // all zero-arg functions in SEP-41). + var contractHash xdr.Hash + copy(contractHash[:], contractBytes[:32]) + + invokeArgs := xdr.InvokeContractArgs{ + ContractAddress: xdr.ScAddress{ + Type: xdr.ScAddressTypeScAddressTypeContract, + ContractId: (*xdr.ContractId)(&contractHash), + }, + FunctionName: xdr.ScSymbol(functionName), + Args: nil, // zero-arg call + } + + hostFn := xdr.HostFunction{ + Type: xdr.HostFunctionTypeHostFunctionTypeInvokeContract, + InvokeContract: &invokeArgs, + } + + // Build a minimal transaction envelope for simulation. + // Source account: a dummy MuxedAccount (the RPC doesn't validate it for + // simulation). We use a public-key type with a zeroed key. + var sourceAccount xdr.MuxedAccount + sourceAccount.Type = xdr.CryptoKeyTypeKeyTypeEd25519 + var zeroKey xdr.Uint256 + sourceAccount.Ed25519 = &zeroKey + + op := xdr.Operation{ + SourceAccount: nil, + Body: xdr.OperationBody{ + Type: xdr.OperationTypeInvokeHostFunction, + InvokeHostFunctionOp: &xdr.InvokeHostFunctionOp{ + HostFunction: hostFn, + Auth: nil, // no auth needed for simulation + }, + }, + } + + tx := xdr.Transaction{ + SourceAccount: sourceAccount, + Fee: 100, // minimal fee for simulation + SeqNum: 0, + Operations: []xdr.Operation{op}, + Memo: xdr.Memo{Type: xdr.MemoTypeMemoNone}, + } + + env := xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{Tx: tx}, + } + + txBase64, err := xdr.MarshalBase64(env) + if err != nil { + return "", fmt.Errorf("marshaling transaction envelope: %w", err) + } + + resp, err := w.rpc.SimulateTransaction(ctx, rpc.SimulateTransactionRequest{ + Transaction: txBase64, + }) + if err != nil { + return "", fmt.Errorf("simulateTransaction: %w", err) + } + + if resp.Error != "" { + return "", &simulationError{msg: resp.Error} + } + + if len(resp.Results) == 0 { + return "", fmt.Errorf("simulateTransaction returned no results") + } + + // Results are JSON strings containing base64-encoded ScVal XDR. + var resultB64 string + if err := json.Unmarshal(resp.Results[0], &resultB64); err != nil { + return "", fmt.Errorf("unmarshaling result JSON: %w", err) + } + + return scValFromBase64(resultB64) +} + +// simulationError is returned when the RPC simulation itself fails (contract +// trap, missing function, etc.), not when the RPC call fails. +type simulationError struct { + msg string +} + +func (e *simulationError) Error() string { return "simulation error: " + e.msg } + +// isContractTrap reports whether the error indicates the contract rejected +// the call (e.g. unknown function). +func isContractTrap(err error) bool { + var simErr *simulationError + return errors.As(err, &simErr) +} + +// scValFromBase64 decodes a base64-encoded ScVal XDR string and extracts +// its string representation. Handles the common ScVal types returned by +// token functions: symbol, string, u32. +func scValFromBase64(b64 string) (string, error) { + var val xdr.ScVal + if err := xdr.SafeUnmarshalBase64(b64, &val); err != nil { + return "", fmt.Errorf("unmarshaling ScVal: %w", err) + } + switch val.Type { + case xdr.ScValTypeScvSymbol: + if val.Sym != nil { + return string(*val.Sym), nil + } + case xdr.ScValTypeScvString: + if val.Str != nil { + return string(*val.Str), nil + } + case xdr.ScValTypeScvU32: + if val.U32 != nil { + return fmt.Sprintf("%d", uint32(*val.U32)), nil + } + case xdr.ScValTypeScvU64: + if val.U64 != nil { + return fmt.Sprintf("%d", uint64(*val.U64)), nil + } + default: + return "", fmt.Errorf("unsupported ScVal type %d", val.Type) + } + return "", fmt.Errorf("empty ScVal") +} + +// parseDecimals converts a string representation of a u32 returned by the +// decimals() function into an int. +func parseDecimals(raw string) (int, error) { + var n uint32 + if _, err := fmt.Sscanf(raw, "%d", &n); err != nil { + return 0, fmt.Errorf("parsing decimals %q: %w", raw, err) + } + return int(n), nil +} + +// decodeContractID converts a Stellar contract ID string (C... base32 strkey) +// into its raw 32-byte hash. +func decodeContractID(id string) ([]byte, error) { + if len(id) != 56 || id[0] != 'C' { + return nil, fmt.Errorf("invalid contract ID format %q", id) + } + raw, err := strkey.Decode(strkey.VersionByteContract, id) + if err != nil { + return nil, fmt.Errorf("decoding contract ID: %w", err) + } + if len(raw) < 32 { + return nil, fmt.Errorf("decoded contract ID too short: len=%d", len(raw)) + } + return raw[:32], nil +} diff --git a/internal/meta/worker_test.go b/internal/meta/worker_test.go new file mode 100644 index 00000000..f17747ae --- /dev/null +++ b/internal/meta/worker_test.go @@ -0,0 +1,431 @@ +package meta + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sorotrail/sorotrail/internal/rpc" + "github.com/sorotrail/sorotrail/internal/store" +) + +func testLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// mockRPC records simulateTransaction calls and returns scripted responses. +type mockRPC struct { + mu sync.Mutex + simCalls []rpc.SimulateTransactionRequest + simResps []rpc.SimulateTransactionResponse + simErrs []error +} + +func (m *mockRPC) GetEvents(context.Context, rpc.GetEventsRequest) (rpc.GetEventsResponse, error) { + return rpc.GetEventsResponse{}, nil +} +func (m *mockRPC) GetLatestLedger(context.Context) (rpc.LatestLedger, error) { + return rpc.LatestLedger{}, nil +} +func (m *mockRPC) GetHealth(context.Context) (rpc.Health, error) { + return rpc.Health{}, nil +} +func (m *mockRPC) GetLedgerEntries(context.Context, rpc.GetLedgerEntriesRequest) (rpc.GetLedgerEntriesResponse, error) { + return rpc.GetLedgerEntriesResponse{}, nil +} +func (m *mockRPC) SimulateTransaction(_ context.Context, req rpc.SimulateTransactionRequest) (rpc.SimulateTransactionResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.simCalls = append(m.simCalls, req) + idx := len(m.simCalls) - 1 + var resp rpc.SimulateTransactionResponse + if idx < len(m.simResps) { + resp = m.simResps[idx] + } + var err error + if idx < len(m.simErrs) { + err = m.simErrs[idx] + } + return resp, err +} + +// stubStore records contract meta operations in memory. +type stubStore struct { + mu sync.Mutex + meta map[string]store.ContractMeta + contractIDs []string + refreshErr error +} + +func newStubStore() *stubStore { + return &stubStore{meta: map[string]store.ContractMeta{}} +} + +func (s *stubStore) UpsertEvents(context.Context, []store.Event) (int64, error) { return 0, nil } +func (s *stubStore) ReplaceEventsInRange(context.Context, []store.Event, int64, int64) error { + return nil +} +func (s *stubStore) GetEvent(context.Context, string, store.Scope) (store.Event, error) { + return store.Event{}, store.ErrNotFound +} +func (s *stubStore) GetEventsByTxHash(context.Context, string, string) ([]store.Event, error) { + return nil, nil +} +func (s *stubStore) EventExists(context.Context, string, store.Scope) (bool, error) { return false, nil } +func (s *stubStore) QueryEvents(context.Context, store.EventFilter) ([]store.Event, string, error) { + return nil, "", nil +} +func (s *stubStore) CountEvents(context.Context, store.EventFilter) (int64, error) { return 0, nil } +func (s *stubStore) AggregateEvents(context.Context, store.EventFilter, string) ([]store.AggregateBucket, error) { + return nil, nil +} +func (s *stubStore) LedgerRangeCensus(context.Context, int64, int64, bool) ([]store.LedgerCensus, error) { + return nil, nil +} +func (s *stubStore) ListContracts(context.Context, store.ContractsFilter) ([]store.ContractSummary, string, error) { + return nil, "", nil +} +func (s *stubStore) CountContracts(context.Context, store.ContractsFilter) (int64, error) { + return 0, nil +} +func (s *stubStore) DeadLetterEvent(context.Context, store.DeadLetterInput) (store.DeadLetter, error) { + return store.DeadLetter{}, nil +} +func (s *stubStore) ListDeadLetters(context.Context, string, int, string) ([]store.DeadLetter, string, error) { + return nil, "", nil +} +func (s *stubStore) GetDeadLetter(context.Context, int64) (store.DeadLetter, error) { + return store.DeadLetter{}, store.ErrNotFound +} +func (s *stubStore) DeleteDeadLetter(context.Context, int64) error { return nil } +func (s *stubStore) GetIngestionState(context.Context) (store.IngestionState, error) { + return store.IngestionState{}, store.ErrNotFound +} +func (s *stubStore) SaveIngestionState(context.Context, store.IngestionState) error { return nil } +func (s *stubStore) GetAuditState(context.Context, string) (store.AuditState, error) { + return store.AuditState{}, store.ErrNotFound +} +func (s *stubStore) SaveAuditState(context.Context, store.AuditState) error { return nil } +func (s *stubStore) SaveAuditStateIfGreater(context.Context, string, int64) (store.AuditState, error) { + return store.AuditState{}, nil +} +func (s *stubStore) ListWatchedContracts(context.Context) ([]store.WatchedContract, error) { + return nil, nil +} +func (s *stubStore) AddWatchedContract(context.Context, string) error { return nil } +func (s *stubStore) RemoveWatchedContract(context.Context, string) error { return nil } +func (s *stubStore) GetContractCursor(context.Context, string) (store.ContractCursor, error) { + return store.ContractCursor{}, store.ErrNotFound +} +func (s *stubStore) SaveContractCursor(context.Context, store.ContractCursor) error { return nil } +func (s *stubStore) DeleteContractCursor(context.Context, string) error { return nil } +func (s *stubStore) ListContractCursors(context.Context) ([]store.ContractCursor, error) { + return nil, nil +} +func (s *stubStore) RecordAuditFinding(context.Context, store.AuditFinding) (store.AuditFinding, error) { + return store.AuditFinding{}, nil +} +func (s *stubStore) UpdateAuditFinding(context.Context, store.AuditFinding) error { return nil } +func (s *stubStore) ListOpenFindingsByRange(context.Context, string, int64, int64) (store.AuditFinding, error) { + return store.AuditFinding{}, store.ErrNotFound +} +func (s *stubStore) Stats(context.Context, store.Scope) (store.Stats, error) { return store.Stats{}, nil } +func (s *stubStore) Ping(context.Context) error { return nil } +func (s *stubStore) GetContractSpec(context.Context, string) ([]byte, error) { + return nil, store.ErrNotFound +} +func (s *stubStore) SetContractSpec(context.Context, string, string, []byte) error { + return nil +} +func (s *stubStore) CreateSubscription(_ context.Context, sub store.Subscription) (store.Subscription, error) { + sub.ID = 1 + return sub, nil +} +func (s *stubStore) GetSubscription(_ context.Context, id int64, _ store.SubscriptionOwner) (store.Subscription, error) { + return store.Subscription{}, store.ErrNotFound +} +func (s *stubStore) ListSubscriptions(context.Context, store.SubscriptionOwner) ([]store.Subscription, error) { + return nil, nil +} +func (s *stubStore) UpdateSubscription(_ context.Context, sub store.Subscription, _ store.SubscriptionOwner) (store.Subscription, error) { + return sub, nil +} +func (s *stubStore) DeleteSubscription(context.Context, int64, store.SubscriptionOwner) error { + return nil +} +func (s *stubStore) ListEnabledSubscriptions(context.Context) ([]store.Subscription, error) { + return nil, nil +} +func (s *stubStore) IncrementSubscriptionFailures(context.Context, int64, int) (int, bool, error) { + return 0, false, nil +} +func (s *stubStore) ResetSubscriptionFailures(context.Context, int64) error { return nil } +func (s *stubStore) RecordDeliveryAttempt(_ context.Context, a store.DeliveryAttempt) (store.DeliveryAttempt, error) { + a.ID = 1 + return a, nil +} +func (s *stubStore) ListDeliveryAttempts(context.Context, int64, int, store.SubscriptionOwner) ([]store.DeliveryAttempt, error) { + return nil, nil +} +func (s *stubStore) DeleteEventsBeforeLedger(context.Context, int64) (int64, error) { return 0, nil } +func (s *stubStore) DeleteEventsBefore(context.Context, int64, time.Time, int) (int64, error) { + return 0, nil +} +func (s *stubStore) UpsertAddressRefs(context.Context, []store.AddressRef) error { return nil } +func (s *stubStore) QueryAddressEvents(context.Context, string, store.EventFilter) ([]store.Event, string, error) { + return nil, "", nil +} +func (s *stubStore) CountAddressEvents(context.Context, string) (int64, error) { return 0, nil } +func (s *stubStore) GetAddressSummary(context.Context, string) (store.AddressSummary, error) { + return store.AddressSummary{}, nil +} +func (s *stubStore) MigrationVersion(context.Context) (int, bool, error) { return 0, false, nil } +func (s *stubStore) GetContractMeta(_ context.Context, contractID string) (store.ContractMeta, error) { + s.mu.Lock() + defer s.mu.Unlock() + if m, ok := s.meta[contractID]; ok { + return m, nil + } + return store.ContractMeta{}, store.ErrNotFound +} +func (s *stubStore) UpsertContractMeta(_ context.Context, m store.ContractMeta) error { + s.mu.Lock() + defer s.mu.Unlock() + s.meta[m.ContractID] = m + return nil +} +func (s *stubStore) ListContractIDs(context.Context) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.contractIDs, nil +} +func (s *stubStore) ListContractsNeedingRefresh(_ context.Context, _ time.Time) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.contractIDs, s.refreshErr +} +func (s *stubStore) CountContractEvents(context.Context, string) (int64, error) { return 0, nil } + +// makeSymbolResult returns a simulateTransaction result that encodes a +// symbol ScVal. The result JSON is a string containing base64-encoded XDR. +func makeSymbolResult(t *testing.T, sym string) rpc.SimulateTransactionResponse { + t.Helper() + // Encode a ScVal symbol using stellar/go xdr. + val := xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: (*xdr.ScSymbol)(&sym), + } + b64, err := xdr.MarshalBase64(val) + require.NoError(t, err) + resultJSON, err := json.Marshal(b64) + require.NoError(t, err) + return rpc.SimulateTransactionResponse{ + Results: []json.RawMessage{resultJSON}, + } +} + +// makeU32Result returns a simulateTransaction result that encodes a +// u32 ScVal (for decimals). +func makeU32Result(t *testing.T, n uint32) rpc.SimulateTransactionResponse { + t.Helper() + val := xdr.ScVal{ + Type: xdr.ScValTypeScvU32, + U32: (*xdr.Uint32)(&n), + } + b64, err := xdr.MarshalBase64(val) + require.NoError(t, err) + resultJSON, err := json.Marshal(b64) + require.NoError(t, err) + return rpc.SimulateTransactionResponse{ + Results: []json.RawMessage{resultJSON}, + } +} + +// makeTrapResult returns a simulateTransaction response where the simulation +// failed (contract trapped), meaning it doesn't have the called function. +func makeTrapResult(t *testing.T) rpc.SimulateTransactionResponse { + t.Helper() + return rpc.SimulateTransactionResponse{ + Error: "HostError: Error(Contract, #1)", + } +} + +// testContract is a known testnet contract used in tests. +const testContract = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC" + +func TestFetchTokenMeta_Success(t *testing.T) { + rpc := &mockRPC{ + simResps: []rpc.SimulateTransactionResponse{ + makeSymbolResult(t, "USD Coin"), // name() + makeSymbolResult(t, "USDC"), // symbol() + makeU32Result(t, 6), // decimals() + }, + } + st := newStubStore() + w := New(rpc, st, testLogger(), 0, time.Minute) + + meta, err := w.fetchTokenMeta(context.Background(), testContract) + require.NoError(t, err) + assert.True(t, meta.IsToken) + require.NotNil(t, meta.Name) + assert.Equal(t, "USD Coin", *meta.Name) + require.NotNil(t, meta.Symbol) + assert.Equal(t, "USDC", *meta.Symbol) + require.NotNil(t, meta.Decimals) + assert.Equal(t, 6, *meta.Decimals) + assert.Equal(t, 3, len(rpc.simCalls), "one call each for name, symbol, decimals") +} + +func TestFetchTokenMeta_NonTokenContract_NegativeCache(t *testing.T) { + // First call (name) traps — contract doesn't implement SEP-41. + rpc := &mockRPC{ + simResps: []rpc.SimulateTransactionResponse{ + makeTrapResult(t), + }, + } + st := newStubStore() + w := New(rpc, st, testLogger(), 0, time.Minute) + + meta, err := w.fetchTokenMeta(context.Background(), testContract) + require.NoError(t, err) + assert.False(t, meta.IsToken, "contract that traps on name() is not a token") + assert.Nil(t, meta.Name) + assert.Nil(t, meta.Symbol) + assert.Nil(t, meta.Decimals) +} + +func TestFetchTokenMeta_RPCError_ReturnsError(t *testing.T) { + rpc := &mockRPC{ + simErrs: []error{errors.New("connection refused")}, + } + st := newStubStore() + w := New(rpc, st, testLogger(), 0, time.Minute) + + _, err := w.fetchTokenMeta(context.Background(), testContract) + require.Error(t, err) + assert.Contains(t, err.Error(), "connection refused") +} + +func TestWorker_RunOnce_EnrichesAndPersists(t *testing.T) { + rpc := &mockRPC{ + simResps: []rpc.SimulateTransactionResponse{ + makeSymbolResult(t, "Test Token"), + makeSymbolResult(t, "TEST"), + makeU32Result(t, 18), + }, + } + st := newStubStore() + st.contractIDs = []string{testContract} + w := New(rpc, st, testLogger(), 0, time.Minute) + + err := w.runOnce(context.Background()) + require.NoError(t, err) + + // Verify meta was persisted. + meta, err := st.GetContractMeta(context.Background(), testContract) + require.NoError(t, err) + assert.True(t, meta.IsToken) + require.NotNil(t, meta.Name) + assert.Equal(t, "Test Token", *meta.Name) +} + +func TestWorker_RunOnce_NegativeCachesNonToken(t *testing.T) { + rpc := &mockRPC{ + simResps: []rpc.SimulateTransactionResponse{ + makeTrapResult(t), + }, + } + st := newStubStore() + st.contractIDs = []string{testContract} + w := New(rpc, st, testLogger(), 0, time.Minute) + + err := w.runOnce(context.Background()) + require.NoError(t, err) + + meta, err := st.GetContractMeta(context.Background(), testContract) + require.NoError(t, err) + assert.False(t, meta.IsToken, "non-token contract should be negatively cached") +} + +func TestWorker_RunOnce_NoContracts(t *testing.T) { + rpc := &mockRPC{} + st := newStubStore() + st.contractIDs = nil + w := New(rpc, st, testLogger(), 0, time.Minute) + + err := w.runOnce(context.Background()) + require.NoError(t, err) + assert.Empty(t, rpc.simCalls, "no RPC calls when there are no contracts") +} + +func TestWorker_RunOnce_TransientRPCError_Skip(t *testing.T) { + rpc := &mockRPC{ + simErrs: []error{errors.New("timeout")}, + } + st := newStubStore() + st.contractIDs = []string{testContract} + w := New(rpc, st, testLogger(), 0, time.Minute) + + err := w.runOnce(context.Background()) + require.NoError(t, err) + + // Transient errors should NOT write a meta row (no IsToken=true with nil fields). + _, err = st.GetContractMeta(context.Background(), testContract) + assert.ErrorIs(t, err, store.ErrNotFound, "transient RPC errors should not persist any meta row") +} + +func TestParseDecimals(t *testing.T) { + tests := []struct { + raw string + want int + errMsg string + }{ + {"6", 6, ""}, + {"18", 18, ""}, + {"0", 0, ""}, + {"7", 7, ""}, + {"abc", 0, "parsing decimals"}, + } + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + got, err := parseDecimals(tt.raw) + if tt.errMsg != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestIsContractTrap(t *testing.T) { + assert.True(t, isContractTrap(&simulationError{msg: "trapped"})) + assert.False(t, isContractTrap(errors.New("connection refused"))) + assert.False(t, isContractTrap(nil)) +} + +func TestDecodeContractID(t *testing.T) { + // Valid contract ID. + b, err := decodeContractID(testContract) + require.NoError(t, err) + assert.Len(t, b, 32) + + // Invalid length. + _, err = decodeContractID("short") + assert.Error(t, err) + + // Wrong prefix. + _, err = decodeContractID("GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + assert.Error(t, err) +} diff --git a/internal/pruner/pruner_test.go b/internal/pruner/pruner_test.go index d7b1f47b..bba9dcd1 100644 --- a/internal/pruner/pruner_test.go +++ b/internal/pruner/pruner_test.go @@ -66,7 +66,7 @@ func (m *mockStore) LedgerRangeCensus(context.Context, int64, int64, bool) ([]st return nil, nil } -func (m *mockStore) GetAuditState(context.Context) (store.AuditState, error) { +func (m *mockStore) GetAuditState(context.Context, string) (store.AuditState, error) { return store.AuditState{}, store.ErrNotFound } @@ -74,7 +74,7 @@ func (m *mockStore) SaveAuditState(_ context.Context, s store.AuditState) error return nil } -func (m *mockStore) SaveAuditStateIfGreater(_ context.Context, ledger int64) (store.AuditState, error) { +func (m *mockStore) SaveAuditStateIfGreater(_ context.Context, _ string, ledger int64) (store.AuditState, error) { return store.AuditState{VerifiedThroughLedger: ledger}, nil } @@ -87,7 +87,7 @@ func (m *mockStore) UpdateAuditFinding(context.Context, store.AuditFinding) erro return nil } -func (m *mockStore) ListOpenFindingsByRange(context.Context, int64, int64) (store.AuditFinding, error) { +func (m *mockStore) ListOpenFindingsByRange(context.Context, string, int64, int64) (store.AuditFinding, error) { return store.AuditFinding{}, store.ErrNotFound } diff --git a/internal/rpc/circuit_breaker_test.go b/internal/rpc/circuit_breaker_test.go index 85d1b2ca..3c7b844c 100644 --- a/internal/rpc/circuit_breaker_test.go +++ b/internal/rpc/circuit_breaker_test.go @@ -49,6 +49,10 @@ func (m *mockClient) GetLedgerEntries(ctx context.Context, req GetLedgerEntriesR } return GetLedgerEntriesResponse{}, nil } +func (m *mockClient) SimulateTransaction(ctx context.Context, req SimulateTransactionRequest) (SimulateTransactionResponse, error) { + m.callCount.Add(1) + return SimulateTransactionResponse{}, nil +} var _ Client = (*mockClient)(nil) diff --git a/internal/rpc/client.go b/internal/rpc/client.go index 63d22a5a..120c0fc5 100644 --- a/internal/rpc/client.go +++ b/internal/rpc/client.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" - "github.com/khaylebfortune/sorotrail/internal/metrics" + "github.com/sorotrail/sorotrail/internal/metrics" ) @@ -32,6 +32,11 @@ type Client interface { // Keys are base64-encoded LedgerKey XDR, returned entries include the // base64-encoded LedgerEntry XDR. GetLedgerEntries(ctx context.Context, req GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) + // SimulateTransaction simulates a transaction (typically a contract + // invocation) against the current ledger state. Used by the contract + // metadata worker to call SEP-41 token interface functions (name, + // symbol, decimals) without submitting a real transaction. + SimulateTransaction(ctx context.Context, req SimulateTransactionRequest) (SimulateTransactionResponse, error) } // RequestObserver is called after each RPC call completes so callers can @@ -143,14 +148,14 @@ func (c *HTTPClient) GetEvents(ctx context.Context, req GetEventsRequest) (GetEv var resp GetEventsResponse start := time.Now() err := c.call(ctx, "getEvents", req, &resp) - metrics.RPCCallDuration.WithLabelValues("getEvents").Observe(time.Since(start).Seconds()) + metrics.RPCCallLatency.Observe(time.Since(start).Seconds()) if err != nil && isXDRFormatRejected(err) { // Older server: remember and retry once without the param. c.xdrJSONUnsupported.Store(true) req.XDRFormat = "" start = time.Now() err = c.call(ctx, "getEvents", req, &resp) - metrics.RPCCallDuration.WithLabelValues("getEvents").Observe(time.Since(start).Seconds()) + metrics.RPCCallLatency.Observe(time.Since(start).Seconds()) } return resp, err } @@ -160,7 +165,7 @@ func (c *HTTPClient) GetLatestLedger(ctx context.Context) (LatestLedger, error) var resp LatestLedger start := time.Now() err := c.call(ctx, "getLatestLedger", nil, &resp) - metrics.RPCCallDuration.WithLabelValues("getLatestLedger").Observe(time.Since(start).Seconds()) + metrics.RPCCallLatency.Observe(time.Since(start).Seconds()) return resp, err } @@ -169,7 +174,7 @@ func (c *HTTPClient) GetHealth(ctx context.Context) (Health, error) { var resp Health start := time.Now() err := c.call(ctx, "getHealth", nil, &resp) - metrics.RPCCallDuration.WithLabelValues("getHealth").Observe(time.Since(start).Seconds()) + metrics.RPCCallLatency.Observe(time.Since(start).Seconds()) return resp, err } @@ -177,7 +182,13 @@ func (c *HTTPClient) GetLedgerEntries(ctx context.Context, req GetLedgerEntriesR var resp GetLedgerEntriesResponse start := time.Now() err := c.call(ctx, "getLedgerEntries", req, &resp) - metrics.RPCCallDuration.WithLabelValues("getLedgerEntries").Observe(time.Since(start).Seconds()) + metrics.RPCCallLatency.Observe(time.Since(start).Seconds()) + return resp, err +} + +func (c *HTTPClient) SimulateTransaction(ctx context.Context, req SimulateTransactionRequest) (SimulateTransactionResponse, error) { + var resp SimulateTransactionResponse + err := c.call(ctx, "simulateTransaction", req, &resp) return resp, err } diff --git a/internal/rpc/client_test.go b/internal/rpc/client_test.go index f3ef7016..712454dc 100644 --- a/internal/rpc/client_test.go +++ b/internal/rpc/client_test.go @@ -118,6 +118,28 @@ func TestRPCErrorSurfaced(t *testing.T) { assert.True(t, IsLedgerOutOfRange(err)) } +func TestSimulateTransaction(t *testing.T) { + srv := jsonRPCServer(t, func(method string, params json.RawMessage) (any, *Error) { + require.Equal(t, "simulateTransaction", method) + return SimulateTransactionResponse{ + TransactionData: "AAAAAg==", + Cost: SimulationCost{ + CPUInstructions: 1000, + MemoryBytes: 4096, + }, + }, nil + }) + defer srv.Close() + + c := NewHTTPClient(srv.URL, WithMinRequestInterval(0)) + resp, err := c.SimulateTransaction(context.Background(), SimulateTransactionRequest{ + Transaction: "AAAAAg...", + }) + require.NoError(t, err) + assert.Equal(t, "AAAAAg==", resp.TransactionData) + assert.Equal(t, uint64(1000), resp.Cost.CPUInstructions) +} + func TestGetHealthAndLatestLedger(t *testing.T) { srv := jsonRPCServer(t, func(method string, _ json.RawMessage) (any, *Error) { switch method { diff --git a/internal/rpc/failover_test.go b/internal/rpc/failover_test.go index 2ed90bd9..f53d0b0e 100644 --- a/internal/rpc/failover_test.go +++ b/internal/rpc/failover_test.go @@ -20,8 +20,8 @@ func testLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } -// mockClient is a controllable rpc.Client for failover tests. -type mockClient struct { +// failoverMockClient is a controllable rpc.Client for failover tests. +type failoverMockClient struct { mu sync.Mutex url string getEventsResp []GetEventsResponse @@ -31,7 +31,7 @@ type mockClient struct { callCount atomic.Int32 } -func (m *mockClient) GetEvents(_ context.Context, _ GetEventsRequest) (GetEventsResponse, error) { +func (m *failoverMockClient) GetEvents(_ context.Context, _ GetEventsRequest) (GetEventsResponse, error) { m.mu.Lock() defer m.mu.Unlock() idx := int(m.callCount.Add(1) - 1) @@ -44,11 +44,11 @@ func (m *mockClient) GetEvents(_ context.Context, _ GetEventsRequest) (GetEvents return GetEventsResponse{LatestLedger: 100}, nil } -func (m *mockClient) GetLatestLedger(_ context.Context) (LatestLedger, error) { +func (m *failoverMockClient) GetLatestLedger(_ context.Context) (LatestLedger, error) { return LatestLedger{Sequence: 100}, nil } -func (m *mockClient) GetHealth(_ context.Context) (Health, error) { +func (m *failoverMockClient) GetHealth(_ context.Context) (Health, error) { m.mu.Lock() defer m.mu.Unlock() if len(m.getHealthErr) > 0 { @@ -64,17 +64,17 @@ func (m *mockClient) GetHealth(_ context.Context) (Health, error) { return Health{Status: "healthy", LatestLedger: 100, OldestLedger: 10}, nil } -func (m *mockClient) GetLedgerEntries(_ context.Context, _ GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) { +func (m *failoverMockClient) GetLedgerEntries(_ context.Context, _ GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) { return GetLedgerEntriesResponse{}, nil } -func (m *mockClient) SimulateTransaction(_ context.Context, _ SimulateTransactionRequest) (SimulateTransactionResponse, error) { +func (m *failoverMockClient) SimulateTransaction(_ context.Context, _ SimulateTransactionRequest) (SimulateTransactionResponse, error) { return SimulateTransactionResponse{}, nil } // resetCallCount resets the call counter. Must only be called between test // phases (not concurrently with GetEvents/GetHealth). -func (m *mockClient) resetCallCount() { +func (m *failoverMockClient) resetCallCount() { m.mu.Lock() defer m.mu.Unlock() m.callCount.Store(0) @@ -82,8 +82,8 @@ func (m *mockClient) resetCallCount() { // newFailoverTestClient creates a FailoverClient backed by mockClients. // Returns the failover client and the individual mocks for scripting. -func newFailoverTestClient(urls []string, opts ...FailoverOption) (*FailoverClient, []*mockClient) { - mocks := make([]*mockClient, len(urls)) +func newFailoverTestClient(urls []string, opts ...FailoverOption) (*FailoverClient, []*failoverMockClient) { + mocks := make([]*failoverMockClient, len(urls)) newClient := func(url string, _ float64) Client { for i, u := range urls { if u == url { @@ -93,7 +93,7 @@ func newFailoverTestClient(urls []string, opts ...FailoverOption) (*FailoverClie panic("unexpected url: " + url) } for i, u := range urls { - mocks[i] = &mockClient{url: u} + mocks[i] = &failoverMockClient{url: u} } fc := NewFailoverClient(urls, 10.0, newClient, opts...) return fc, mocks diff --git a/internal/rpc/metrics.go b/internal/rpc/metrics.go index 0ea0af10..cf542011 100644 --- a/internal/rpc/metrics.go +++ b/internal/rpc/metrics.go @@ -84,3 +84,10 @@ func (c *CountingClient) GetLedgerEntries(ctx context.Context, req GetLedgerEntr } return resp, err } + +// SimulateTransaction passes through to the wrapped client. Simulation +// calls are not counted toward ingestion error totals (spec lookups and +// contract-metadata enrichment use this method). +func (c *CountingClient) SimulateTransaction(ctx context.Context, req SimulateTransactionRequest) (SimulateTransactionResponse, error) { + return c.inner.SimulateTransaction(ctx, req) +} diff --git a/internal/rpc/metrics_test.go b/internal/rpc/metrics_test.go index 7f205dcb..8c257279 100644 --- a/internal/rpc/metrics_test.go +++ b/internal/rpc/metrics_test.go @@ -30,6 +30,9 @@ func (s *stubClient) GetHealth(_ context.Context) (Health, error) { func (s *stubClient) GetLedgerEntries(_ context.Context, _ GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) { return GetLedgerEntriesResponse{}, s.errGetLedgerEntries } +func (s *stubClient) SimulateTransaction(_ context.Context, _ SimulateTransactionRequest) (SimulateTransactionResponse, error) { + return SimulateTransactionResponse{}, nil +} func TestCountingClient_CountsErrorsByMethod(t *testing.T) { sentinel := errors.New("rpc failure") diff --git a/internal/rpc/retry.go b/internal/rpc/retry.go index 28b3ef8a..5976ae6c 100644 --- a/internal/rpc/retry.go +++ b/internal/rpc/retry.go @@ -221,6 +221,16 @@ func (c *RetryClient) GetHealth(ctx context.Context) (Health, error) { return resp, err } +func (c *RetryClient) SimulateTransaction(ctx context.Context, req SimulateTransactionRequest) (SimulateTransactionResponse, error) { + var resp SimulateTransactionResponse + err := c.doWithRetry(ctx, func(ctx context.Context) error { + var innerErr error + resp, innerErr = c.inner.SimulateTransaction(ctx, req) + return innerErr + }) + return resp, err +} + func (c *RetryClient) GetLedgerEntries(ctx context.Context, req GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) { var resp GetLedgerEntriesResponse err := c.doWithRetry(ctx, func(ctx context.Context) error { diff --git a/internal/rpc/retry_test.go b/internal/rpc/retry_test.go index 9b841679..e1fa9d3c 100644 --- a/internal/rpc/retry_test.go +++ b/internal/rpc/retry_test.go @@ -10,45 +10,49 @@ import ( "github.com/stretchr/testify/require" ) -// mockClient implements Client for testing. -type mockClient struct { +// retryMockClient implements Client for testing. +type retryMockClient struct { getEvents func(ctx context.Context, req GetEventsRequest) (GetEventsResponse, error) getLatestLedger func(ctx context.Context) (LatestLedger, error) getHealth func(ctx context.Context) (Health, error) getLedgerEntries func(ctx context.Context, req GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) } -func (m *mockClient) GetEvents(ctx context.Context, req GetEventsRequest) (GetEventsResponse, error) { +func (m *retryMockClient) GetEvents(ctx context.Context, req GetEventsRequest) (GetEventsResponse, error) { if m.getEvents != nil { return m.getEvents(ctx, req) } return GetEventsResponse{}, nil } -func (m *mockClient) GetLatestLedger(ctx context.Context) (LatestLedger, error) { +func (m *retryMockClient) GetLatestLedger(ctx context.Context) (LatestLedger, error) { if m.getLatestLedger != nil { return m.getLatestLedger(ctx) } return LatestLedger{}, nil } -func (m *mockClient) GetHealth(ctx context.Context) (Health, error) { +func (m *retryMockClient) GetHealth(ctx context.Context) (Health, error) { if m.getHealth != nil { return m.getHealth(ctx) } return Health{}, nil } -func (m *mockClient) GetLedgerEntries(ctx context.Context, req GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) { +func (m *retryMockClient) GetLedgerEntries(ctx context.Context, req GetLedgerEntriesRequest) (GetLedgerEntriesResponse, error) { if m.getLedgerEntries != nil { return m.getLedgerEntries(ctx, req) } return GetLedgerEntriesResponse{}, nil } +func (m *retryMockClient) SimulateTransaction(ctx context.Context, req SimulateTransactionRequest) (SimulateTransactionResponse, error) { + return SimulateTransactionResponse{}, nil +} + func TestRetryClient_SuccessOnFirstAttempt(t *testing.T) { var calls int - inner := &mockClient{ + inner := &retryMockClient{ getHealth: func(ctx context.Context) (Health, error) { calls++ return Health{Status: "healthy", LatestLedger: 100}, nil @@ -63,7 +67,7 @@ func TestRetryClient_SuccessOnFirstAttempt(t *testing.T) { func TestRetryClient_RetriesOnTransientError(t *testing.T) { var calls int - inner := &mockClient{ + inner := &retryMockClient{ getHealth: func(ctx context.Context) (Health, error) { calls++ if calls < 3 { @@ -81,7 +85,7 @@ func TestRetryClient_RetriesOnTransientError(t *testing.T) { func TestRetryClient_ExhaustsRetries(t *testing.T) { var calls int - inner := &mockClient{ + inner := &retryMockClient{ getHealth: func(ctx context.Context) (Health, error) { calls++ return Health{}, &Error{Code: 0, Message: "persistent error"} @@ -96,7 +100,7 @@ func TestRetryClient_ExhaustsRetries(t *testing.T) { func TestRetryClient_NonRetryableError(t *testing.T) { var calls int - inner := &mockClient{ + inner := &retryMockClient{ getHealth: func(ctx context.Context) (Health, error) { calls++ return Health{}, &Error{Code: -32601, Message: "Method not found"} @@ -113,7 +117,7 @@ func TestRetryClient_ContextCancellation(t *testing.T) { cancel() var calls int - inner := &mockClient{ + inner := &retryMockClient{ getHealth: func(ctx context.Context) (Health, error) { calls++ return Health{Status: "healthy"}, nil @@ -129,7 +133,7 @@ func TestRetryClient_ContextCancellation(t *testing.T) { func TestRetryClient_BackoffRespectsMax(t *testing.T) { var calls int - inner := &mockClient{ + inner := &retryMockClient{ getHealth: func(ctx context.Context) (Health, error) { calls++ return Health{}, errors.New("EOF") @@ -167,7 +171,7 @@ func TestIsRetryable_ErrorCodes(t *testing.T) { } func TestNewRetryClient_DefaultConfig(t *testing.T) { - rc := NewRetryClient(&mockClient{}, RetryConfig{}) + rc := NewRetryClient(&retryMockClient{}, RetryConfig{}) assert.Equal(t, 3, rc.config.MaxAttempts) assert.Equal(t, 500*time.Millisecond, rc.config.BaseBackoff) assert.Equal(t, 30*time.Second, rc.config.MaxBackoff) diff --git a/internal/rpc/types.go b/internal/rpc/types.go index 9bd8ca60..e81232b3 100644 --- a/internal/rpc/types.go +++ b/internal/rpc/types.go @@ -109,9 +109,36 @@ type GetLedgerEntriesResponse struct { // LedgerEntryResult is one entry returned by getLedgerEntries. type LedgerEntryResult struct { - Key string `json:"key"` // base64-encoded LedgerKey XDR - XDR string `json:"xdr"` // base64-encoded LedgerEntry XDR - LastModifiedLedgerSeq uint32 `json:"lastModifiedLedgerSeq"` + Key string `json:"key"` // base64-encoded LedgerKey XDR + XDR string `json:"xdr"` // base64-encoded LedgerEntry XDR + LastModifiedLedgerSeq uint32 `json:"lastModifiedLedgerSeq"` // LiveUntilLedgerSeq is set for entries with a time-to-live (e.g. temporary entries). LiveUntilLedgerSeq *uint32 `json:"liveUntilLedgerSeq,omitempty"` } + +// SimulateTransactionRequest is the params for the simulateTransaction method. +// Transaction is a base64-encoded TransactionEnvelope XDR. +type SimulateTransactionRequest struct { + Transaction string `json:"transaction"` +} + +// SimulateTransactionResponse is the result of simulateTransaction. +type SimulateTransactionResponse struct { + // TransactionData is base64-encoded TransactionMeta XDR. + TransactionData string `json:"transactionData"` + // Events are the diagnostic events emitted during simulation. + Events []Event `json:"events,omitempty"` + // Cost describes the resource cost of the simulated transaction. + Cost SimulationCost `json:"cost"` + // Results are the return values of each host function invocation. + // Each result is a base64-encoded ScVal XDR. + Results []json.RawMessage `json:"results,omitempty"` + // Error is present when the simulation failed (e.g. contract trapped). + Error string `json:"error,omitempty"` +} + +// SimulationCost describes the compute resources consumed by a simulation. +type SimulationCost struct { + CPUInstructions uint64 `json:"cpuInsns,string"` + MemoryBytes uint64 `json:"memBytes,string"` +} diff --git a/internal/simtest/chain.go b/internal/simtest/chain.go index 55df3e1d..3ddae98c 100644 --- a/internal/simtest/chain.go +++ b/internal/simtest/chain.go @@ -445,3 +445,9 @@ func BuildEvent(id string, ledger uint32, contractID string) rpc.Event { func (c *VirtualChain) GetLedgerEntries(context.Context, rpc.GetLedgerEntriesRequest) (rpc.GetLedgerEntriesResponse, error) { return rpc.GetLedgerEntriesResponse{}, nil } + +// SimulateTransaction satisfies rpc.Client. The virtual chain does not model +// contract simulation, so it returns an empty response. +func (c *VirtualChain) SimulateTransaction(context.Context, rpc.SimulateTransactionRequest) (rpc.SimulateTransactionResponse, error) { + return rpc.SimulateTransactionResponse{}, nil +} diff --git a/internal/simtest/simtest_test.go b/internal/simtest/simtest_test.go index 14f627e3..f753f628 100644 --- a/internal/simtest/simtest_test.go +++ b/internal/simtest/simtest_test.go @@ -148,11 +148,11 @@ var _ store.Store = (*mockStore)(nil) // Remaining store.Store methods that simtest doesn't exercise. -func (m *mockStore) GetAuditState(context.Context) (store.AuditState, error) { +func (m *mockStore) GetAuditState(context.Context, string) (store.AuditState, error) { return store.AuditState{}, store.ErrNotFound } func (m *mockStore) SaveAuditState(_ context.Context, s store.AuditState) error { return nil } -func (m *mockStore) SaveAuditStateIfGreater(_ context.Context, ledger int64) (store.AuditState, error) { +func (m *mockStore) SaveAuditStateIfGreater(_ context.Context, _ string, ledger int64) (store.AuditState, error) { return store.AuditState{VerifiedThroughLedger: ledger}, nil } func (m *mockStore) RecordAuditFinding(_ context.Context, f store.AuditFinding) (store.AuditFinding, error) { @@ -160,7 +160,7 @@ func (m *mockStore) RecordAuditFinding(_ context.Context, f store.AuditFinding) return f, nil } func (m *mockStore) UpdateAuditFinding(context.Context, store.AuditFinding) error { return nil } -func (m *mockStore) ListOpenFindingsByRange(context.Context, int64, int64) (store.AuditFinding, error) { +func (m *mockStore) ListOpenFindingsByRange(context.Context, string, int64, int64) (store.AuditFinding, error) { return store.AuditFinding{}, store.ErrNotFound } func (m *mockStore) Ping(context.Context) error { return nil } diff --git a/internal/store/clickhouse.go b/internal/store/clickhouse.go index e53c7b9c..6a6e5eec 100644 --- a/internal/store/clickhouse.go +++ b/internal/store/clickhouse.go @@ -25,6 +25,18 @@ type clickHouseConfig struct { ssl bool } +// Contract metadata (token enrichment) is Postgres-only; the ClickHouse +// backend reports "not found"/empty so the enrichment worker stays a no-op. +func (c *ClickHouse) ListContractIDs(context.Context) ([]string, error) { return nil, nil } +func (c *ClickHouse) GetContractMeta(context.Context, string) (ContractMeta, error) { + return ContractMeta{}, ErrNotFound +} +func (c *ClickHouse) UpsertContractMeta(context.Context, ContractMeta) error { return nil } +func (c *ClickHouse) CountContractEvents(context.Context, string) (int64, error) { return 0, nil } +func (c *ClickHouse) ListContractsNeedingRefresh(context.Context, time.Time) ([]string, error) { + return nil, nil +} + func parseClickHouseConfig(raw string) (clickHouseConfig, error) { u, err := url.Parse(raw) if err != nil { diff --git a/internal/store/contract_meta_postgres.go b/internal/store/contract_meta_postgres.go new file mode 100644 index 00000000..04570f38 --- /dev/null +++ b/internal/store/contract_meta_postgres.go @@ -0,0 +1,111 @@ +package store + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// GetContractMeta returns the cached metadata for a contract, or ErrNotFound. +func (p *Postgres) GetContractMeta(ctx context.Context, contractID string) (ContractMeta, error) { + var m ContractMeta + err := p.pool.QueryRow(ctx, ` + SELECT contract_id, name, symbol, decimals, is_token, fetched_at + FROM contract_meta WHERE contract_id = $1`, contractID, + ).Scan(&m.ContractID, &m.Name, &m.Symbol, &m.Decimals, &m.IsToken, &m.FetchedAt) + if errors.Is(err, pgx.ErrNoRows) { + return ContractMeta{}, ErrNotFound + } + if err != nil { + return ContractMeta{}, fmt.Errorf("getting contract meta for %s: %w", contractID, err) + } + return m, nil +} + +// UpsertContractMeta inserts or updates a contract metadata row. +func (p *Postgres) UpsertContractMeta(ctx context.Context, m ContractMeta) error { + _, err := p.pool.Exec(ctx, ` + INSERT INTO contract_meta (contract_id, name, symbol, decimals, is_token, fetched_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (contract_id) DO UPDATE SET + name = EXCLUDED.name, + symbol = EXCLUDED.symbol, + decimals = EXCLUDED.decimals, + is_token = EXCLUDED.is_token, + fetched_at = EXCLUDED.fetched_at`, + m.ContractID, m.Name, m.Symbol, m.Decimals, m.IsToken, m.FetchedAt, + ) + if err != nil { + return fmt.Errorf("upserting contract meta for %s: %w", m.ContractID, err) + } + return nil +} + +// CountContractEvents returns the number of events for a given contract. +func (p *Postgres) CountContractEvents(ctx context.Context, contractID string) (int64, error) { + var count int64 + err := p.pool.QueryRow(ctx, + `SELECT count(*) FROM events WHERE contract_id = $1`, contractID, + ).Scan(&count) + if err != nil { + return 0, fmt.Errorf("counting events for %s: %w", contractID, err) + } + return count, nil +} + +// ListContractIDs returns all distinct contract IDs that have emitted events. +func (p *Postgres) ListContractIDs(ctx context.Context) ([]string, error) { + rows, err := p.pool.Query(ctx, `SELECT DISTINCT contract_id FROM events ORDER BY contract_id`) + if err != nil { + return nil, fmt.Errorf("listing contract IDs: %w", err) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scanning contract ID: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// ListContractsNeedingRefresh returns contract IDs whose metadata is stale +// (fetched before olderThan) or has never been fetched. This includes: +// - Contracts in events that have NO contract_meta row at all (never probed) +// - Contracts with fetched_at < olderThan (expired TTL) +// - Contracts with is_token=true but null name/symbol/decimals (failed fetch, retry) +// +// Non-token contracts (is_token=false) are never re-fetched regardless of +// their fetched_at, so negative caching is permanent. +func (p *Postgres) ListContractsNeedingRefresh(ctx context.Context, olderThan time.Time) ([]string, error) { + rows, err := p.pool.Query(ctx, ` + SELECT DISTINCT e.contract_id + FROM events e + WHERE NOT EXISTS ( + SELECT 1 FROM contract_meta cm + WHERE cm.contract_id = e.contract_id + AND (cm.is_token = false OR (cm.is_token = true AND cm.fetched_at > $1 AND cm.name IS NOT NULL)) + ) + ORDER BY e.contract_id`, olderThan, + ) + if err != nil { + return nil, fmt.Errorf("listing contracts needing refresh: %w", err) + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scanning contract ID: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} diff --git a/internal/store/events_integration_test.go b/internal/store/events_integration_test.go index f7439e24..a7cc1ce5 100644 --- a/internal/store/events_integration_test.go +++ b/internal/store/events_integration_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/khaylebfortune/sorotrail/internal/testdb" + "github.com/sorotrail/sorotrail/internal/testdb" ) // TestUpsertEvents_SameTOIDTwiceYieldsOneRow is the headline diff --git a/internal/store/guarded_store.go b/internal/store/guarded_store.go index 6875e1ac..c1cbac00 100644 --- a/internal/store/guarded_store.go +++ b/internal/store/guarded_store.go @@ -143,7 +143,7 @@ func (s *guardedStore) GetIngestionState(ctx context.Context) (IngestionState, e ctx, cancel := s.wrapContext(ctx, "store.GetIngestionState") defer cancel() start := time.Now() - state, err := s.Store.GetIngestionState(ctx, network) + state, err := s.Store.GetIngestionState(ctx) s.logSlowQuery("store.GetIngestionState", start, err) return state, err } @@ -388,51 +388,6 @@ func (s *guardedStore) Stats(ctx context.Context, sc Scope) (Stats, error) { return stats, err } -func (s *guardedStore) UpsertTokenBalances(ctx context.Context, network string, state TokenBalanceState, updates []TokenBalanceUpdate) error { - ctx, cancel := s.wrapContext(ctx, "store.UpsertTokenBalances") - defer cancel() - start := time.Now() - err := s.Store.UpsertTokenBalances(ctx, network, state, updates) - s.logSlowQuery("store.UpsertTokenBalances", start, err) - return err -} - -func (s *guardedStore) GetTokenBalances(ctx context.Context, contractID, network, minBalance string, cursor string, limit int) ([]TokenBalance, string, error) { - ctx, cancel := s.wrapContext(ctx, "store.GetTokenBalances") - defer cancel() - start := time.Now() - balances, next, err := s.Store.GetTokenBalances(ctx, contractID, network, minBalance, cursor, limit) - s.logSlowQuery("store.GetTokenBalances", start, err) - return balances, next, err -} - -func (s *guardedStore) GetTokenBalanceState(ctx context.Context, network, contractID string) (TokenBalanceState, error) { - ctx, cancel := s.wrapContext(ctx, "store.GetTokenBalanceState") - defer cancel() - start := time.Now() - state, err := s.Store.GetTokenBalanceState(ctx, network, contractID) - s.logSlowQuery("store.GetTokenBalanceState", start, err) - return state, err -} - -func (s *guardedStore) UpsertTokenBalanceState(ctx context.Context, state TokenBalanceState) error { - ctx, cancel := s.wrapContext(ctx, "store.UpsertTokenBalanceState") - defer cancel() - start := time.Now() - err := s.Store.UpsertTokenBalanceState(ctx, state) - s.logSlowQuery("store.UpsertTokenBalanceState", start, err) - return err -} - -func (s *guardedStore) GetEarliestLedger(ctx context.Context, network, contractID string) (int64, error) { - ctx, cancel := s.wrapContext(ctx, "store.GetEarliestLedger") - defer cancel() - start := time.Now() - earliest, err := s.Store.GetEarliestLedger(ctx, network, contractID) - s.logSlowQuery("store.GetEarliestLedger", start, err) - return earliest, err -} - func (s *guardedStore) Ping(ctx context.Context) error { ctx, cancel := s.wrapContext(ctx, "store.Ping") defer cancel() diff --git a/internal/store/migrations/0005_contract_meta.down.sql b/internal/store/migrations/0005_contract_meta.down.sql new file mode 100644 index 00000000..87cdc785 --- /dev/null +++ b/internal/store/migrations/0005_contract_meta.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS contract_meta; diff --git a/internal/store/migrations/0005_contract_meta.up.sql b/internal/store/migrations/0005_contract_meta.up.sql new file mode 100644 index 00000000..6993316d --- /dev/null +++ b/internal/store/migrations/0005_contract_meta.up.sql @@ -0,0 +1,16 @@ +-- Contract metadata cache: token name, symbol, decimals fetched via RPC +-- simulation for contracts that emit SEP-41 (token) events. +-- Null fields + is_token=false = negative cache (non-token contract, don't re-fetch). +-- Null fields + is_token=true = token contract whose metadata hasn't been resolved yet. +CREATE TABLE contract_meta ( + contract_id text PRIMARY KEY, + name text, + symbol text, + decimals int, + is_token boolean NOT NULL DEFAULT false, + fetched_at timestamptz NOT NULL DEFAULT now() +); + +-- Supports TTL-based refresh: the worker queries rows whose fetched_at is +-- older than the configured TTL, ordered so the stalest are refreshed first. +CREATE INDEX idx_contract_meta_fetched_at ON contract_meta (fetched_at); diff --git a/internal/store/order_by_test.go b/internal/store/order_by_test.go index 5c9edc41..132212c0 100644 --- a/internal/store/order_by_test.go +++ b/internal/store/order_by_test.go @@ -1,3 +1,6 @@ +//go:build integration + + package store import ( diff --git a/internal/store/postgres.go b/internal/store/postgres.go index 075fc08d..a8ef4264 100644 --- a/internal/store/postgres.go +++ b/internal/store/postgres.go @@ -8,7 +8,6 @@ import ( "fmt" "hash/fnv" "log/slog" - "math/big" "strings" "sync" "sync/atomic" @@ -18,7 +17,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/prometheus/client_golang/prometheus" - "github.com/khaylebfortune/sorotrail/internal/metrics" + "github.com/sorotrail/sorotrail/internal/metrics" ) // ErrNotFound is returned when a lookup matches no rows. @@ -28,12 +27,7 @@ var ErrNotFound = errors.New("not found") // requested ordering. It is caller error: the API maps it to 400, not 500. var ErrInvalidCursor = errors.New("invalid cursor") -// DefaultQueryLimit applies when EventFilter.Limit is unset; MaxQueryLimit -// caps requested page sizes as a server-side safety net (the API layer -// enforces its own configurable limit via API_MAX_LIMIT). const ( - DefaultQueryLimit = 50 - MaxQueryLimit = 500 DefaultEventPartitionSpan = 120960 poolHealthCheckInterval = 5 * time.Second @@ -43,6 +37,7 @@ const ( ) // Postgres implements Store on a pgx connection pool. +// type Postgres struct { pool *pgxpool.Pool partitionSpan int64 @@ -203,46 +198,25 @@ func (p *Postgres) attemptReconnect(ctx context.Context, logger *slog.Logger) er return nil } -func (p *Postgres) UpsertEvents(ctx context.Context, events []Event) ([]Event, error) { +func (p *Postgres) UpsertEvents(ctx context.Context, events []Event) (int64, error) { if len(events) == 0 { - return nil, nil + return 0, nil } return p.upsertEvents(ctx, events, false) } // insertEventsBatch builds the single batch used by upsertEvents (idempotent // ingest, DO NOTHING) and ReplaceEventsInRange (auditor repair, DO UPDATE). -// Both paths write all 15 columns of the events table so topics_xdr / -// value_xdr are never silently dropped on the way in; a repair that -// arrives without XDR preserves what was already stored via the coalesce() -// clauses in the UPDATE branch (`sorotrail replay` relies on that). +// Both paths write the full events row so raw XDR is never silently dropped +// on the way in; a repair that arrives without XDR preserves what was already +// stored via the coalesce() clauses in the UPDATE branch (`sorotrail replay` +// relies on that). func insertEventsBatch(events []Event, onUpdate bool) *pgx.Batch { - conflict := `ON CONFLICT (network, ledger, id) DO NOTHING` - batch := &pgx.Batch{} - conflict := `ON CONFLICT (id) DO NOTHING` - if onUpdate { - conflict = `ON CONFLICT (id) DO UPDATE SET - contract_id = COALESCE(EXCLUDED.contract_id, events.contract_id), - ledger = COALESCE(EXCLUDED.ledger, events.ledger), - type = COALESCE(EXCLUDED.type, events.type), - tx_hash = COALESCE(EXCLUDED.tx_hash, events.tx_hash), - tx_index = COALESCE(EXCLUDED.tx_index, events.tx_index), - op_index = COALESCE(EXCLUDED.op_index, events.op_index), - in_successful_call = COALESCE(EXCLUDED.in_successful_call, events.in_successful_call), - topics = COALESCE(EXCLUDED.topics, events.topics), - value = COALESCE(EXCLUDED.value, events.value), - created_at = COALESCE(EXCLUDED.created_at, events.created_at), - topics_xdr = COALESCE(EXCLUDED.topics_xdr, events.topics_xdr), - value_xdr = COALESCE(EXCLUDED.value_xdr, events.value_xdr), - raw_topic_xdr = COALESCE(EXCLUDED.raw_topic_xdr, events.raw_topic_xdr), - raw_value_xdr = COALESCE(EXCLUDED.raw_value_xdr, events.raw_value_xdr)` - conflict := "ON CONFLICT (id) DO NOTHING" - if onUpdate { - conflict = `ON CONFLICT (id) DO UPDATE SET conflict := `ON CONFLICT (ledger, id) DO NOTHING` if onUpdate { - conflict = `ON CONFLICT (network, ledger, id) DO UPDATE SET + conflict = `ON CONFLICT (ledger, id) DO UPDATE SET contract_id = EXCLUDED.contract_id, + ledger = EXCLUDED.ledger, type = EXCLUDED.type, tx_hash = EXCLUDED.tx_hash, tx_index = EXCLUDED.tx_index, @@ -250,50 +224,26 @@ func insertEventsBatch(events []Event, onUpdate bool) *pgx.Batch { in_successful_call = EXCLUDED.in_successful_call, topics = EXCLUDED.topics, value = EXCLUDED.value, - decoded_payload = EXCLUDED.decoded_payload, - decoded_by = EXCLUDED.decoded_by, created_at = EXCLUDED.created_at, raw_topic_xdr = COALESCE(EXCLUDED.raw_topic_xdr, events.raw_topic_xdr), raw_value_xdr = COALESCE(EXCLUDED.raw_value_xdr, events.raw_value_xdr)` - topics_xdr = coalesce(EXCLUDED.topics_xdr, events.topics_xdr), - value_xdr = coalesce(EXCLUDED.value_xdr, events.value_xdr)` - raw_topic_xdr = coalesce(EXCLUDED.raw_topic_xdr, events.raw_topic_xdr), - raw_value_xdr = coalesce(EXCLUDED.raw_value_xdr, events.raw_value_xdr)` } - batch := &pgx.Batch{} - sql := ` + stmt := ` INSERT INTO events - (network, id, contract_id, ledger, type, tx_hash, tx_index, op_index, + (id, contract_id, ledger, type, tx_hash, tx_index, op_index, in_successful_call, topics, value, created_at, raw_topic_xdr, raw_value_xdr) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ` + conflict + batch := &pgx.Batch{} for _, e := range events { - batch.Queue(sql, - e.Network, e.ID, e.ContractID, e.Ledger, e.Type, e.TxHash, e.TxIndex, - batch.Queue(` - INSERT INTO events - (id, contract_id, ledger, type, tx_hash, tx_index, op_index, - in_successful_call, topics, value, created_at, raw_topic_xdr, raw_value_xdr) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) `+conflict, - in_successful_call, topics, value, created_at, - raw_topic_xdr, raw_value_xdr) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) - `+conflict, - e.ID, e.ContractID, e.Ledger, e.Type, e.TxHash, e.TxIndex, - e.OpIndex, e.InSuccessfulCall, e.Topics, e.Value, e.CreatedAt, - nullableTextArray(e.RawTopicXDR), nullableText(e.RawValueXDR), // 13 placeholders → 13 args. nullable helpers turn empty raw XDR // into SQL NULL so the column has one representation of "absent" // rather than two. batch.Queue(stmt, e.ID, e.ContractID, e.Ledger, e.Type, e.TxHash, e.TxIndex, - e.OpIndex, e.InSuccessfulCall, e.Topics, e.Value, - nil, /* decoded_payload */ - nil, /* decoded_by */ - e.CreatedAt, - nullableStringSlice(e.RawTopicXDR), - nullableText(e.RawValueXDR), + e.OpIndex, e.InSuccessfulCall, e.Topics, e.Value, e.CreatedAt, + nullableStringSlice(e.RawTopicXDR), nullableText(e.RawValueXDR), ) } return batch @@ -321,7 +271,7 @@ func (p *Postgres) ensureEventPartitions(ctx context.Context, events []Event) er func (p *Postgres) upsertEvents(ctx context.Context, events []Event, onUpdate bool) (int64, error) { if len(events) == 0 { - return nil, nil + return 0, nil } if err := p.ensureEventPartitions(ctx, events); err != nil { return 0, err @@ -329,14 +279,14 @@ func (p *Postgres) upsertEvents(ctx context.Context, events []Event, onUpdate bo results := p.pool.SendBatch(ctx, insertEventsBatch(events, onUpdate)) defer results.Close() - var inserted []Event - for i := range events { + var inserted int64 + for range events { tag, err := results.Exec() if err != nil { - return nil, fmt.Errorf("upserting events: %w", err) + return 0, fmt.Errorf("upserting events: %w", err) } if tag.RowsAffected() > 0 { - inserted = append(inserted, events[i]) + inserted++ } } return inserted, nil @@ -390,7 +340,6 @@ func (p *Postgres) ReplaceEventsInRange(ctx context.Context, events []Event, fro if err := tx.Commit(ctx); err != nil { return fmt.Errorf("committing repair tx: %w", err) } - metrics.DBWriteDuration.Observe(time.Since(start).Seconds()) return nil } @@ -826,17 +775,6 @@ func buildEventWhereClause(f EventFilter) ([]string, []any) { where = append(where, fmt.Sprintf("topics->%d = %s::jsonb", i, arg(topic))) } - if len(f.TopicContains) > 0 { - // Direct containment — caller controls the shape (object wrapped in - // array for element match, multi-element arrays for subset match). - where = append(where, "topics @> "+arg(string(f.TopicContains))+"::jsonb") - } - if f.TopicCount != nil { - where = append(where, "jsonb_array_length(topics) = "+arg(*f.TopicCount)) - } - if f.TxHash != "" { - where = append(where, "tx_hash = "+arg(f.TxHash)) - } if f.HasValue != nil { if *f.HasValue { where = append(where, "value IS NOT NULL") @@ -1079,7 +1017,7 @@ func (p *Postgres) GetIngestionState(ctx context.Context) (IngestionState, error return IngestionState{}, ErrNotFound } if err != nil { - return IngestionState{}, fmt.Errorf("loading ingestion state for network %q: %w", network, err) + return IngestionState{}, fmt.Errorf("loading ingestion state: %w", err) } return s, nil } @@ -1229,6 +1167,79 @@ func (p *Postgres) AddWatchedContract(ctx context.Context, contractID string) er return nil } +// GetContractCursor returns the stored resume position for one watched +// contract, or ErrNotFound when the contract has no cursor row yet. +func (p *Postgres) GetContractCursor(ctx context.Context, contractID string) (ContractCursor, error) { + var ( + c ContractCursor + ts time.Time + ) + err := p.pool.QueryRow(ctx, + `SELECT contract_id, last_ingested_ledger, last_cursor, updated_at + FROM contract_cursors WHERE contract_id = $1`, contractID, + ).Scan(&c.ContractID, &c.LastIngestedLedger, &c.LastCursor, &ts) + if errors.Is(err, pgx.ErrNoRows) { + return ContractCursor{}, ErrNotFound + } + if err != nil { + return ContractCursor{}, fmt.Errorf("loading cursor for contract %s: %w", contractID, err) + } + c.UpdatedAt = ts + return c, nil +} + +// SaveContractCursor upserts one contract's resume position. +func (p *Postgres) SaveContractCursor(ctx context.Context, c ContractCursor) error { + _, err := p.pool.Exec(ctx, ` + INSERT INTO contract_cursors (contract_id, last_ingested_ledger, last_cursor, updated_at) + VALUES ($1, $2, $3, now()) + ON CONFLICT (contract_id) DO UPDATE SET + last_ingested_ledger = EXCLUDED.last_ingested_ledger, + last_cursor = EXCLUDED.last_cursor, + updated_at = now()`, c.ContractID, c.LastIngestedLedger, c.LastCursor) + if err != nil { + return fmt.Errorf("saving cursor for contract %s: %w", c.ContractID, err) + } + return nil +} + +// DeleteContractCursor removes a contract's cursor row (used when a +// contract leaves the watch list). +func (p *Postgres) DeleteContractCursor(ctx context.Context, contractID string) error { + tag, err := p.pool.Exec(ctx, + `DELETE FROM contract_cursors WHERE contract_id = $1`, contractID) + if err != nil { + return fmt.Errorf("deleting cursor for contract %s: %w", contractID, err) + } + if tag.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +// ListContractCursors returns every tracked per-contract resume position. +func (p *Postgres) ListContractCursors(ctx context.Context) ([]ContractCursor, error) { + rows, err := p.pool.Query(ctx, + `SELECT contract_id, last_ingested_ledger, last_cursor, updated_at FROM contract_cursors`) + if err != nil { + return nil, fmt.Errorf("listing contract cursors: %w", err) + } + defer rows.Close() + var out []ContractCursor + for rows.Next() { + var ( + c ContractCursor + ts time.Time + ) + if err := rows.Scan(&c.ContractID, &c.LastIngestedLedger, &c.LastCursor, &ts); err != nil { + return nil, fmt.Errorf("scanning contract cursor: %w", err) + } + c.UpdatedAt = ts + out = append(out, c) + } + return out, rows.Err() +} + // Stats aggregates within sc. The event-derived counters (total, oldest // ledger, distinct contracts, watched contracts) are restricted to the // caller's contracts; the ingestion and audit frontiers are not, because diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index a55c5b86..128c08fb 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -916,7 +916,7 @@ func (s *SQLite) SaveIngestionState(ctx context.Context, st IngestionState) erro return nil } -func (s *SQLite) GetAuditState(ctx context.Context) (AuditState, error) { +func (s *SQLite) GetAuditState(ctx context.Context, _ string) (AuditState, error) { var ( st AuditState ts string @@ -950,7 +950,7 @@ func (s *SQLite) SaveAuditState(ctx context.Context, st AuditState) error { return nil } -func (s *SQLite) SaveAuditStateIfGreater(ctx context.Context, ledger int64) (AuditState, error) { +func (s *SQLite) SaveAuditStateIfGreater(ctx context.Context, _ string, ledger int64) (AuditState, error) { now := formatTime(time.Now().UTC()) // First INSERT will succeed when the table is empty. @@ -973,10 +973,10 @@ func (s *SQLite) SaveAuditStateIfGreater(ctx context.Context, ledger int64) (Aud } n, _ := res.RowsAffected() if n > 0 { - return s.GetAuditState(ctx) + return s.GetAuditState(ctx, "") } // Candidate was not greater — return the current stored state. - return s.GetAuditState(ctx) + return s.GetAuditState(ctx, "") } func (s *SQLite) ListWatchedContracts(ctx context.Context) ([]WatchedContract, error) { @@ -1066,7 +1066,7 @@ func (s *SQLite) UpdateAuditFinding(ctx context.Context, f AuditFinding) error { return nil } -func (s *SQLite) ListOpenFindingsByRange(ctx context.Context, fromLedger, toLedger int64) (AuditFinding, error) { +func (s *SQLite) ListOpenFindingsByRange(ctx context.Context, _ string, fromLedger, toLedger int64) (AuditFinding, error) { row := s.db.QueryRowContext(ctx, ` SELECT id, from_ledger, to_ledger, expected_count, actual_count, missing_ids, status, attempts, last_attempted_at, last_error, created_at @@ -1437,92 +1437,31 @@ func nullableXDRTopics(s []string) any { return string(b) } -// AggregateEvents is not implemented for the SQLite backend: the analytics -// endpoints are Postgres-only. Returning an error beats returning empty -// buckets, which a caller would read as "no events in range". -func (s *SQLite) AggregateEvents(context.Context, EventFilter, string) ([]AggregateBucket, error) { - return nil, fmt.Errorf("AggregateEvents: not supported by the sqlite backend") +// Contract metadata (token enrichment) is Postgres-only; the SQLite backend +// reports "not found"/empty so the enrichment worker stays a no-op. +func (s *SQLite) ListContractIDs(context.Context) ([]string, error) { return nil, nil } +func (s *SQLite) GetContractMeta(context.Context, string) (ContractMeta, error) { + return ContractMeta{}, ErrNotFound } - -// CountAddressEvents is not implemented for the SQLite backend: the address -// activity index is Postgres-only. -func (s *SQLite) CountAddressEvents(context.Context, string) (int64, error) { - return 0, fmt.Errorf("CountAddressEvents: not supported by the sqlite backend") +func (s *SQLite) UpsertContractMeta(context.Context, ContractMeta) error { return nil } +func (s *SQLite) CountContractEvents(context.Context, string) (int64, error) { return 0, nil } +func (s *SQLite) ListContractsNeedingRefresh(context.Context, time.Time) ([]string, error) { + return nil, nil } -// CountContracts is not implemented for the SQLite backend: the contract +// ListContracts is not implemented for the SQLite backend: the contract // inventory endpoint is Postgres-only. -func (s *SQLite) CountContracts(context.Context, ContractsFilter) (int64, error) { - return 0, fmt.Errorf("CountContracts: not supported by the sqlite backend") -} - -// CountEvents is not implemented for the SQLite backend: total-count -// pagination metadata is Postgres-only. -func (s *SQLite) CountEvents(context.Context, EventFilter) (int64, error) { - return 0, fmt.Errorf("CountEvents: not supported by the sqlite backend") -} - -// DeadLetterEvent is not implemented for the SQLite backend. -func (s *SQLite) DeadLetterEvent(context.Context, DeadLetterInput) (DeadLetter, error) { - return DeadLetter{}, fmt.Errorf("DeadLetterEvent: not supported by the sqlite backend") -} - -// DeleteDeadLetter is not implemented for the SQLite backend. -func (s *SQLite) DeleteDeadLetter(context.Context, int64) error { - return fmt.Errorf("DeleteDeadLetter: not supported by the sqlite backend") -} - -// DeleteEventsBefore is not implemented for the SQLite backend. -func (s *SQLite) DeleteEventsBefore(context.Context, int64, time.Time, int) (int64, error) { - return 0, fmt.Errorf("DeleteEventsBefore: not supported by the sqlite backend") -} - -// DeleteEventsBeforeLedger is not implemented for the SQLite backend. -func (s *SQLite) DeleteEventsBeforeLedger(context.Context, int64) (int64, error) { - return 0, fmt.Errorf("DeleteEventsBeforeLedger: not supported by the sqlite backend") -} - -// GetAddressSummary is not implemented for the SQLite backend. -func (s *SQLite) GetAddressSummary(context.Context, string) (AddressSummary, error) { - return AddressSummary{}, fmt.Errorf("GetAddressSummary: not supported by the sqlite backend") -} - -// GetDeadLetter is not implemented for the SQLite backend. -func (s *SQLite) GetDeadLetter(context.Context, int64) (DeadLetter, error) { - return DeadLetter{}, fmt.Errorf("GetDeadLetter: not supported by the sqlite backend") -} - -// GetEventsByTxHash is not implemented for the SQLite backend. -func (s *SQLite) GetEventsByTxHash(context.Context, string, string) ([]Event, error) { - return nil, fmt.Errorf("GetEventsByTxHash: not supported by the sqlite backend") -} - -// ListContracts is not implemented for the SQLite backend. func (s *SQLite) ListContracts(context.Context, ContractsFilter) ([]ContractSummary, string, error) { return nil, "", fmt.Errorf("ListContracts: not supported by the sqlite backend") } -// ListDeadLetters is not implemented for the SQLite backend. -func (s *SQLite) ListDeadLetters(context.Context, string, int, string) ([]DeadLetter, string, error) { - return nil, "", fmt.Errorf("ListDeadLetters: not supported by the sqlite backend") +// Per-contract cursors are not implemented for the SQLite backend: watched +// ingestion always uses the single global ingestion_state row. +func (s *SQLite) GetContractCursor(context.Context, string) (ContractCursor, error) { + return ContractCursor{}, ErrNotFound } - -// MigrationVersion is not implemented for the SQLite backend. -func (s *SQLite) MigrationVersion(context.Context) (int, bool, error) { - return 0, false, fmt.Errorf("MigrationVersion: not supported by the sqlite backend") -} - -// QueryAddressEvents is not implemented for the SQLite backend. -func (s *SQLite) QueryAddressEvents(context.Context, string, EventFilter) ([]Event, string, error) { - return nil, "", fmt.Errorf("QueryAddressEvents: not supported by the sqlite backend") -} - -// RemoveWatchedContract is not implemented for the SQLite backend. -func (s *SQLite) RemoveWatchedContract(context.Context, string) error { - return fmt.Errorf("RemoveWatchedContract: not supported by the sqlite backend") -} - -// UpsertAddressRefs is not implemented for the SQLite backend. -func (s *SQLite) UpsertAddressRefs(context.Context, []AddressRef) error { - return fmt.Errorf("UpsertAddressRefs: not supported by the sqlite backend") +func (s *SQLite) SaveContractCursor(context.Context, ContractCursor) error { return nil } +func (s *SQLite) DeleteContractCursor(context.Context, string) error { return nil } +func (s *SQLite) ListContractCursors(context.Context) ([]ContractCursor, error) { + return nil, nil } diff --git a/internal/store/store.go b/internal/store/store.go index 8d394d2c..62748723 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -15,8 +15,10 @@ import ( const DefaultQueryLimit = 50 // MaxQueryLimit is the upper bound for the ?limit= parameter; values above -// this are rejected by the API layer before the store sees them. -const MaxQueryLimit = 200 +// this are rejected by the API layer before the store sees them. The API +// enforces its own configurable limit (API_MAX_LIMIT, default 500), so +// this value only matters as a store-level safety net. +const MaxQueryLimit = 500 // Event is a Soroban contract event as persisted by SoroTrail. type Event struct { @@ -115,6 +117,26 @@ type ReplayState struct { CompletedAt *time.Time } +// ContractMeta holds cached token metadata fetched via RPC simulation. +// Fields are nil when unknown — a row with IsToken=false means the contract +// was probed and does not implement the SEP-41 token interface (negative +// cache). A row with IsToken=true and nil fields means the contract IS a +// token but the RPC call hasn't resolved yet (should be rare). +type ContractMeta struct { + ContractID string `json:"contract_id"` + Name *string `json:"name"` + Symbol *string `json:"symbol"` + Decimals *int `json:"decimals"` + IsToken bool `json:"is_token"` + FetchedAt time.Time `json:"fetched_at"` +} + +// HasMetadata reports whether the contract has resolved token metadata +// (name, symbol, and decimals are all non-nil). +func (m ContractMeta) HasMetadata() bool { + return m.IsToken && m.Name != nil && m.Symbol != nil && m.Decimals != nil +} + // Done reports whether the recorded run finished its whole range. func (s ReplayState) Done() bool { return s.CompletedAt != nil } @@ -158,12 +180,6 @@ type EventFilter struct { // arrays: topic_contains=[{"symbol":"transfer"},{"address":"C..."}]. // Uses the GIN index on events.topics. TopicContains json.RawMessage - // Topic0-Topic3 match the exact JSON value at that specific topic array - // position. Unspecified positions are wildcards. - Topic0 json.RawMessage - Topic1 json.RawMessage - Topic2 json.RawMessage - Topic3 json.RawMessage // HasValue filters events by whether they carry a value payload. // nil means no constraint; true means value IS NOT NULL; // false means value IS NULL. @@ -412,54 +428,9 @@ func (f SubscriptionFilter) MatchesEvent(e Event) bool { return false } } - if len(f.TopicContains) > 0 && len(e.Topics) > 0 { - var topics []json.RawMessage - if err := json.Unmarshal(e.Topics, &topics); err != nil { - return false - } - // Unwrap a single-element array so topic_contains=[{...}] works - // the same way in-memory as it does in Postgres @> containment. - needle := f.TopicContains - var arr []json.RawMessage - if err := json.Unmarshal(f.TopicContains, &arr); err == nil && len(arr) == 1 { - needle = arr[0] - } - matched := false - for _, t := range topics { - if jsonbContains(t, needle) { - matched = true - break - } - } - if !matched { - return false - } - } return true } -// jsonbContains reports whether the container jsonb-contains the contained -// value. For objects it checks that every key in contained exists with the -// same raw JSON value in container; for scalars/arrays it falls back to -// direct byte comparison (JSON string equality). This mirrors the Postgres -// @> operator's semantics for the topic-matching use case. -func jsonbContains(container, contained json.RawMessage) bool { - // If both are objects, check key-value subset. - var cMap, dMap map[string]json.RawMessage - if json.Unmarshal(container, &cMap) == nil && json.Unmarshal(contained, &dMap) == nil { - for k, v := range dMap { - cv, ok := cMap[k] - if !ok || string(cv) != string(v) { - return false - } - } - return true - } - // Fallback: exact JSON string match (handles strings, numbers, and - // cases where unmarshalling into map failed — e.g. arrays). - return string(container) == string(contained) -} - // Subscription is one registered webhook callback. type Subscription struct { ID int64 `json:"id"` @@ -723,6 +694,14 @@ type Store interface { // API can surface 404 for typos. RemoveWatchedContract(ctx context.Context, contractID string) error + // Per-contract resume cursors for watched-contract ingestion. A + // contract that falls behind does not hold back the others; each row + // tracks its own last ingested ledger + pagination cursor. + GetContractCursor(ctx context.Context, contractID string) (ContractCursor, error) + SaveContractCursor(ctx context.Context, c ContractCursor) error + DeleteContractCursor(ctx context.Context, contractID string) error + ListContractCursors(ctx context.Context) ([]ContractCursor, error) + RecordAuditFinding(ctx context.Context, f AuditFinding) (AuditFinding, error) UpdateAuditFinding(ctx context.Context, f AuditFinding) error // ListOpenFindingsByRange returns the most recent open finding whose @@ -790,6 +769,13 @@ type Store interface { // tenants' ingestion activity. Stats(ctx context.Context, sc Scope) (Stats, error) Ping(ctx context.Context) error + + // Contract metadata (token enrichment). + ListContractIDs(ctx context.Context) ([]string, error) + GetContractMeta(ctx context.Context, contractID string) (ContractMeta, error) + UpsertContractMeta(ctx context.Context, m ContractMeta) error + CountContractEvents(ctx context.Context, contractID string) (int64, error) + ListContractsNeedingRefresh(ctx context.Context, olderThan time.Time) ([]string, error) } // DeadLetterInput is the payload handed to Store.DeadLetterEvent. The diff --git a/internal/store/store_integration_test.go b/internal/store/store_integration_test.go index 9920ea15..0f46e2db 100644 --- a/internal/store/store_integration_test.go +++ b/internal/store/store_integration_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/khaylebfortune/sorotrail/internal/testdb" + "github.com/sorotrail/sorotrail/internal/testdb" ) // TestMigrations_ApplyFromEmptyLand asserts that every embedded diff --git a/internal/store/tenant_postgres_test.go b/internal/store/tenant_postgres_test.go index 11de29f6..7e8089f7 100644 --- a/internal/store/tenant_postgres_test.go +++ b/internal/store/tenant_postgres_test.go @@ -1,3 +1,6 @@ +//go:build integration + + package store import ( diff --git a/internal/store/topic_contains_test.go b/internal/store/topic_contains_test.go index 02c360de..6eab71a9 100644 --- a/internal/store/topic_contains_test.go +++ b/internal/store/topic_contains_test.go @@ -1,3 +1,6 @@ +//go:build integration + + package store import ( diff --git a/internal/store/tracing.go b/internal/store/tracing.go index 87aed935..fd722071 100644 --- a/internal/store/tracing.go +++ b/internal/store/tracing.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -50,11 +51,11 @@ func (s *TracingStore) ReplaceEventsInRange(ctx context.Context, events []Event, return err } -func (s *TracingStore) GetEvent(ctx context.Context, id string) (Event, error) { +func (s *TracingStore) GetEvent(ctx context.Context, id string, sc Scope) (Event, error) { ctx, span := s.tracer.Start(ctx, "store.GetEvent") defer span.End() span.SetAttributes(attribute.String("store.event_id", id)) - event, err := s.Store.GetEvent(ctx, id) + event, err := s.Store.GetEvent(ctx, id, sc) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -62,11 +63,11 @@ func (s *TracingStore) GetEvent(ctx context.Context, id string) (Event, error) { return event, err } -func (s *TracingStore) EventExists(ctx context.Context, id string) (bool, error) { +func (s *TracingStore) EventExists(ctx context.Context, id string, sc Scope) (bool, error) { ctx, span := s.tracer.Start(ctx, "store.EventExists") defer span.End() span.SetAttributes(attribute.String("store.event_id", id)) - exists, err := s.Store.EventExists(ctx, id) + exists, err := s.Store.EventExists(ctx, id, sc) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -77,7 +78,7 @@ func (s *TracingStore) EventExists(ctx context.Context, id string) (bool, error) func (s *TracingStore) QueryEvents(ctx context.Context, f EventFilter) ([]Event, string, error) { ctx, span := s.tracer.Start(ctx, "store.QueryEvents") defer span.End() - span.SetAttributes(attribute.String("store.contract_id", f.ContractID), attribute.String("store.type", f.Type)) + span.SetAttributes(attribute.String("store.contract_id", f.ContractID), attribute.String("store.types", fmt.Sprint(f.Types))) events, cursor, err := s.Store.QueryEvents(ctx, f) if err != nil { span.RecordError(err)