From 085359529586ae6eef50a3f0cc307a9adfb90a4c Mon Sep 17 00:00:00 2001 From: Tibor Kircsi Date: Tue, 4 Aug 2026 15:14:44 +0200 Subject: [PATCH 1/3] feat(dir): dht only remote discovery Signed-off-by: Tibor Kircsi --- cli/cmd/daemon/daemon.config.yaml | 11 - cli/cmd/doctor/bootstrap_checks.go | 4 +- cli/cmd/doctor/bootstrap_checks_test.go | 7 +- cli/go.mod | 1 - cli/go.sum | 2 - docs/content/dir/dir-cli-reference.md | 2 - docs/content/dir/dir-deployment-kubernetes.md | 2 - docs/content/dir/dir-deployment-local.md | 2 - docs/content/dir/dir-federation-aws-eks.md | 2 - install/charts/dir/apiserver/values.yaml | 7 - install/charts/dir/values.yaml | 18 - install/docker/apiserver.env | 1 - reconciler/tasks/metrics/task.go | 99 +- reconciler/tasks/metrics/task_test.go | 200 ++++ reconciler/tasks/signature/task_test.go | 9 + server/config/config.go | 16 - server/config/config_test.go | 6 - server/controller/publication.go | 20 +- server/database/gorm/record.go | 34 + server/database/gorm/record_labels.go | 61 ++ server/database/gorm/record_labels_test.go | 60 ++ server/go.mod | 5 +- server/go.sum | 3 - server/ingest/ingest.go | 4 +- server/publication/publication.go | 31 +- server/publication/scheduler.go | 45 +- server/publication/scheduler_test.go | 239 +++++ server/routing/ROUTING.md | 677 ------------- server/routing/advertise.go | 248 +++++ server/routing/advertise_test.go | 239 +++++ server/routing/autosync/autosync.go | 446 --------- server/routing/autosync/autosync_test.go | 407 -------- server/routing/cleanup_core_test.go | 287 ------ server/routing/cleanup_tasks.go | 447 --------- server/routing/config/config.go | 105 +- server/routing/config/config_test.go | 97 -- server/routing/constants.go | 71 +- server/routing/handler.go | 107 --- server/routing/handler_test.go | 74 -- server/routing/internal/p2p/constants.go | 21 +- server/routing/internal/p2p/host.go | 2 +- server/routing/internal/p2p/mockrpc/rpc.go | 168 ---- .../routing/internal/p2p/mockstream/stream.go | 77 -- server/routing/internal/p2p/options.go | 14 - server/routing/internal/p2p/server.go | 11 - server/routing/label_keys.go | 141 +++ server/routing/label_keys_test.go | 167 ++++ server/routing/label_utils.go | 87 -- server/routing/label_utils_test.go | 420 -------- server/routing/metrics.go | 117 --- server/routing/pubsub/constants.go | 32 - server/routing/pubsub/events.go | 134 --- server/routing/pubsub/manager.go | 379 -------- server/routing/query_matching.go | 62 -- server/routing/query_matching_test.go | 299 ------ server/routing/routing.go | 144 ++- server/routing/routing_local.go | 381 ++++---- server/routing/routing_local_test.go | 293 ++---- server/routing/routing_remote.go | 891 +---------------- .../routing/routing_remote_or_logic_test.go | 162 ---- server/routing/rpc/query_records.go | 231 +++++ server/routing/rpc/query_records_test.go | 422 +++++++++ server/routing/rpc/rpc.go | 30 +- server/routing/search_remote.go | 366 +++++++ server/routing/search_remote_network_test.go | 210 ++++ server/routing/search_remote_test.go | 174 ++++ server/routing/search_simple_test.go | 373 -------- server/routing/test_utils.go | 28 +- server/routing/validators/validators.go | 440 --------- server/routing/validators/validators_test.go | 893 ------------------ server/server.go | 10 +- server/skill/publisher.go | 10 +- server/types/database.go | 13 + server/types/label.go | 50 +- server/types/search.go | 9 + .../testenv/default/dir-daemon-config.yaml | 2 - .../testenv/external/dir-daemon-config.yaml | 2 - .../testenv/local/dir-daemon-config.yaml | 2 - tests/e2e/network/04_gossipsub_test.go | 365 ------- tests/e2e/network/cleanup.go | 4 - .../local/daemon-bootstrap-config.tpl.yaml | 2 - tests/go.mod | 1 - tests/go.sum | 2 - .../server/testenv/test-config.yaml | 2 - 84 files changed, 3491 insertions(+), 8248 deletions(-) create mode 100644 reconciler/tasks/metrics/task_test.go create mode 100644 server/database/gorm/record_labels.go create mode 100644 server/database/gorm/record_labels_test.go create mode 100644 server/publication/scheduler_test.go delete mode 100644 server/routing/ROUTING.md create mode 100644 server/routing/advertise.go create mode 100644 server/routing/advertise_test.go delete mode 100644 server/routing/autosync/autosync.go delete mode 100644 server/routing/autosync/autosync_test.go delete mode 100644 server/routing/cleanup_core_test.go delete mode 100644 server/routing/cleanup_tasks.go delete mode 100644 server/routing/config/config_test.go delete mode 100644 server/routing/handler.go delete mode 100644 server/routing/handler_test.go delete mode 100644 server/routing/internal/p2p/mockrpc/rpc.go delete mode 100644 server/routing/internal/p2p/mockstream/stream.go create mode 100644 server/routing/label_keys.go create mode 100644 server/routing/label_keys_test.go delete mode 100644 server/routing/label_utils.go delete mode 100644 server/routing/label_utils_test.go delete mode 100644 server/routing/metrics.go delete mode 100644 server/routing/pubsub/constants.go delete mode 100644 server/routing/pubsub/events.go delete mode 100644 server/routing/pubsub/manager.go delete mode 100644 server/routing/routing_remote_or_logic_test.go create mode 100644 server/routing/rpc/query_records.go create mode 100644 server/routing/rpc/query_records_test.go create mode 100644 server/routing/search_remote.go create mode 100644 server/routing/search_remote_network_test.go create mode 100644 server/routing/search_remote_test.go delete mode 100644 server/routing/search_simple_test.go delete mode 100644 server/routing/validators/validators.go delete mode 100644 server/routing/validators/validators_test.go delete mode 100644 tests/e2e/network/04_gossipsub_test.go diff --git a/cli/cmd/daemon/daemon.config.yaml b/cli/cmd/daemon/daemon.config.yaml index 186f67802..0daceed5b 100644 --- a/cli/cmd/daemon/daemon.config.yaml +++ b/cli/cmd/daemon/daemon.config.yaml @@ -57,17 +57,6 @@ server: # more announcement traffic and more work for every subscriber. republish_interval: "36h" - gossipsub: - enabled: true - # DHT-based record + referrer autosync (deny-by-default; disabled unless set). - # When enabled, records announced by an allow-listed peer are pulled and - # ingested locally over the libp2p/DHT transport. - autosync: - enabled: false - # peerlist is a list of trusted source peers (by libp2p peer ID). - # peerlist: - # - peer: "12D3KooW...peerID1" - # - peer: "12D3KooW...peerID2" # Circuit-relay v2 for NAT traversal. # relay_service: enable a relay service on this node (only on publicly # reachable nodes, e.g. bootstrap) so it can relay traffic for NAT'd peers. diff --git a/cli/cmd/doctor/bootstrap_checks.go b/cli/cmd/doctor/bootstrap_checks.go index ff7679fe7..f4a4e3658 100644 --- a/cli/cmd/doctor/bootstrap_checks.go +++ b/cli/cmd/doctor/bootstrap_checks.go @@ -9,6 +9,7 @@ import ( "strings" "time" + serverrouting "github.com/agntcy/dir/server/routing" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" @@ -246,9 +247,8 @@ func addPeerProtocolDetails(details map[string]string, protocols []protocol.ID, details["protocol_count"] = fmt.Sprintf("%d", len(protocolStrings)) details["protocols"] = strings.Join(protocolStrings, ",") - details["has_kad_dht_protocol"] = fmt.Sprintf("%t", hasProtocolPrefix(protocolStrings, "/ipfs/kad") || hasProtocolPrefix(protocolStrings, "dir/kad")) + details["has_kad_dht_protocol"] = fmt.Sprintf("%t", hasProtocolPrefix(protocolStrings, "/ipfs/kad") || hasProtocolPrefix(protocolStrings, serverrouting.ProtocolPrefix+"/kad")) details["has_dir_rpc_protocol"] = fmt.Sprintf("%t", hasProtocolPrefix(protocolStrings, "/dir/rpc")) - details["has_gossipsub_protocol"] = fmt.Sprintf("%t", hasProtocolPrefix(protocolStrings, "/meshsub") || hasProtocolPrefix(protocolStrings, "/floodsub")) } func hasProtocolPrefix(protocols []string, prefix string) bool { diff --git a/cli/cmd/doctor/bootstrap_checks_test.go b/cli/cmd/doctor/bootstrap_checks_test.go index 139ac3119..5794157a3 100644 --- a/cli/cmd/doctor/bootstrap_checks_test.go +++ b/cli/cmd/doctor/bootstrap_checks_test.go @@ -108,13 +108,12 @@ func TestBootstrapPeerValidationHelpers(t *testing.T) { func TestAddPeerProtocolDetails(t *testing.T) { details := map[string]string{} - addPeerProtocolDetails(details, []protocol.ID{"/ipfs/kad/1.0.0", "/dir/rpc/0.1.0", "/meshsub/1.1.0"}, nil) + addPeerProtocolDetails(details, []protocol.ID{"/dir/2/kad/1.0.0", "/dir/rpc/2.0.0"}, nil) - assert.Equal(t, "3", details["protocol_count"]) + assert.Equal(t, "2", details["protocol_count"]) assert.Equal(t, "true", details["has_kad_dht_protocol"]) assert.Equal(t, "true", details["has_dir_rpc_protocol"]) - assert.Equal(t, "true", details["has_gossipsub_protocol"]) - assert.True(t, hasProtocolPrefix([]string{"/dir/rpc/0.1.0"}, "/dir/rpc")) + assert.True(t, hasProtocolPrefix([]string{"/dir/rpc/2.0.0"}, "/dir/rpc")) assert.False(t, hasProtocolPrefix([]string{"/other/1.0.0"}, "/dir/rpc")) } diff --git a/cli/go.mod b/cli/go.mod index 0b9ea1ccc..563043c92 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -520,7 +520,6 @@ require ( github.com/libp2p/go-libp2p-gorpc v0.6.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.41.0 github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect - github.com/libp2p/go-libp2p-pubsub v0.16.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect diff --git a/cli/go.sum b/cli/go.sum index 145051e35..3390f8a47 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -1162,8 +1162,6 @@ github.com/libp2p/go-libp2p-kad-dht v0.41.0 h1:sDigz5SgV20Crj8ItJmJpEAM+eJrzC/Sa github.com/libp2p/go-libp2p-kad-dht v0.41.0/go.mod h1:2qc4QGLvmIdznYbNg++FF76vp4q2SaBZyr76jHV8xgs= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= -github.com/libp2p/go-libp2p-pubsub v0.16.0 h1:j7G2C8kJwkcAQqYR7Wmq3d75d3Sgw/N0Hhiv0dVx7OY= -github.com/libp2p/go-libp2p-pubsub v0.16.0/go.mod h1:lr4oE8bFgQaifRcoc2uWhWWiK6tPdOEKpUuR408GFN4= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= diff --git a/docs/content/dir/dir-cli-reference.md b/docs/content/dir/dir-cli-reference.md index 58f7dc6d5..59659f4bb 100644 --- a/docs/content/dir/dir-cli-reference.md +++ b/docs/content/dir/dir-cli-reference.md @@ -312,8 +312,6 @@ The daemon ships with sensible built-in defaults. To customize, pass a YAML conf routing: listen_address: "/ip4/0.0.0.0/tcp/8999" datastore_dir: "routing" - gossipsub: - enabled: true database: type: "sqlite" sqlite: diff --git a/docs/content/dir/dir-deployment-kubernetes.md b/docs/content/dir/dir-deployment-kubernetes.md index 70f2b45b1..3184080f1 100644 --- a/docs/content/dir/dir-deployment-kubernetes.md +++ b/docs/content/dir/dir-deployment-kubernetes.md @@ -124,8 +124,6 @@ The Agent Directory Service can be deployed using Helm or GitOps / Argo CD. Helm listen_address: "/ip4/0.0.0.0/tcp/5555" datastore_dir: /etc/routing/datastore directory_api_address: "dir-apiserver.dir-dev-dir.svc.cluster.local:8888" - gossipsub: - enabled: false sync: auth_config: username: "user" diff --git a/docs/content/dir/dir-deployment-local.md b/docs/content/dir/dir-deployment-local.md index a3323c132..e4695273e 100644 --- a/docs/content/dir/dir-deployment-local.md +++ b/docs/content/dir/dir-deployment-local.md @@ -144,8 +144,6 @@ server: datastore_dir: "routing" bootstrap_peers: - "/dns4/remote-dir.example.com/tcp/8999/p2p/" - gossipsub: - enabled: true database: type: "sqlite" sqlite: diff --git a/docs/content/dir/dir-federation-aws-eks.md b/docs/content/dir/dir-federation-aws-eks.md index 8ada2f195..5bf7c7ed3 100644 --- a/docs/content/dir/dir-federation-aws-eks.md +++ b/docs/content/dir/dir-federation-aws-eks.md @@ -382,8 +382,6 @@ This guide does not try to provision the AWS infrastructure from zero in the mai key_path: /etc/routing/node.privkey datastore_dir: /etc/routing/datastore directory_api_address: "${DIR_API_HOST}:443" - gossipsub: - enabled: true sync: auth_config: username: "user" diff --git a/install/charts/dir/apiserver/values.yaml b/install/charts/dir/apiserver/values.yaml index 3aeae41b3..97873c3d8 100644 --- a/install/charts/dir/apiserver/values.yaml +++ b/install/charts/dir/apiserver/values.yaml @@ -99,13 +99,6 @@ config: bootstrap_peers: - /dns4/routing.ads.outshift.io/tcp/5555/p2p/12D3KooWLf9p3cedc86xGQBaqak6rAFmQk1HxKAK1yh7umHE3amu - # GossipSub configuration for efficient label announcements - # When enabled, labels are propagated via GossipSub mesh to ALL subscribed peers - # When disabled, falls back to DHT+Pull mechanism (higher bandwidth, limited reach) - # Default: true (recommended for production) - gossipsub: - enabled: true - # Sync configuration sync: # Authentication configuration for sync operations diff --git a/install/charts/dir/values.yaml b/install/charts/dir/values.yaml index 0aea153dd..8d82c7397 100644 --- a/install/charts/dir/values.yaml +++ b/install/charts/dir/values.yaml @@ -203,24 +203,6 @@ apiserver: bootstrap_peers: - /dns4/routing.ads.outshift.io/tcp/5555/p2p/12D3KooWLf9p3cedc86xGQBaqak6rAFmQk1HxKAK1yh7umHE3amu - # GossipSub configuration for efficient label announcements - # When enabled, labels are propagated via GossipSub mesh to ALL subscribed peers - # When disabled, falls back to DHT+Pull mechanism (higher bandwidth, limited reach) - # Default: true (recommended for production) - gossipsub: - enabled: true - - # DHT-based record + referrer autosync. - # Deny-by-default: when enabled, only records announced by a peer in - # peerlist are pulled and ingested locally over the libp2p/DHT transport. - # peerlist is a list of trusted source peers (by libp2p peer ID) and is - # configured via values/config only (not a single environment variable). - autosync: - enabled: false - # peerlist: - # - peer: "12D3KooW...peerID1" - # - peer: "12D3KooW...peerID2" - # Circuit-relay v2 for NAT traversal. # relay_service: run a relay service on this node so it can relay traffic # for NAT'd peers. Enable only on publicly-reachable nodes (e.g. the diff --git a/install/docker/apiserver.env b/install/docker/apiserver.env index c48846e03..670401305 100644 --- a/install/docker/apiserver.env +++ b/install/docker/apiserver.env @@ -40,7 +40,6 @@ DIRECTORY_SERVER_ROUTING_DIRECTORY_API_ADDRESS= DIRECTORY_SERVER_ROUTING_BOOTSTRAP_PEERS= DIRECTORY_SERVER_ROUTING_KEY_PATH= DIRECTORY_SERVER_ROUTING_DATASTORE_DIR= -DIRECTORY_SERVER_ROUTING_GOSSIPSUB_ENABLED=true # Database Configuration (PostgreSQL) DIRECTORY_SERVER_DATABASE_TYPE=postgres diff --git a/reconciler/tasks/metrics/task.go b/reconciler/tasks/metrics/task.go index 319b5ac10..043124c52 100644 --- a/reconciler/tasks/metrics/task.go +++ b/reconciler/tasks/metrics/task.go @@ -12,6 +12,7 @@ package metrics import ( "context" "fmt" + "sync" "time" "github.com/agntcy/dir/server/types" @@ -20,6 +21,26 @@ import ( var logger = logging.Logger("reconciler/metrics") +// providerCountWorkers bounds how many provider lookups are in flight at once. +const providerCountWorkers = 8 + +// maxProviderCount is an upper bound for a stored provider count. A real DHT +// lookup returns orders of magnitude fewer peers; the cap exists so the +// conversion to the column's width cannot wrap. +const maxProviderCount = 1 << 20 + +func clampProviderCount(count int) uint32 { + if count < 0 { + return 0 + } + + if count > maxProviderCount { + return maxProviderCount + } + + return uint32(count) +} + // ProviderCounterAPI is the minimal interface required by the metrics task to // query provider counts. It is satisfied by types.RoutingAPI (daemon mode, where // the routing layer is shared in-process) and by GRPCProviderCounter (standalone @@ -96,28 +117,21 @@ func (t *Task) refreshProviderCounts(ctx context.Context) error { var updated, failed int - for _, cid := range cids { - select { - case <-ctx.Done(): - return fmt.Errorf("context cancelled: %w", ctx.Err()) - default: - } - - count, err := t.counters.GetProviderCount(ctx, cid) - if err != nil { - logger.Warn("Failed to get provider count", "cid", cid, "error", err) + // Persist serially while the lookups run concurrently. The lookups are the + // slow part; the metrics table serialises writers regardless. + for res := range t.countProviders(ctx, cids) { + if res.err != nil { + logger.Warn("Failed to get provider count", "cid", res.cid, "error", res.err) failed++ continue } - if count < 0 { - count = 0 - } + count := clampProviderCount(res.count) - if err := t.db.SetProviderCount(cid, uint32(count)); err != nil { - logger.Warn("Failed to set provider count", "cid", cid, "count", count, "error", err) + if err := t.db.SetProviderCount(res.cid, count); err != nil { + logger.Warn("Failed to set provider count", "cid", res.cid, "count", count, "error", err) failed++ @@ -127,7 +141,62 @@ func (t *Task) refreshProviderCounts(ctx context.Context) error { updated++ } + if err := ctx.Err(); err != nil { + return fmt.Errorf("context cancelled: %w", err) + } + logger.Info("Provider count refresh complete", "updated", updated, "failed", failed) return nil } + +type providerCountResult struct { + cid string + count int + err error +} + +// countProviders fans the CID list out over a bounded worker pool and streams +// results back as they land. Each GetProviderCount is a DHT lookup, so running +// one CID at a time would exceed the reconciliation interval on any sizeable +// corpus. +func (t *Task) countProviders(ctx context.Context, cids []string) <-chan providerCountResult { + jobs := make(chan string) + results := make(chan providerCountResult) + workers := min(providerCountWorkers, len(cids)) + + var wg sync.WaitGroup + + for range workers { + wg.Go(func() { + for cid := range jobs { + count, err := t.counters.GetProviderCount(ctx, cid) + + select { + case results <- providerCountResult{cid: cid, count: count, err: err}: + case <-ctx.Done(): + return + } + } + }) + } + + go func() { + defer close(jobs) + + for _, cid := range cids { + select { + case jobs <- cid: + case <-ctx.Done(): + return + } + } + }() + + go func() { + wg.Wait() + close(results) + }() + + return results +} diff --git a/reconciler/tasks/metrics/task_test.go b/reconciler/tasks/metrics/task_test.go new file mode 100644 index 000000000..884efb98b --- /dev/null +++ b/reconciler/tasks/metrics/task_test.go @@ -0,0 +1,200 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "context" + "errors" + "fmt" + "maps" + "sync" + "testing" + + "github.com/agntcy/dir/server/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeMetricsDB implements the two narrow database interfaces the task holds. +// Everything else is inherited as nil and would panic if the task reached for +// it, which is the point. +type fakeMetricsDB struct { + types.DatabaseAPI + + cids []string + + mu sync.Mutex + written map[string]uint32 + writeErr map[string]error +} + +func newFakeMetricsDB(cids ...string) *fakeMetricsDB { + return &fakeMetricsDB{ + cids: cids, + written: make(map[string]uint32), + writeErr: make(map[string]error), + } +} + +func (f *fakeMetricsDB) snapshot() map[string]uint32 { + f.mu.Lock() + defer f.mu.Unlock() + + out := make(map[string]uint32, len(f.written)) + maps.Copy(out, f.written) + + return out +} + +func (f *fakeMetricsDB) GetRecordCIDs(_ ...types.FilterOption) ([]string, error) { + return f.cids, nil +} + +func (f *fakeMetricsDB) SetProviderCount(cid string, count uint32) error { + f.mu.Lock() + defer f.mu.Unlock() + + if err, ok := f.writeErr[cid]; ok { + return err + } + + f.written[cid] = count + + return nil +} + +// fakeCounter records every CID it is asked about. It is called concurrently by +// the worker pool, so all state is mutex-guarded. +type fakeCounter struct { + mu sync.Mutex + asked []string + counts map[string]int + errs map[string]error +} + +func newFakeCounter() *fakeCounter { + return &fakeCounter{ + counts: make(map[string]int), + errs: make(map[string]error), + } +} + +func (f *fakeCounter) GetProviderCount(ctx context.Context, cid string) (int, error) { + if err := ctx.Err(); err != nil { + return 0, fmt.Errorf("lookup cancelled: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.asked = append(f.asked, cid) + + if err, ok := f.errs[cid]; ok { + return 0, err + } + + return f.counts[cid], nil +} + +func (f *fakeCounter) askedCount() int { + f.mu.Lock() + defer f.mu.Unlock() + + return len(f.asked) +} + +func newTestTask(db *fakeMetricsDB, counters ProviderCounterAPI) *Task { + return &Task{ + config: Config{Enabled: true}, + db: db, + search: db, + counters: counters, + } +} + +func TestRefreshProviderCountsPersistsEveryCID(t *testing.T) { + t.Parallel() + + // More CIDs than workers, so the pool has to cycle. + cids := make([]string, 0, providerCountWorkers*3) + counter := newFakeCounter() + + for i := range cap(cids) { + cid := fmt.Sprintf("cid-%d", i) + cids = append(cids, cid) + counter.counts[cid] = i + } + + db := newFakeMetricsDB(cids...) + + require.NoError(t, newTestTask(db, counter).refreshProviderCounts(t.Context())) + + written := db.snapshot() + assert.Len(t, written, len(cids)) + + for i, cid := range cids { + assert.Equal(t, uint32(i), written[cid], "cid %s", cid) //nolint:gosec + } +} + +func TestRefreshProviderCountsSkipsFailedLookups(t *testing.T) { + t.Parallel() + + counter := newFakeCounter() + counter.counts["good"] = 3 + counter.errs["bad"] = errors.New("lookup failed") + + db := newFakeMetricsDB("good", "bad") + + require.NoError(t, newTestTask(db, counter).refreshProviderCounts(t.Context())) + + // A failed lookup must not zero the existing gauge, and must not stop the + // remaining CIDs from being refreshed. + assert.Equal(t, map[string]uint32{"good": 3}, db.snapshot()) +} + +func TestRefreshProviderCountsSurvivesWriteFailure(t *testing.T) { + t.Parallel() + + counter := newFakeCounter() + counter.counts["ok"] = 1 + counter.counts["unwritable"] = 2 + + db := newFakeMetricsDB("unwritable", "ok") + db.writeErr["unwritable"] = errors.New("db is locked") + + require.NoError(t, newTestTask(db, counter).refreshProviderCounts(t.Context())) + + assert.Equal(t, map[string]uint32{"ok": 1}, db.snapshot()) +} + +func TestRefreshProviderCountsStopsOnCancel(t *testing.T) { + t.Parallel() + + cids := make([]string, 0, 500) + for i := range cap(cids) { + cids = append(cids, fmt.Sprintf("cid-%d", i)) + } + + counter := newFakeCounter() + db := newFakeMetricsDB(cids...) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + err := newTestTask(db, counter).refreshProviderCounts(ctx) + require.ErrorIs(t, err, context.Canceled) + + // Cancellation must unwind the pool rather than grinding through the corpus. + assert.Less(t, counter.askedCount(), len(cids)) +} + +func TestRefreshProviderCountsNoRecords(t *testing.T) { + t.Parallel() + + counter := newFakeCounter() + + require.NoError(t, newTestTask(newFakeMetricsDB(), counter).refreshProviderCounts(t.Context())) + assert.Equal(t, 0, counter.askedCount()) +} diff --git a/reconciler/tasks/signature/task_test.go b/reconciler/tasks/signature/task_test.go index 3b997ff46..8b0c9104a 100644 --- a/reconciler/tasks/signature/task_test.go +++ b/reconciler/tasks/signature/task_test.go @@ -204,6 +204,10 @@ func (f *fakeSignatureDB) GetRecords(opts ...types.FilterOption) ([]coretypes.Re return nil, nil } +func (f *fakeSignatureDB) GetRecordLabels(cids []string) (map[string][]types.Label, error) { + return nil, nil +} + func (f *fakeSignatureDB) GetCatalogEntries(opts ...types.CatalogQueryOption) ([]*catalogv1.CatalogEntry, bool, error) { return nil, false, nil } @@ -218,6 +222,11 @@ func (f *fakeSignatureDB) ListCatalogTags() ([]*catalogv1.CatalogTag, error) { func (f *fakeSignatureDB) RemoveRecord(cid string) error { return nil } func (f *fakeSignatureDB) SetRecordSigned(recordCID string) error { return nil } + +func (f *fakeSignatureDB) SetRecordPublished(recordCID string, advertised bool) error { + return nil +} + func (f *fakeSignatureDB) CreateSync(remoteURL string, cids []string, remoteRegistryURL string, repositoryName string) (string, error) { return "", nil } diff --git a/server/config/config.go b/server/config/config.go index da9b6a3e7..be9e3741c 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -517,22 +517,6 @@ func LoadConfig(opts ...ConfigOption) (*Config, error) { _ = v.BindEnv("routing.republish_interval") - // - // Routing GossipSub configuration - // Note: Only enable/disable is configurable. Protocol parameters (topic, message size) - // are hardcoded in server/routing/pubsub/constants.go for network compatibility. - // - _ = v.BindEnv("routing.gossipsub.enabled") - v.SetDefault("routing.gossipsub.enabled", routing.DefaultGossipSubEnabled) - - // - // Routing autosync configuration (DHT-based record + referrer sync). - // Note: routing.autosync.peerlist is a list of objects and is configured via - // config file/YAML only (it cannot be bound to a single environment variable). - // - _ = v.BindEnv("routing.autosync.enabled") - v.SetDefault("routing.autosync.enabled", routing.DefaultAutosyncEnabled) - // // Routing relay configuration (circuit-relay v2 for NAT traversal). // Note: routing.static_relays is a list and is configured via config diff --git a/server/config/config_test.go b/server/config/config_test.go index 91ca1cc22..377197645 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -100,9 +100,6 @@ func TestConfig(t *testing.T) { KeyPath: "/path/to/key", RefreshInterval: 5 * time.Second, RepublishInterval: 10 * time.Minute, - GossipSub: routing.GossipSubConfig{ - Enabled: true, // Default value - }, }, Database: dbconfig.Config{ Type: "postgres", @@ -176,9 +173,6 @@ func TestConfig(t *testing.T) { Routing: routing.Config{ ListenAddress: routing.DefaultListenAddress, BootstrapPeers: routing.DefaultBootstrapPeers, - GossipSub: routing.GossipSubConfig{ - Enabled: routing.DefaultGossipSubEnabled, - }, }, Database: dbconfig.Config{ Type: dbconfig.DefaultType, diff --git a/server/controller/publication.go b/server/controller/publication.go index e3b14648f..dbe47f028 100644 --- a/server/controller/publication.go +++ b/server/controller/publication.go @@ -19,19 +19,25 @@ var publicationLogger = logging.Logger("controller/publication") // publicationCtlr implements the PublicationService gRPC interface. type publicationCtlr struct { routingv1.UnimplementedPublicationServiceServer - db types.DatabaseAPI - opts types.APIOptions + db types.DatabaseAPI + publication types.PublicationAPI + opts types.APIOptions } // NewPublicationController creates a new publication controller. -func NewPublicationController(db types.DatabaseAPI, opts types.APIOptions) routingv1.PublicationServiceServer { +// +// Creation goes through the publication service rather than the database so +// that this path schedules work as promptly as RoutingService.Publish; reads +// and deletes stay on the database. +func NewPublicationController(db types.DatabaseAPI, publication types.PublicationAPI, opts types.APIOptions) routingv1.PublicationServiceServer { return &publicationCtlr{ - db: db, - opts: opts, + db: db, + publication: publication, + opts: opts, } } -func (c *publicationCtlr) CreatePublication(_ context.Context, req *routingv1.PublishRequest) (*routingv1.CreatePublicationResponse, error) { +func (c *publicationCtlr) CreatePublication(ctx context.Context, req *routingv1.PublishRequest) (*routingv1.CreatePublicationResponse, error) { publicationLogger.Debug("Called publication controller's CreatePublication method") // Validate the publish request @@ -53,7 +59,7 @@ func (c *publicationCtlr) CreatePublication(_ context.Context, req *routingv1.Pu return nil, status.Errorf(codes.InvalidArgument, "invalid publish request: must specify record_refs, queries, or all_records") } - id, err := c.db.CreatePublication(req) + id, err := c.publication.CreatePublication(ctx, req) if err != nil { return nil, fmt.Errorf("failed to create publication: %w", err) } diff --git a/server/database/gorm/record.go b/server/database/gorm/record.go index d0d34ce65..13a23fd2b 100644 --- a/server/database/gorm/record.go +++ b/server/database/gorm/record.go @@ -54,6 +54,14 @@ type Record struct { Authors []string `gorm:"column:authors;serializer:json"` // Stored as JSON array Signed bool `gorm:"column:signed;default:false"` // Whether at least one signature is attached + // Published is whether this node announces the record to the network. + // + // It defaults to false: holding a record makes it servable to anyone who + // knows its CID, but not discoverable. Only Publish sets it, so a record + // can be pushed, signed and scanned before anyone can find it, and a + // replica pulled in by sync stays private unless it is published here too. + Published bool `gorm:"column:published;default:false;not null"` + Skills []Skill `gorm:"foreignKey:RecordCID;references:RecordCID;constraint:OnDelete:CASCADE"` Locators []Locator `gorm:"foreignKey:RecordCID;references:RecordCID;constraint:OnDelete:CASCADE"` Modules []Module `gorm:"foreignKey:RecordCID;references:RecordCID;constraint:OnDelete:CASCADE"` @@ -583,6 +591,11 @@ func (d *DB) handleFilterOptions(query *gorm.DB, cfg *types.RecordFilters) *gorm } } + // Handle published filter (whether this node announces the record). + if cfg.Published != nil { + query = query.Where("records.published = ?", *cfg.Published) + } + // Handle description filters with wildcard support. if len(cfg.Descriptions) > 0 { condition, args := utils.BuildWildcardCondition("records.description", cfg.Descriptions) @@ -628,6 +641,27 @@ func scanSeveritiesGTE(threshold string) []string { return order[idx:] } +// SetRecordPublished sets whether this node announces the record to the +// network. It is what makes both Publish and Unpublish survive a restart: the +// reprovide cycle enumerates published records only. +func (d *DB) SetRecordPublished(recordCID string, published bool) error { + result := d.gormDB.Model(&Record{}). + Where("record_cid = ?", recordCID). + Update("published", published) + + if result.Error != nil { + return fmt.Errorf("failed to set record published flag: %w", result.Error) + } + + if result.RowsAffected == 0 { + return fmt.Errorf("record not found: %s", recordCID) + } + + logger.Debug("Set record published flag", "record_cid", recordCID, "published", published) + + return nil +} + // SetRecordSigned marks a record as signed. // This is called when a signature is attached to a record. func (d *DB) SetRecordSigned(recordCID string) error { diff --git a/server/database/gorm/record_labels.go b/server/database/gorm/record_labels.go new file mode 100644 index 000000000..07f01baa8 --- /dev/null +++ b/server/database/gorm/record_labels.go @@ -0,0 +1,61 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package gorm + +import ( + "fmt" + + "github.com/agntcy/dir/server/types" +) + +// labelRow is the shared scan target for the four label tables. +type labelRow struct { + RecordCID string `gorm:"column:record_cid"` + Name string `gorm:"column:name"` +} + +// GetRecordLabels returns the routing labels of each given record, keyed by CID. +// +// One query per label table rather than preloading whole records: callers want +// the labels and nothing else, and the associations on Record would drag in +// every column of every skill, module, domain and locator row. +func (d *DB) GetRecordLabels(cids []string) (map[string][]types.Label, error) { + labels := make(map[string][]types.Label, len(cids)) + + if len(cids) == 0 { + return labels, nil + } + + // Locators are keyed by type rather than name, matching how + // types.GetLabelsFromRecord builds them. + sources := []struct { + table string + nameCol string + labelType types.LabelType + }{ + {"skills", "name", types.LabelTypeSkill}, + {"domains", "name", types.LabelTypeDomain}, + {"modules", "name", types.LabelTypeModule}, + {"locators", "type", types.LabelTypeLocator}, + } + + for _, source := range sources { + var rows []labelRow + + err := d.gormDB. + Table(source.table). + Select("record_cid, "+source.nameCol+" AS name"). + Where("record_cid IN ?", cids). + Find(&rows).Error + if err != nil { + return nil, fmt.Errorf("failed to load %s labels: %w", source.table, err) + } + + for _, row := range rows { + labels[row.RecordCID] = append(labels[row.RecordCID], source.labelType.LabelKey(row.Name)) + } + } + + return labels, nil +} diff --git a/server/database/gorm/record_labels_test.go b/server/database/gorm/record_labels_test.go new file mode 100644 index 000000000..49f82b104 --- /dev/null +++ b/server/database/gorm/record_labels_test.go @@ -0,0 +1,60 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package gorm + +import ( + "testing" + + typesv1alpha1 "buf.build/gen/go/agntcy/oasf/protocolbuffers/go/agntcy/oasf/types/v1alpha1" + corev1 "github.com/agntcy/dir/api/core/v1" + "github.com/agntcy/dir/server/types" + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// TestGetRecordLabelsMatchesTheRecord guards the seam between the two ways a +// record's labels reach the DHT. Publish advertises what +// types.GetLabelsFromRecord derives from the record; every reprovide afterwards +// advertises what this method reads back from the index. If they drift, a +// record is announced once and then silently goes dark when its provider +// records expire. +func TestGetRecordLabelsMatchesTheRecord(t *testing.T) { + gdb, err := gorm.Open(sqlite.Open("file::memory:"), &gorm.Config{}) + require.NoError(t, err) + + db := &DB{gormDB: gdb} + require.NoError(t, db.migrate()) + + record := corev1.New(&typesv1alpha1.Record{ + Name: "label-parity", + SchemaVersion: "0.7.0", + Skills: []*typesv1alpha1.Skill{ + {Name: "AI/ML"}, + {Name: "AI/NLP"}, + }, + Domains: []*typesv1alpha1.Domain{ + {Name: "healthcare"}, + }, + Modules: []*typesv1alpha1.Module{ + {Name: "runtime/python"}, + }, + Locators: []*typesv1alpha1.Locator{ + {Type: "docker-image", Url: "https://example.test/image"}, + }, + }) + + adapter, err := record.Decode() + require.NoError(t, err) + require.NoError(t, db.AddRecord(adapter)) + + fromIndex, err := db.GetRecordLabels([]string{record.GetCid()}) + require.NoError(t, err) + + assert.ElementsMatch(t, + types.GetLabelsFromRecord(adapter), + fromIndex[record.GetCid()], + "labels read back from the index must match the ones Publish advertises") +} diff --git a/server/go.mod b/server/go.mod index 76ebf7ce3..ebffad67c 100644 --- a/server/go.mod +++ b/server/go.mod @@ -29,8 +29,6 @@ require ( github.com/libp2p/go-libp2p v0.48.0 github.com/libp2p/go-libp2p-gorpc v0.6.0 github.com/libp2p/go-libp2p-kad-dht v0.41.0 - github.com/libp2p/go-libp2p-pubsub v0.16.0 - github.com/libp2p/go-libp2p-record v0.3.1 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c github.com/opencontainers/image-spec v1.1.1 github.com/spf13/cobra v1.10.2 @@ -55,6 +53,7 @@ require ( github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/golang/mock v1.7.0-rc.1 // indirect github.com/google/cel-go v0.29.0 // indirect + github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pion/turn/v5 v5.0.3 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect @@ -89,13 +88,11 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/boxo v0.39.0 // indirect diff --git a/server/go.sum b/server/go.sum index 05f127e2b..5e556e797 100644 --- a/server/go.sum +++ b/server/go.sum @@ -169,7 +169,6 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -377,8 +376,6 @@ github.com/libp2p/go-libp2p-kad-dht v0.41.0 h1:sDigz5SgV20Crj8ItJmJpEAM+eJrzC/Sa github.com/libp2p/go-libp2p-kad-dht v0.41.0/go.mod h1:2qc4QGLvmIdznYbNg++FF76vp4q2SaBZyr76jHV8xgs= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= -github.com/libp2p/go-libp2p-pubsub v0.16.0 h1:j7G2C8kJwkcAQqYR7Wmq3d75d3Sgw/N0Hhiv0dVx7OY= -github.com/libp2p/go-libp2p-pubsub v0.16.0/go.mod h1:lr4oE8bFgQaifRcoc2uWhWWiK6tPdOEKpUuR408GFN4= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= diff --git a/server/ingest/ingest.go b/server/ingest/ingest.go index b84c1748b..203485984 100644 --- a/server/ingest/ingest.go +++ b/server/ingest/ingest.go @@ -5,8 +5,8 @@ // records and referrers with full parity to a normal push: // content store + search index + referrer-derived database state. // -// It is used by the gRPC store controller and by DHT-based autosync so that -// content received from any source is stored and indexed identically. +// It is the single path for incoming content, so that records received from +// any source are stored and indexed identically. package ingest import ( diff --git a/server/publication/publication.go b/server/publication/publication.go index 8b58f30dc..aa0c7d665 100644 --- a/server/publication/publication.go +++ b/server/publication/publication.go @@ -26,6 +26,11 @@ type Service struct { scheduler *Scheduler workers []*Worker + // wakeCh is owned by the service rather than the scheduler so that it is + // usable before Start and never reassigned, letting CreatePublication + // signal without synchronising on the scheduler field. + wakeCh chan struct{} + stopCh chan struct{} wg sync.WaitGroup } @@ -37,13 +42,33 @@ func New(db types.DatabaseAPI, store types.StoreAPI, routing types.RoutingAPI, o store: store, routing: routing, config: opts.Config().Publication, + wakeCh: make(chan struct{}, 1), stopCh: make(chan struct{}), }, nil } -// CreatePublication creates a new publication task to be processed. +// CreatePublication creates a new publication task and wakes the scheduler to +// pick it up, so a publish takes effect in the time it takes to reach the DHT +// rather than by the next scheduler interval. func (s *Service) CreatePublication(_ context.Context, req *routingv1.PublishRequest) (string, error) { - return s.db.CreatePublication(req) //nolint:wrapcheck + id, err := s.db.CreatePublication(req) + if err != nil { + return "", err //nolint:wrapcheck + } + + s.wake() + + return id, nil +} + +// wake asks the scheduler to sweep for pending publications. The signal is +// dropped when one is already queued: a sweep that has not yet run will observe +// every publication committed before it, so one pending wake covers a burst. +func (s *Service) wake() { + select { + case s.wakeCh <- struct{}{}: + default: + } } // Start begins the publication service operations. @@ -54,7 +79,7 @@ func (s *Service) Start(ctx context.Context) error { workQueue := make(chan publypes.WorkItem, 100) //nolint:mnd // Create and start scheduler - s.scheduler = NewScheduler(s.db, workQueue, s.config.SchedulerInterval) + s.scheduler = NewScheduler(s.db, workQueue, s.config.SchedulerInterval, s.wakeCh) // Create and start workers s.workers = make([]*Worker, s.config.WorkerCount) diff --git a/server/publication/scheduler.go b/server/publication/scheduler.go index 9523d7051..9ede053a1 100644 --- a/server/publication/scheduler.go +++ b/server/publication/scheduler.go @@ -17,14 +17,20 @@ type Scheduler struct { db types.PublicationDatabaseAPI workQueue chan<- publypes.WorkItem interval time.Duration + wakeCh <-chan struct{} } // NewScheduler creates a new scheduler instance. -func NewScheduler(db types.PublicationDatabaseAPI, workQueue chan<- publypes.WorkItem, interval time.Duration) *Scheduler { +// +// wakeCh lets a caller run a sweep ahead of the next tick. The ticker remains as +// a backstop for publications a sweep left behind, such as those skipped while +// the work queue was full. +func NewScheduler(db types.PublicationDatabaseAPI, workQueue chan<- publypes.WorkItem, interval time.Duration, wakeCh <-chan struct{}) *Scheduler { return &Scheduler{ db: db, workQueue: workQueue, interval: interval, + wakeCh: wakeCh, } } @@ -36,7 +42,7 @@ func (s *Scheduler) Run(ctx context.Context, stopCh <-chan struct{}) { defer ticker.Stop() // Process immediately on start - s.processPendingPublications(ctx) + s.processPendingPublications(ctx, stopCh) for { select { @@ -49,13 +55,20 @@ func (s *Scheduler) Run(ctx context.Context, stopCh <-chan struct{}) { return case <-ticker.C: - s.processPendingPublications(ctx) + s.processPendingPublications(ctx, stopCh) + case <-s.wakeCh: + s.processPendingPublications(ctx, stopCh) } } } -// processPendingPublications finds pending publications and dispatches them to workers. -func (s *Scheduler) processPendingPublications(ctx context.Context) { +// processPendingPublications dispatches every pending publication to the workers. +// +// The send blocks when the queue is full, so the queue acts as backpressure and +// a backlog larger than the queue drains at whatever rate the workers sustain. +// Dropping the overflow instead would strand those publications in the pending +// state until some later sweep, leaving idle workers alongside pending work. +func (s *Scheduler) processPendingPublications(ctx context.Context, stopCh <-chan struct{}) { logger.Debug("Processing pending publications") publications, err := s.db.GetPublicationsByStatus(routingv1.PublicationStatus_PUBLICATION_STATUS_PENDING) @@ -71,18 +84,16 @@ func (s *Scheduler) processPendingPublications(ctx context.Context) { logger.Info("Stopping publication processing due to context cancellation") return - default: - // Try to dispatch work item - select { - case s.workQueue <- publypes.WorkItem{PublicationID: publication.GetID()}: - logger.Debug("Dispatched publication to worker", "publication_id", publication.GetID()) - - // Update status to in progress - if err := s.db.UpdatePublicationStatus(publication.GetID(), routingv1.PublicationStatus_PUBLICATION_STATUS_IN_PROGRESS); err != nil { - logger.Error("Failed to update publication status", "publication_id", publication.GetID(), "error", err) - } - default: - logger.Debug("Work queue is full, skipping publication", "publication_id", publication.GetID()) + case <-stopCh: + logger.Info("Stopping publication processing due to stop signal") + + return + case s.workQueue <- publypes.WorkItem{PublicationID: publication.GetID()}: + logger.Debug("Dispatched publication to worker", "publication_id", publication.GetID()) + + // Update status to in progress + if err := s.db.UpdatePublicationStatus(publication.GetID(), routingv1.PublicationStatus_PUBLICATION_STATUS_IN_PROGRESS); err != nil { + logger.Error("Failed to update publication status", "publication_id", publication.GetID(), "error", err) } } } diff --git a/server/publication/scheduler_test.go b/server/publication/scheduler_test.go new file mode 100644 index 000000000..8758070e5 --- /dev/null +++ b/server/publication/scheduler_test.go @@ -0,0 +1,239 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package publication + +import ( + "fmt" + "slices" + "sync" + "testing" + "time" + + routingv1 "github.com/agntcy/dir/api/routing/v1" + publypes "github.com/agntcy/dir/server/publication/types" + "github.com/agntcy/dir/server/types" +) + +// neverInterval is long enough that any sweep observed during a test must have +// come from the wake channel or from the initial sweep, never from the ticker. +const neverInterval = time.Hour + +// fakeSchedulerDB serves a fixed set of pending publications and records the +// status transitions the scheduler applies. +type fakeSchedulerDB struct { + types.PublicationDatabaseAPI + + mu sync.Mutex + pending []types.PublicationObject + inFlight []string + + // swept reports each sweep, letting a test order its actions against the + // scheduler's startup sweep instead of racing it. + swept chan struct{} +} + +func (f *fakeSchedulerDB) GetPublicationsByStatus(routingv1.PublicationStatus) ([]types.PublicationObject, error) { + f.mu.Lock() + out := make([]types.PublicationObject, len(f.pending)) + copy(out, f.pending) + f.mu.Unlock() + + select { + case f.swept <- struct{}{}: + default: + } + + return out, nil +} + +func (f *fakeSchedulerDB) UpdatePublicationStatus(publicationID string, _ routingv1.PublicationStatus) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.inFlight = append(f.inFlight, publicationID) + // A dispatched publication leaves the pending set, so later sweeps do not + // redispatch it. + f.pending = slices.DeleteFunc(f.pending, func(p types.PublicationObject) bool { + return p.GetID() == publicationID + }) + + return nil +} + +func (f *fakeSchedulerDB) setPending(ids ...string) { + f.mu.Lock() + defer f.mu.Unlock() + + f.pending = nil + for _, id := range ids { + f.pending = append(f.pending, fakePublication{id: id}) + } +} + +type fakePublication struct { + types.PublicationObject + + id string +} + +func (p fakePublication) GetID() string { return p.id } + +// awaitWorkItem waits for a dispatch, failing rather than hanging if the +// scheduler never sweeps. +func awaitWorkItem(t *testing.T, queue <-chan publypes.WorkItem) publypes.WorkItem { + t.Helper() + + select { + case item := <-queue: + return item + case <-time.After(5 * time.Second): + t.Fatal("scheduler did not dispatch a publication") + + return publypes.WorkItem{} + } +} + +// awaitSweep waits for the scheduler to query for pending publications. +func awaitSweep(t *testing.T, db *fakeSchedulerDB) { + t.Helper() + + select { + case <-db.swept: + case <-time.After(5 * time.Second): + t.Fatal("scheduler did not sweep for pending publications") + } +} + +func TestWakeDispatchesWithoutWaitingForTheInterval(t *testing.T) { + t.Parallel() + + db := &fakeSchedulerDB{swept: make(chan struct{}, 16)} + queue := make(chan publypes.WorkItem, 8) + wakeCh := make(chan struct{}, 1) + + scheduler := NewScheduler(db, queue, neverInterval, wakeCh) + + stopCh := make(chan struct{}) + defer close(stopCh) + + go scheduler.Run(t.Context(), stopCh) + + // Let the startup sweep observe an empty queue first. Without this the + // publication below could be picked up by that sweep, and the test would + // pass whether or not the wake is wired up. + awaitSweep(t, db) + + // The publication now arrives while the service is running, so only a wake + // can surface it before the hour is up. + db.setPending("pub-1") + + wakeCh <- struct{}{} + + if got := awaitWorkItem(t, queue).PublicationID; got != "pub-1" { + t.Fatalf("dispatched publication = %q, want %q", got, "pub-1") + } +} + +func TestSchedulerSweepsOnStart(t *testing.T) { + t.Parallel() + + db := &fakeSchedulerDB{swept: make(chan struct{}, 16)} + db.setPending("pub-existing") + + queue := make(chan publypes.WorkItem, 8) + scheduler := NewScheduler(db, queue, neverInterval, make(chan struct{}, 1)) + + stopCh := make(chan struct{}) + defer close(stopCh) + + go scheduler.Run(t.Context(), stopCh) + + if got := awaitWorkItem(t, queue).PublicationID; got != "pub-existing" { + t.Fatalf("dispatched publication = %q, want %q", got, "pub-existing") + } +} + +func TestBacklogLargerThanTheQueueFullyDrains(t *testing.T) { + t.Parallel() + + const ( + queueSize = 4 + backlog = queueSize * 5 + ) + + db := &fakeSchedulerDB{swept: make(chan struct{}, 16)} + + ids := make([]string, 0, backlog) + for i := range backlog { + ids = append(ids, fmt.Sprintf("pub-%d", i)) + } + + db.setPending(ids...) + + queue := make(chan publypes.WorkItem, queueSize) + scheduler := NewScheduler(db, queue, neverInterval, make(chan struct{}, 1)) + + stopCh := make(chan struct{}) + defer close(stopCh) + + go scheduler.Run(t.Context(), stopCh) + + // Consume as a worker would. A scheduler that dropped the overflow would + // dispatch only queueSize items from this single sweep and then stall, + // since neither the hour-long ticker nor a wake will fire. + dispatched := make([]string, 0, backlog) + for range backlog { + dispatched = append(dispatched, awaitWorkItem(t, queue).PublicationID) + } + + if !slices.Equal(dispatched, ids) { + t.Fatalf("dispatched %d publications, want all %d in order", len(dispatched), backlog) + } +} + +func TestBacklogDispatchStopsOnShutdown(t *testing.T) { + t.Parallel() + + db := &fakeSchedulerDB{swept: make(chan struct{}, 16)} + db.setPending("pub-0", "pub-1", "pub-2") + + // An unbuffered queue with no consumer leaves the scheduler blocked mid + // dispatch, which is where shutdown has to remain responsive. + queue := make(chan publypes.WorkItem) + scheduler := NewScheduler(db, queue, neverInterval, make(chan struct{}, 1)) + + stopCh := make(chan struct{}) + done := make(chan struct{}) + + go func() { + defer close(done) + + scheduler.Run(t.Context(), stopCh) + }() + + awaitSweep(t, db) + close(stopCh) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("scheduler did not stop while blocked on a full queue") + } +} + +func TestWakeIsDroppedWhenOneIsAlreadyQueued(t *testing.T) { + t.Parallel() + + svc := &Service{wakeCh: make(chan struct{}, 1)} + + // A burst of publishes against an unstarted scheduler must not block; the + // single buffered slot absorbs them. + for range 100 { + svc.wake() + } + + if len(svc.wakeCh) != 1 { + t.Fatalf("queued wakes = %d, want 1", len(svc.wakeCh)) + } +} diff --git a/server/routing/ROUTING.md b/server/routing/ROUTING.md deleted file mode 100644 index c73bc21c4..000000000 --- a/server/routing/ROUTING.md +++ /dev/null @@ -1,677 +0,0 @@ -# Routing System Documentation - -This document provides comprehensive documentation for the routing system, including architecture, operations, and storage interactions. - -## Summary - -The routing system manages record discovery and announcement across both local storage and distributed networks using a **pull-based architecture** designed for scalability to hundreds of peers. It provides three main operations: - -- **Publish**: Announces CID availability to DHT network, triggering pull-based label discovery -- **List**: Efficiently queries local records with optional filtering (local-only) -- **Search**: Discovers remote records using OR logic with minimum threshold matching - -The system uses a **pull-based discovery architecture**: -- **OCI Storage**: Immutable record content (container images/artifacts) -- **Local KV Storage**: Fast indexing and cached remote labels (BadgerDB/In-memory) -- **DHT Storage**: Content provider announcements only (libp2p DHT) -- **RPC Layer**: On-demand content fetching for label extraction - -**Key Architectural Benefits:** -- **Scalable**: Works with hundreds of peers (not limited by DHT k-closest constraints) -- **Reliable**: Uses proven DHT provider system instead of unreliable label propagation -- **Fresh**: Labels extracted directly from content, preventing drift -- **Efficient**: Local caching for fast queries, background maintenance for staleness - ---- - -## Constants - -### Import - -```go -import "github.com/agntcy/dir/server/routing" -``` - -### Timing Constants - -```go -// DHT Record TTL (48 hours) -routing.DHTRecordTTL - -// Label Republishing Interval (36 hours) -routing.LabelRepublishInterval - -// Remote Label Cleanup Interval (48 hours) -routing.RemoteLabelCleanupInterval - -// Provider Record TTL (48 hours) -routing.ProviderRecordTTL - -// DHT Refresh Interval (30 seconds) -routing.RefreshInterval -``` - -### Protocol Constants - -```go -// Protocol prefix for DHT -routing.ProtocolPrefix // "dir" - -// Rendezvous string for peer discovery -routing.ProtocolRendezvous // "dir/connect" -``` - -### Validation Constants - -```go -// Maximum hops for distributed queries -routing.MaxHops // 20 - -// Notification channel buffer size -routing.NotificationChannelSize // 1000 - -// Minimum parts required in enhanced label keys (after string split) -routing.MinLabelKeyParts // 5 - -// Default minimum match score for OR logic (proto-compliant) -routing.DefaultMinMatchScore // 1 -``` - -### Usage Examples - -```go -// Cleanup task using consistent interval -ticker := time.NewTicker(routing.RemoteLabelCleanupInterval) -defer ticker.Stop() - -// DHT configuration with consistent TTL -dht, err := dht.New(ctx, host, - dht.MaxRecordAge(routing.DHTRecordTTL), - dht.ProtocolPrefix(protocol.ID(routing.ProtocolPrefix)), -) - -// Validate enhanced label key format -parts := strings.Split(labelKey, "/") -if len(parts) < routing.MinLabelKeyParts { - return errors.New("invalid enhanced key format: expected ////") -} -``` - ---- - -## Enhanced Key Format - -The routing system uses a self-descriptive key format that embeds all essential information directly in the key structure. - -### Key Structure - -**Format**: `////` - -**Examples**: -``` -/skills/AI/Machine Learning/baeabc123.../12D3KooWExample... -/domains/technology/web/baedef456.../12D3KooWOther... -/modules/search/semantic/baeghi789.../12D3KooWAnother... -``` - -### Benefits - -1. **πŸ“– Self-Documenting**: Keys tell the complete story at a glance -2. **⚑ Efficient Filtering**: PeerID extraction without JSON parsing -3. **🧹 Cleaner Storage**: Minimal JSON metadata (only timestamps) -4. **πŸ” Better Debugging**: Database inspection shows relationships immediately -5. **🎯 Consistent**: Same format used in local storage and DHT network - -### Utility Functions - -```go -// Build enhanced keys -key := BuildEnhancedLabelKey("/skills/AI", "CID123", "Peer1") -// β†’ "/skills/AI/CID123/Peer1" - -// Parse enhanced keys -label, cid, peerID, err := ParseEnhancedLabelKey(key) -// β†’ ("/skills/AI", "CID123", "Peer1", nil) - -// Extract components -peerID := ExtractPeerIDFromKey(key) // β†’ "Peer1" -cid := ExtractCIDFromKey(key) // β†’ "CID123" -isLocal := IsLocalKey(key, "Peer1") // β†’ true -``` - -### Storage Examples - -**Local Storage**: -``` -/records/CID123 β†’ (empty) # Local record index -/skills/AI/ML/CID123/Peer1 β†’ {"timestamp": "..."} # Enhanced label metadata -/domains/tech/CID123/Peer1 β†’ {"timestamp": "..."} # Enhanced domain metadata -``` - -**DHT Network**: -``` -/skills/AI/ML/CID123/Peer1 β†’ "CID123" # Enhanced network announcement -/domains/tech/CID123/Peer1 β†’ "CID123" # Enhanced domain announcement -``` - ---- - -## Publish - -The Publish operation announces records for discovery by storing metadata in both local storage and the distributed DHT network. - -### Flow Diagram - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ PUBLISH REQUEST β”‚ - β”‚ (gRPC Controller) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ controller.Publish() β”‚ - β”‚ β”‚ - β”‚ 1. getRecord() - Validates RecordRef β”‚ - β”‚ β”œβ”€ store.Lookup(ctx, ref) [READ: OCI Storage] β”‚ - β”‚ └─ store.Pull(ctx, ref) [READ: OCI Storage] β”‚ - β”‚ β”‚ - β”‚ 2. routing.Publish(ctx, ref, record) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ routing.Publish() β”‚ - β”‚ (Main Router) β”‚ - β”‚ β”‚ - β”‚ 1. local.Publish(ctx, ref, record) β”‚ - β”‚ 2. if hasPeersInRoutingTable(): β”‚ - β”‚ remote.Publish(ctx, ref, record) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ LOCAL PUBLISH β”‚ β”‚ - β”‚ (routing_local.go) β”‚ β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ β”‚ - β–Ό β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ LOCAL KV STORAGE β”‚ β”‚ - β”‚ (Routing Datastore) β”‚ β”‚ - β”‚ β”‚ β”‚ - β”‚ 1. loadMetrics() [READ: KV] β”‚ β”‚ - β”‚ 2. dstore.Has(recordKey) [READ: KV] β”‚ β”‚ - β”‚ 3. batch.Put(recordKey) [WRITE: KV] β”‚ β”‚ - β”‚ └─ "/records/CID123" β†’ (empty) β”‚ β”‚ - β”‚ 4. For each label: [WRITE: KV] β”‚ β”‚ - β”‚ └─ "/skills/AI/CID123/Peer1" β†’ LabelMetadata β”‚ β”‚ - β”‚ 5. metrics.update() [WRITE: KV] β”‚ β”‚ - β”‚ └─ "/metrics" β†’ JSON β”‚ β”‚ - β”‚ 6. batch.Commit() [COMMIT: KV] β”‚ β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ REMOTE PUBLISH β”‚ - β”‚ (routing_remote.go) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ DHT STORAGE β”‚ - β”‚ (Distributed Network) β”‚ - β”‚ β”‚ - β”‚ 1. DHT().Provide(CID) [WRITE: DHT] β”‚ - β”‚ └─ Announce CID to network β”‚ - β”‚ └─ Triggers pull-based label discovery β”‚ - β”‚ β”‚ - β”‚ ❌ REMOVED: Individual label announcements β”‚ - β”‚ No more DHT.PutValue() for labels β”‚ - β”‚ Labels discovered via content pulling β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Storage Operations - -**OCI Storage (Object Storage):** -- `READ`: `store.Lookup(RecordRef)` - Verify record exists -- `READ`: `store.Pull(RecordRef)` - Get full record content - -**Local KV Storage (Routing Datastore):** -- `READ`: `loadMetrics("/metrics")` - Get current metrics -- `READ`: `dstore.Has("/records/CID123")` - Check if already published -- `WRITE`: `"/records/CID123" β†’ (empty)` - Mark as local record -- `WRITE`: `"/skills/AI/ML/CID123/Peer1" β†’ LabelMetadata` - Store enhanced label metadata -- `WRITE`: `"/domains/tech/CID123/Peer1" β†’ LabelMetadata` - Store enhanced domain metadata -- `WRITE`: `"/modules/search/CID123/Peer1" β†’ LabelMetadata` - Store enhanced module metadata -- `WRITE`: `"/metrics" β†’ JSON` - Update metrics - -**DHT Storage (Distributed Network):** -- `WRITE`: `DHT().Provide(CID123)` - Announce CID provider to network -- ❌ **REMOVED**: Individual label announcements via `DHT.PutValue()` -- **Pull-Based Discovery**: Remote peers discover labels by pulling content directly - -**Remote Peer Pull-Based Flow (Triggered by CID Provider Announcements):** -- `TRIGGER`: DHT provider notification received -- `RPC`: `service.Pull(ctx, peerID, recordRef)` - Fetch content from announcing peer -- `EXTRACT`: `GetLabels(record)` - Extract all labels from content -- `CACHE`: Store enhanced keys locally: `"/skills/AI/CID123/RemotePeerID" β†’ LabelMetadata` - ---- - -## List - -The List operation efficiently queries local records with optional filtering. It's designed as a local-only operation that never accesses the network or OCI storage. - -### Flow Diagram - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ LIST REQUEST β”‚ - β”‚ (gRPC Controller) β”‚ - β”‚ + RecordQuery[] (optional) β”‚ - β”‚ + Limit (optional) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ controller.List() β”‚ - β”‚ β”‚ - β”‚ 1. routing.List(ctx, req) β”‚ - β”‚ 2. Stream ListResponse items to client β”‚ - β”‚ └─ NO OCI Storage access needed! β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ routing.List() β”‚ - β”‚ (Main Router) β”‚ - β”‚ β”‚ - β”‚ βœ… Always local-only operation β”‚ - β”‚ return local.List(ctx, req) β”‚ - β”‚ β”‚ - β”‚ ❌ NO remote.List() - Network not involved β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ LOCAL LIST ONLY β”‚ - β”‚ (routing_local.go) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ LOCAL KV STORAGE β”‚ - β”‚ (Routing Datastore) β”‚ - β”‚ β”‚ - β”‚ STEP 1: Get Local Record CIDs β”‚ - β”‚ β”œβ”€ READ: dstore.Query("/records/") [READ: KV] β”‚ - β”‚ β”‚ └─ Returns: "/records/CID123", "/records/CID456", ... β”‚ - β”‚ β”‚ └─ βœ… Pre-filtered: Only LOCAL records β”‚ - β”‚ β”‚ - β”‚ STEP 2: For Each CID, Check Query Matching β”‚ - β”‚ β”œβ”€ matchesAllQueries(cid, queries): β”‚ - β”‚ β”‚ β”‚ β”‚ - β”‚ β”‚ └─ getRecordLabelsEfficiently(cid): β”‚ - β”‚ β”‚ β”œβ”€ READ: dstore.Query("/skills/") [READ: KV] β”‚ - β”‚ β”‚ β”‚ └─ Find: "/skills/AI/ML/CID123/Peer1" β”‚ - β”‚ β”‚ β”‚ └─ Extract: "/skills/AI/ML" β”‚ - β”‚ β”‚ β”œβ”€ READ: dstore.Query("/domains/") [READ: KV] β”‚ - β”‚ β”‚ β”‚ └─ Find: "/domains/tech/CID123/Peer1" β”‚ - β”‚ β”‚ β”‚ └─ Extract: "/domains/tech" β”‚ - β”‚ β”‚ └─ READ: dstore.Query("/modules/") [READ: KV] β”‚ - β”‚ β”‚ └─ Find: "/modules/search/CID123/Peer1" β”‚ - β”‚ β”‚ └─ Extract: "/modules/search" β”‚ - β”‚ β”‚ β”‚ - β”‚ β”‚ └─ queryMatchesLabels(query, labels): β”‚ - β”‚ β”‚ └─ Check if ALL queries match labels (AND logic) β”‚ - β”‚ β”‚ β”‚ - β”‚ └─ If matches: Return {RecordRef: CID123, Labels: [...]} β”‚ - β”‚ β”‚ - β”‚ ❌ NO OCI Storage access - Labels extracted from KV keys! β”‚ - β”‚ ❌ NO DHT Storage access - Local-only operation! β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Storage Operations - -**OCI Storage (Object Storage):** -- ❌ **NO ACCESS** - List doesn't need record content! - -**Local KV Storage (Routing Datastore):** -- `READ`: `"/records/*"` - Get all local record CIDs -- `READ`: `"/skills/*"` - Extract skill labels for each CID -- `READ`: `"/domains/*"` - Extract domain labels for each CID -- `READ`: `"/modules/*"` - Extract module labels for each CID - -**DHT Storage (Distributed Network):** -- ❌ **NO ACCESS** - List is local-only operation! - -### Performance Characteristics - -**List vs Publish Storage Comparison:** -``` -PUBLISH: LIST: -β”œβ”€ OCI: 2 reads (validate) β”œβ”€ OCI: 0 reads βœ… -β”œβ”€ Local KV: 1 read + 5+ writes β”œβ”€ Local KV: 4+ reads only βœ… -└─ DHT: 0 reads + 4+ writes └─ DHT: 0 reads βœ… - -Result: List is much lighter! -``` - -**Key Optimizations:** -1. **No OCI Access**: Labels extracted from KV keys, not record content -2. **Local-Only**: No network/DHT interaction required -3. **Efficient Filtering**: Uses `/records/` index as starting point -4. **Key-Based Labels**: No expensive record parsing - -**Read Pattern**: `O(1 + 3Γ—N)` KV reads where N = number of local records - ---- - -## Search - -The Search operation discovers remote records from other peers using **pull-based label caching** and **OR logic with minimum threshold**. It's designed for network-wide discovery at scale (hundreds of peers) and filters out local records, returning only records from remote peers that match at least `minMatchScore` queries. - -### Pull-Based Discovery Flow - -``` -PHASE 1: REMOTE PEER PUBLISHES CONTENT - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Remote Peer: DHT.Provide(CID) β”‚ - β”‚ β”‚ - β”‚ 1. Remote peer publishes content β”‚ - β”‚ 2. DHT().Provide(CID) announces availability β”‚ - β”‚ 3. Provider announcement propagates to all peers β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -PHASE 2: LOCAL PEER DISCOVERS AND CACHES - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ handleCIDProviderNotification() β”‚ - β”‚ (routing_remote.go) β”‚ - β”‚ β”‚ - β”‚ 1. Receive: CID provider notification β”‚ - β”‚ 2. Check: hasRemoteRecordCached() β†’ false (new record) β”‚ - β”‚ 3. Pull: service.Pull(ctx, peerID, recordRef) β”‚ - β”‚ └─ RPC call to remote peer β”‚ - β”‚ 4. Extract: GetLabels(record) β”‚ - β”‚ └─ Parse skills, domains, modules from content β”‚ - β”‚ 5. Cache: Enhanced keys locally β”‚ - β”‚ β”œβ”€ "/skills/AI/CID123/RemotePeer" β†’ LabelMetadata β”‚ - β”‚ β”œβ”€ "/domains/research/CID123/RemotePeer" β†’ LabelMetadataβ”‚ - β”‚ └─ "/modules/runtime/CID123/RemotePeer" β†’ LabelMetadataβ”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -PHASE 3: USER SEARCHES FOR REMOTE RECORDS - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ SEARCH REQUEST β”‚ - β”‚ (gRPC Controller) β”‚ - β”‚ + RecordQuery[] (skills/domains/modules) β”‚ - β”‚ + MinMatchScore (OR logic threshold) β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ LOCAL KV STORAGE β”‚ - β”‚ (Cached Remote Labels) β”‚ - β”‚ β”‚ - β”‚ STEP 1: Query Cached Remote Labels (Pull-Based Discovery Results) β”‚ - β”‚ β”œβ”€ READ: dstore.Query("/skills/") [READ: KV] β”‚ - β”‚ β”‚ └─ Find: "/skills/AI/CID123/RemotePeer1" (cached via pull) β”‚ - β”‚ β”œβ”€ READ: dstore.Query("/domains/") [READ: KV] β”‚ - β”‚ β”‚ └─ Find: "/domains/research/CID123/RemotePeer1" (cached via pull) β”‚ - β”‚ └─ READ: dstore.Query("/modules/") [READ: KV] β”‚ - β”‚ └─ Find: "/modules/runtime/CID123/RemotePeer1" (cached via pull) β”‚ - β”‚ β”‚ - β”‚ STEP 2: Filter for REMOTE Records Only β”‚ - β”‚ β”œβ”€ ParseEnhancedLabelKey(key) β†’ (label, cid, peerID) β”‚ - β”‚ β”œβ”€ if peerID == localPeerID: continue (skip local) β”‚ - β”‚ └─ βœ… Only process records from remote peers β”‚ - β”‚ β”‚ - β”‚ STEP 3: Apply OR Logic with Minimum Threshold β”‚ - β”‚ β”œβ”€ calculateMatchScore(cid, queries, peerID): β”‚ - β”‚ β”‚ β”œβ”€ For each query: check if it matches ANY label (OR logic) β”‚ - β”‚ β”‚ β”œβ”€ Count matching queries β†’ score β”‚ - β”‚ β”‚ └─ Return: (matchingQueries[], score) β”‚ - β”‚ β”œβ”€ if score >= minMatchScore: include result βœ… β”‚ - β”‚ β”‚ └─ Records returned if they match β‰₯N queries (OR relationship) β”‚ - β”‚ β”œβ”€ Apply deduplicateQueries() for consistent scoring β”‚ - β”‚ └─ Apply limit and duplicate CID filtering β”‚ - β”‚ β”‚ - β”‚ STEP 4: Return SearchResponse with Match Details β”‚ - β”‚ └─ {RecordRef: CID, Peer: RemotePeer, MatchQueries: [...], MatchScore: N} β”‚ - β”‚ β”‚ - β”‚ βœ… Uses cached labels from pull-based discovery β”‚ - β”‚ βœ… Fresh data (labels extracted directly from content) β”‚ - β”‚ ❌ NO DHT label queries - Uses local cache only β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Storage Operations - -**Pull-Based Label Discovery (Background Process):** -- `RPC`: `service.Pull(ctx, remotePeerID, recordRef)` - Fetch content from remote peer -- `EXTRACT`: `GetLabels(record)` - Extract skills/domains/modules from content -- `CACHE`: Store enhanced keys locally for fast search - -**Search Query Execution (User Request):** - -**OCI Storage (Object Storage):** -- ❌ **NO ACCESS** - Search uses cached labels, not record content - -**Local KV Storage (Routing Datastore):** -- `READ`: `"/skills/*"` - Query cached remote skill labels (via pull-based discovery) -- `READ`: `"/domains/*"` - Query cached remote domain labels (via pull-based discovery) -- `READ`: `"/modules/*"` - Query cached remote module labels (via pull-based discovery) -- **Filter**: Only process keys where `peerID != localPeerID` - -**DHT Storage (Distributed Network):** -- ❌ **NO DIRECT ACCESS** - Search uses locally cached data from pull-based discovery - -**RPC Layer (Pull-Based Discovery):** -- `service.Pull(remotePeerID, recordRef)` - On-demand content fetching for new providers -- `service.Lookup(remotePeerID, recordRef)` - Metadata validation for announced content - -### Search vs List Comparison - -| Aspect | **List** | **Search** | -|--------|----------|------------| -| **Scope** | Local records only | Remote records only | -| **Data Source** | `/records/` index | Cached remote labels (pull-based) | -| **Filtering** | `peerID == localPeerID` | `peerID != localPeerID` | -| **Query Logic** | βœ… AND relationship (all must match) | βœ… OR relationship with minMatchScore threshold | -| **Discovery Method** | Direct local storage | Pull-based caching from DHT provider events | -| **Network Access** | ❌ None | βœ… RPC content pulling (background) | -| **Scalability** | Single peer | Hundreds of peers via pull-based discovery | -| **Response Type** | `ListResponse` | `SearchResponse` | -| **Additional Fields** | Labels only | + Peer info, match score, matching queries | -| **Content Freshness** | Always current | Fresh via on-demand content pulling | - -### Performance Characteristics - -**Pull-Based Discovery Performance:** -``` -BACKGROUND LABEL CACHING (per new CID provider announcement): -β”œβ”€ RPC: 1 content pull from remote peer βœ… (only for new records) -β”œβ”€ Local Processing: Label extraction from content βœ… -β”œβ”€ Local KV: N writes (N = number of labels) βœ… -└─ Result: Fresh labels cached locally βœ… - -SEARCH EXECUTION (per user query): -β”œβ”€ Local KV: 3+ reads (cached remote labels) βœ… -β”œβ”€ Query deduplication and OR logic processing βœ… -β”œβ”€ No network access needed βœ… (uses cache) -└─ Result: Fast search with fresh data βœ… -``` - -**Key Optimizations:** -1. **Scalable Caching**: Pull-based discovery works with hundreds of peers -2. **Fresh Content**: Labels extracted directly from source content -3. **Efficient Search**: Query cached labels, no real-time network access -4. **Content Validation**: RPC calls validate remote peer availability -5. **Background Processing**: Label discovery doesn't block user queries -6. **Query Deduplication**: Server-side defense against client bugs -7. **OR Logic Scoring**: Flexible matching with minimum threshold - -**Read Pattern**: -- **Discovery**: `O(1)` RPC call per new remote record -- **Search**: `O(4Γ—M)` KV reads where M = number of cached remote labels (skills, domains, modules, locators) - -### OR Logic with Minimum Threshold - -**Core Concept:** -The Search API uses **OR logic** where records are returned if they match **at least N queries** (where N = `minMatchScore`). This provides flexible, scored matching for complex search scenarios. - -**Match Scoring Algorithm:** -```go -score := 0 -for each query in searchQueries { - if QueryMatchesLabels(query, recordLabels) { - score++ // OR logic: any match increments score - } -} -return score >= minMatchScore // Threshold filtering -``` - -**Production Safety:** -- **Default Behavior**: `minMatchScore = 0` defaults to `1` per proto specification -- **Empty Queries**: Rejected with helpful error (prevents expensive full scans) -- **Query Deduplication**: Server-side deduplication ensures consistent scoring - -### Query Types and Matching - -**Supported Query Types:** -1. **SKILL** (`RECORD_QUERY_TYPE_SKILL`) -2. **LOCATOR** (`RECORD_QUERY_TYPE_LOCATOR`) -3. **DOMAIN** (`RECORD_QUERY_TYPE_DOMAIN`) -4. **MODULE** (`RECORD_QUERY_TYPE_MODULE`) - -**Matching Rules:** - -**Skills & Domains & Modules (Hierarchical Matching):** -``` -Query: "AI" matches: -βœ… /skills/AI (exact match) -βœ… /skills/AI/ML (prefix match) -βœ… /skills/AI/NLP/ChatBot (prefix match) -❌ /skills/Machine Learning (no match) -``` - -**Locators (Exact Matching Only):** -``` -Query: "docker-image" matches: -βœ… /locators/docker-image (exact match only) -❌ /locators/docker-image/latest (no prefix matching) -``` - -### OR Logic Examples - -**Example 1: Flexible Matching** -```bash -# Query: Find records with AI OR Python skills, need at least 1 match -dirctl routing search --skill "AI" --skill "Python" --min-score 1 - -# Results: -# Record A: [AI] β†’ Score: 1/2 β†’ βœ… Returned (β‰₯ minScore=1) -# Record B: [Python] β†’ Score: 1/2 β†’ βœ… Returned (β‰₯ minScore=1) -# Record C: [AI, Python] β†’ Score: 2/2 β†’ βœ… Returned (β‰₯ minScore=1) -# Record D: [Java] β†’ Score: 0/2 β†’ ❌ Filtered out (< minScore=1) -``` - -**Example 2: Strict Matching** -```bash -# Query: Find records with BOTH AI AND Python skills -dirctl routing search --skill "AI" --skill "Python" --min-score 2 - -# Results: -# Record A: [AI] β†’ Score: 1/2 β†’ ❌ Filtered out (< minScore=2) -# Record B: [Python] β†’ Score: 1/2 β†’ ❌ Filtered out (< minScore=2) -# Record C: [AI, Python] β†’ Score: 2/2 β†’ βœ… Returned (β‰₯ minScore=2) -``` - -**Example 3: Mixed Query Types** -```bash -# Query: Multi-type search with threshold -dirctl routing search \ - --skill "AI" \ - --domain "research" \ - --module "runtime/python" \ - --min-score 2 - -# Results: -# Record A: [skills/AI, domains/research] β†’ Score: 2/3 β†’ βœ… Returned -# Record B: [skills/AI] β†’ Score: 1/3 β†’ ❌ Filtered out -# Record C: [domains/research, modules/runtime/python] β†’ Score: 2/3 β†’ βœ… Returned -``` - -### Pull-Based Discovery Benefits - -**Scalability:** -- **Not limited by DHT k-closest peers** (typically ~20) -- **Provider announcements reach all peers** via DHT.Provide() -- **On-demand content pulling** scales to hundreds of peers - -**Reliability:** -- **Uses working DHT components** (provider system, not broken label propagation) -- **Direct content fetching** bypasses DHT propagation issues -- **Fresh labels** always match actual content - -**Performance:** -- **Background caching** doesn't block user queries -- **Local cache queries** are fast (no network access during search) -- **Automatic cache management** via background tasks - ---- - -## Pull-Based Architecture Summary - -### Key Architectural Changes - -**Previous Architecture (Removed):** -- ❌ DHT.PutValue() for individual label announcements -- ❌ handleLabelNotification() event system -- ❌ Complex announcement type routing (CID vs Label) -- ❌ Limited by DHT k-closest peer constraints (~20 peers) - -**New Pull-Based Architecture:** -- βœ… DHT.Provide() for CID provider announcements only -- βœ… handleCIDProviderNotification() with content pulling -- βœ… Unified announcement handling (all are CID provider events) -- βœ… Scalable to hundreds of peers via RPC content fetching - -### Production Benefits - -**Scalability:** -- **Large Networks**: Not constrained by DHT k-closest limitations -- **Efficient Discovery**: Provider announcements reach all peers reliably -- **On-Demand Fetching**: Only pull content when discovery happens - -**Reliability:** -- **Proven Components**: Uses working DHT provider system -- **Fresh Data**: Labels extracted directly from content source -- **Self-Healing**: Failed pulls don't break the system - -**Performance:** -- **Fast Queries**: Local cache provides sub-millisecond search -- **Background Processing**: Label discovery doesn't block user operations -- **Automatic Maintenance**: Background republishing and cleanup - -**API Robustness:** -- **Query Deduplication**: Server defends against client bugs -- **Production Safety**: Proper defaults and validation -- **Complete Query Support**: Skills, locators, domains, modules all supported -- **OR Logic**: Flexible matching with minimum threshold control - -### Migration Notes - -**No Breaking Changes:** -- **API Interface**: Search/List APIs unchanged for existing clients -- **Enhanced Key Format**: Unchanged, maintains compatibility -- **Background Tasks**: Adapted for provider republishing, not removed - -**Improved Behavior:** -- **More Reliable**: Pull-based discovery vs unreliable label propagation -- **Better Scaling**: Hundreds of peers vs ~20 peer DHT limitation -- **Fresher Data**: Labels from content vs potentially stale DHT cache -- **OR Logic**: Proto-compliant search behavior with flexible matching diff --git a/server/routing/advertise.go b/server/routing/advertise.go new file mode 100644 index 000000000..a22703908 --- /dev/null +++ b/server/routing/advertise.go @@ -0,0 +1,248 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/agntcy/dir/server/types" + "github.com/ipfs/go-cid" +) + +// startAdvertiseTask advertises everything this node has published, once at +// startup and then on every reprovide tick. +// +// The startup pass is what makes a restart cheap: provider records expire, and +// a ticker does not fire when it is created, so without it a restarted node +// would be invisible until the first interval elapsed. +func (r *routeRemote) startAdvertiseTask() { + r.wg.Go(func() { + if !r.waitForPeers(r.ctx) { + return + } + + r.advertisePublishedRecords(r.ctx) + + ticker := time.NewTicker(r.reprovideInterval) + defer ticker.Stop() + + remoteLogger.Info("Started reprovide task", "interval", r.reprovideInterval) + + for { + select { + case <-r.ctx.Done(): + remoteLogger.Debug("Stopping reprovide task") + + return + case <-ticker.C: + r.advertisePublishedRecords(r.ctx) + } + } + }) +} + +// waitForPeers blocks until the DHT has someone to advertise to, reporting +// false if the node shut down first. +// +// Provide against an empty routing table fails outright, so there is nothing to +// gain from starting earlier. A lone bootstrap node waits here until its first +// peer arrives, which is exactly when its records become worth announcing. +func (r *routeRemote) waitForPeers(ctx context.Context) bool { + if r.server.DHT().RoutingTable().Size() > 0 { + return true + } + + remoteLogger.Info("Waiting for a DHT peer before advertising held records") + + ticker := time.NewTicker(advertisePollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return false + case <-ticker.C: + if r.server.DHT().RoutingTable().Size() > 0 { + return true + } + } + } +} + +// advertisePublishedRecords announces every record this node has published: +// each CID so the record can be fetched, and the labels, with their ancestors, +// so it can be found. +// +// Labels are advertised as one distinct set rather than per record. Records +// share ancestors heavily β€” every skill under "/skills/A" reprovides that key β€” +// so deduplicating first is the difference between one Provide per label and +// one per record that carries it. +func (r *routeRemote) advertisePublishedRecords(ctx context.Context) { + started := time.Now() + + cids, labels, err := r.publishedRecords(ctx) + if err != nil { + remoteLogger.Error("Failed to enumerate published records for advertising", "error", err) + + return + } + + if len(cids) == 0 { + remoteLogger.Debug("Nothing published to advertise") + + return + } + + keys := make([]cid.Cid, 0, len(cids)) + + for _, cidStr := range cids { + decoded, err := cid.Decode(cidStr) + if err != nil { + remoteLogger.Warn("Skipping held record with an undecodable CID", "cid", cidStr, "error", err) + + continue + } + + keys = append(keys, decoded) + } + + expanded := expandLabels(labels) + + failedCIDs := r.provideKeys(ctx, keys) + failedLabels := r.provideLabels(ctx, expanded) + + remoteLogger.Info("Advertised published records", + "records", len(cids), + "labelKeys", len(expanded), + "failedRecords", failedCIDs, + "failedLabels", failedLabels, + "took", time.Since(started)) +} + +// publishedRecords returns the CIDs of every published record and the labels +// carried by them, paging so a large corpus is never loaded whole. +// +// Records this node merely holds are excluded: they stay servable to anyone who +// knows the CID, but nothing announces them. Unpublishing works the same way, +// by omission rather than withdrawal, because Kademlia has no retraction β€” the +// provider records already out there are left to expire. +func (r *routeRemote) publishedRecords(ctx context.Context) ([]string, []types.Label, error) { + if r.db == nil { + return nil, nil, errNoDatabase + } + + var ( + allCIDs []string + allLabels []types.Label + ) + + for offset := 0; ; offset += advertisePageSize { + if err := ctx.Err(); err != nil { + return nil, nil, err //nolint:wrapcheck + } + + cids, err := r.db.GetRecordCIDs( + types.WithPublished(true), + types.WithLimit(advertisePageSize), + types.WithOffset(offset), + ) + if err != nil { + return nil, nil, err //nolint:wrapcheck + } + + if len(cids) == 0 { + break + } + + labels, err := r.db.GetRecordLabels(cids) + if err != nil { + return nil, nil, err //nolint:wrapcheck + } + + allCIDs = append(allCIDs, cids...) + + for _, recordLabels := range labels { + allLabels = append(allLabels, recordLabels...) + } + + if len(cids) < advertisePageSize { + break + } + } + + return allCIDs, allLabels, nil +} + +// provideLabels advertises each label as a DHT key and returns how many failed. +// +// Failures are counted rather than returned. A record stays reachable to +// anyone who already knows its CID, and the reprovide cycle retries, so +// aborting over one unreachable custodian would be worse than a partially +// advertised record. +func (r *routeRemote) provideLabels(ctx context.Context, labels []types.Label) int { + keys := make([]cid.Cid, 0, len(labels)) + skipped := 0 + + for _, label := range labels { + key, err := labelKey(label) + if err != nil { + remoteLogger.Warn("Skipping label with no derivable DHT key", "label", label, "error", err) + + skipped++ + + continue + } + + keys = append(keys, key) + } + + return skipped + r.provideKeys(ctx, keys) +} + +// provideKeys advertises keys to the DHT concurrently and returns how many +// failed. +// +// Each Provide is a full Kademlia lookup followed by K AddProvider sends, so +// they run in parallel; done one at a time, a node holding a few hundred +// records would take hours to announce itself. +func (r *routeRemote) provideKeys(ctx context.Context, keys []cid.Cid) int { + if len(keys) == 0 { + return 0 + } + + var ( + wg sync.WaitGroup + failed atomic.Int64 + ) + + pending := make(chan cid.Cid) + + for range min(advertiseConcurrency, len(keys)) { + wg.Go(func() { + for key := range pending { + if err := r.server.DHT().Provide(ctx, key, true); err != nil { + remoteLogger.Warn("Failed to announce key", "key", key, "error", err) + failed.Add(1) + } + } + }) + } + +feed: + for _, key := range keys { + select { + case pending <- key: + case <-ctx.Done(): + break feed + } + } + + close(pending) + wg.Wait() + + return int(failed.Load()) +} diff --git a/server/routing/advertise_test.go b/server/routing/advertise_test.go new file mode 100644 index 000000000..20637d621 --- /dev/null +++ b/server/routing/advertise_test.go @@ -0,0 +1,239 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "fmt" + "testing" + "time" + + typesv1alpha1 "buf.build/gen/go/agntcy/oasf/protocolbuffers/go/agntcy/oasf/types/v1alpha1" + corev1 "github.com/agntcy/dir/api/core/v1" + routingv1 "github.com/agntcy/dir/api/routing/v1" + "github.com/agntcy/dir/server/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// pagedRecordsDB honours limit and offset the way the SQL index does, and +// records the pages it was asked for. +type pagedRecordsDB struct { + types.DatabaseAPI + + cids []string + labels map[string][]types.Label + + pages []int +} + +func (p *pagedRecordsDB) GetRecordCIDs(opts ...types.FilterOption) ([]string, error) { + config := &types.RecordFilters{} + for _, opt := range opts { + opt(config) + } + + p.pages = append(p.pages, config.Offset) + + if config.Offset >= len(p.cids) { + return nil, nil + } + + end := min(config.Offset+config.Limit, len(p.cids)) + + return p.cids[config.Offset:end], nil +} + +func (p *pagedRecordsDB) GetRecordLabels(cids []string) (map[string][]types.Label, error) { + labels := make(map[string][]types.Label, len(cids)) + for _, cid := range cids { + labels[cid] = p.labels[cid] + } + + return labels, nil +} + +func TestHeldRecordsPagesThroughTheIndex(t *testing.T) { + // Two and a bit pages, so the walk has to survive a full page, a partial + // one, and the boundary between them. + total := advertisePageSize*2 + 7 + + db := &pagedRecordsDB{labels: make(map[string][]types.Label, total)} + for i := range total { + cid := fmt.Sprintf("record-%04d", i) + db.cids = append(db.cids, cid) + db.labels[cid] = []types.Label{types.Label(fmt.Sprintf("/skills/AI/skill-%04d", i))} + } + + r := &routeRemote{db: db} + + cids, labels, err := r.publishedRecords(t.Context()) + require.NoError(t, err) + + assert.Equal(t, db.cids, cids) + assert.Len(t, labels, total) + assert.Equal(t, []int{0, advertisePageSize, advertisePageSize * 2}, db.pages) +} + +func TestHeldRecordsStopsOnAShortPage(t *testing.T) { + db := &pagedRecordsDB{ + cids: []string{"record-a", "record-b"}, + labels: map[string][]types.Label{"record-a": {"/skills/AI"}, "record-b": nil}, + } + + r := &routeRemote{db: db} + + cids, labels, err := r.publishedRecords(t.Context()) + require.NoError(t, err) + + assert.Equal(t, db.cids, cids) + assert.Equal(t, []types.Label{"/skills/AI"}, labels) + assert.Equal(t, []int{0}, db.pages, "a page shorter than the limit is the last one") +} + +func TestHeldRecordsWithoutADatabase(t *testing.T) { + r := &routeRemote{} + + _, _, err := r.publishedRecords(t.Context()) + + require.ErrorIs(t, err, errNoDatabase) +} + +// TestHeldButUnpublishedIsNotAdvertised is the privacy guarantee: pushing a +// record makes it servable to whoever knows the CID, and nothing more. Nobody +// learns it exists until it is published. +func TestHeldButUnpublishedIsNotAdvertised(t *testing.T) { + record := corev1.New(&typesv1alpha1.Record{ + Name: "private", + SchemaVersion: "0.7.0", + Skills: []*typesv1alpha1.Skill{{Name: "AI/ML"}}, + }) + + adapter, err := record.Decode() + require.NoError(t, err) + + db := newTestDatabase(t) + node := newTestServer(t, t.Context(), nil, db) + + // Indexing is what a push leaves behind. No Publish call follows. + require.NoError(t, db.AddRecord(adapter)) + + cids, labels, err := node.remote.publishedRecords(t.Context()) + require.NoError(t, err) + + assert.Empty(t, cids, "a held record must not be advertised") + assert.Empty(t, labels) + + responses, err := node.List(t.Context(), &routingv1.ListRequest{}) + require.NoError(t, err) + + var listed []string + for response := range responses { + listed = append(listed, response.GetRecordRef().GetCid()) + } + + assert.Empty(t, listed, "List reports what is published, not what is held") +} + +// TestUnpublishSurvivesReadvertising is the point of the published flag. Since +// Kademlia cannot retract a provider record, the only durable effect Unpublish +// can have is to drop the record from every later cycle β€” including the one a +// restart triggers. +func TestUnpublishSurvivesReadvertising(t *testing.T) { + kept := corev1.New(&typesv1alpha1.Record{ + Name: "kept", + SchemaVersion: "0.7.0", + Skills: []*typesv1alpha1.Skill{{Name: "AI/ML"}}, + }) + withdrawn := corev1.New(&typesv1alpha1.Record{ + Name: "withdrawn", + SchemaVersion: "0.7.0", + Skills: []*typesv1alpha1.Skill{{Name: "AI/NLP"}}, + }) + + db := newTestDatabase(t) + node := newTestServer(t, t.Context(), nil, db) + + publishRecord(t, node, db, kept) + publishRecord(t, node, db, withdrawn) + + adapter, err := withdrawn.Decode() + require.NoError(t, err) + require.NoError(t, node.Unpublish(t.Context(), adapter)) + + t.Run("the withdrawn record leaves the published set", func(t *testing.T) { + cids, labels, err := node.remote.publishedRecords(t.Context()) + require.NoError(t, err) + + assert.Equal(t, []string{kept.GetCid()}, cids) + assert.Equal(t, []types.Label{"/skills/AI/ML"}, labels) + }) + + t.Run("its labels stop being announced without a refcount", func(t *testing.T) { + _, labels, err := node.remote.publishedRecords(t.Context()) + require.NoError(t, err) + + // "/skills/AI" is still there because kept carries it; only the branch + // unique to the withdrawn record disappears. + expanded := expandLabels(labels) + assert.Contains(t, expanded, types.Label("/skills/AI")) + assert.NotContains(t, expanded, types.Label("/skills/AI/NLP")) + }) + + t.Run("List stops reporting it", func(t *testing.T) { + responses, err := node.List(t.Context(), &routingv1.ListRequest{}) + require.NoError(t, err) + + var cids []string + for response := range responses { + cids = append(cids, response.GetRecordRef().GetCid()) + } + + assert.Equal(t, []string{kept.GetCid()}, cids) + }) + + t.Run("publishing again restores it", func(t *testing.T) { + require.NoError(t, node.Publish(t.Context(), adapter)) + + cids, _, err := node.remote.publishedRecords(t.Context()) + require.NoError(t, err) + + assert.ElementsMatch(t, []string{kept.GetCid(), withdrawn.GetCid()}, cids) + }) +} + +// TestAdvertiseOnStartup is the restart case: a node that published a record in +// an earlier lifetime has to re-announce it on boot, because provider records +// expire and the reprovide ticker does not fire when it is created. +func TestAdvertiseOnStartup(t *testing.T) { + record := corev1.New(&typesv1alpha1.Record{ + Name: "startup-agent", + SchemaVersion: "0.7.0", + Skills: []*typesv1alpha1.Skill{{Name: "AI/ML"}}, + }) + + adapter, err := record.Decode() + require.NoError(t, err) + + // State a previous run would have left behind: indexed and published. The + // node starts after this, so only the startup pass can advertise it. + db := newTestDatabase(t) + require.NoError(t, db.AddRecord(adapter)) + require.NoError(t, db.SetRecordPublished(record.GetCid(), true)) + + holder := newTestServer(t, t.Context(), nil, db) + searcher := newTestServer(t, t.Context(), holder.remote.server.P2pAddrs(), nil) + + <-holder.remote.server.DHT().RefreshRoutingTable() + <-searcher.remote.server.DHT().RefreshRoutingTable() + + // The holder waits for its first peer before advertising, so how long this + // takes depends on when the searcher shows up in its routing table. + require.Eventually(t, func() bool { + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{skillQuery("AI")}, + }) + + return len(responses) == 1 && responses[0].GetRecordRef().GetCid() == record.GetCid() + }, 60*time.Second, 2*time.Second, "startup advertisement never reached the searcher") +} diff --git a/server/routing/autosync/autosync.go b/server/routing/autosync/autosync.go deleted file mode 100644 index fbd7e23b8..000000000 --- a/server/routing/autosync/autosync.go +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -// Package autosync implements DHT-based record synchronization: it pulls records -// (and their referrers) announced by trusted peers over libp2p and ingests them -// locally with full parity to a normal push. -// -// Trust model (zero-trust): the allow-set is matched against the authenticated -// peer.ID (never a payload field), and all pulled content is verified before -// ingest β€” record CID integrity + OASF schema validation, and per-referrer -// belongs-to-record + type allow-list checks. Any failure is fail-closed -// (the offending item is skipped, never partially trusted). -package autosync - -import ( - "context" - "fmt" - "sync" - "time" - - corev1 "github.com/agntcy/dir/api/core/v1" - "github.com/agntcy/dir/server/ingest" - "github.com/agntcy/dir/server/routing/internal/p2p" - "github.com/agntcy/dir/server/routing/rpc" - "github.com/agntcy/dir/server/types" - "github.com/agntcy/dir/utils/logging" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/peerstore" - ma "github.com/multiformats/go-multiaddr" -) - -var logger = logging.Logger("routing/autosync") - -// Worker pool configuration. -const ( - // workerCount is the number of concurrent workers pulling+ingesting records. - workerCount = 4 - - // queueSize bounds the pending autosync jobs. When full, new announcements - // are dropped (a later re-announcement/republish re-triggers), so autosync - // never blocks the DHT notification handler. - queueSize = 256 - - // jobTimeout bounds a single record's pull + ingest (record + referrers). - jobTimeout = 60 * time.Second - - // maxPullAttempts is the total number of record-pull attempts (initial + - // retries) before giving up. A later re-announcement re-triggers autosync. - maxPullAttempts = 3 - - // pullBackoffBase / pullBackoffMax bound the exponential backoff between - // pull attempts (a NAT'd peer may need a hole-punch/relay handshake to - // settle before the next try). - pullBackoffBase = 1 * time.Second - pullBackoffMax = 8 * time.Second - - // pullAttemptTimeout bounds a single pull attempt so one slow attempt does - // not consume the whole job budget. - pullAttemptTimeout = 15 * time.Second - - // firstRetryAttempt is the (1-based) attempt number of the first retry, used - // to compute the exponential backoff shift. - firstRetryAttempt = 2 -) - -// allowedReferrerTypes is the deny-by-default set of referrer types the autosync -// worker will pull and ingest from a peer. Anything else is rejected. -var allowedReferrerTypes = map[string]struct{}{ - corev1.SignatureReferrerType: {}, - corev1.PublicKeyReferrerType: {}, - corev1.ScanReportReferrerType: {}, -} - -// job is a unit of work: pull+ingest the announced record from a peer. -type job struct { - ref *corev1.RecordRef - peer peer.AddrInfo -} - -// recordFetcher is the subset of the libp2p RPC service the autosync worker -// needs. It is satisfied by *rpc.Service and mocked in tests. -type recordFetcher interface { - Pull(ctx context.Context, peerID peer.ID, ref *corev1.RecordRef) (*corev1.Record, error) - ListReferrers(ctx context.Context, peerID peer.ID, ref *corev1.RecordRef) ([]rpc.ReferrerDescriptor, error) - PullReferrer(ctx context.Context, peerID peer.ID, ref *corev1.RecordRef, desc rpc.ReferrerDescriptor) (*corev1.RecordReferrer, error) -} - -// peerRouter abstracts the reachability operations needed to dial a source peer -// (peerstore address hints + DHT re-resolution). Satisfied by serverPeerRouter. -type peerRouter interface { - AddAddrs(peerID peer.ID, addrs []ma.Multiaddr) - FindPeer(ctx context.Context, peerID peer.ID) (peer.AddrInfo, error) -} - -// serverPeerRouter adapts *p2p.Server to peerRouter. -type serverPeerRouter struct { - server *p2p.Server -} - -func (s serverPeerRouter) AddAddrs(peerID peer.ID, addrs []ma.Multiaddr) { - s.server.Host().Peerstore().AddAddrs(peerID, addrs, peerstore.TempAddrTTL) -} - -func (s serverPeerRouter) FindPeer(ctx context.Context, peerID peer.ID) (peer.AddrInfo, error) { - return s.server.DHT().FindPeer(ctx, peerID) //nolint:wrapcheck -} - -// Manager pulls records (and their referrers) announced by trusted peers over -// libp2p and ingests them locally with full parity to a normal push. -type Manager struct { - allowSet map[peer.ID]struct{} - transport recordFetcher - router peerRouter - ingestor ingest.Ingestor - store types.StoreAPI - validator corev1.Validator - - queue chan job - inFlight map[string]struct{} - mu sync.Mutex -} - -// NewManager creates an autosync Manager wired to the live libp2p RPC service -// and p2p server. -func NewManager( - allowSet map[peer.ID]struct{}, - service *rpc.Service, - server *p2p.Server, - ingestor ingest.Ingestor, - store types.StoreAPI, - validator corev1.Validator, -) *Manager { - return newManager(allowSet, service, serverPeerRouter{server: server}, ingestor, store, validator) -} - -// newManager is the interface-based constructor used by NewManager and tests. -func newManager( - allowSet map[peer.ID]struct{}, - transport recordFetcher, - router peerRouter, - ingestor ingest.Ingestor, - store types.StoreAPI, - validator corev1.Validator, -) *Manager { - return &Manager{ - allowSet: allowSet, - transport: transport, - router: router, - ingestor: ingestor, - store: store, - validator: validator, - queue: make(chan job, queueSize), - inFlight: make(map[string]struct{}), - } -} - -// Start launches the bounded worker pool. Workers stop when ctx is cancelled. -func (m *Manager) Start(ctx context.Context, wg *sync.WaitGroup) { - for range workerCount { - wg.Go(func() { - m.worker(ctx) - }) - } - - logger.Info("Autosync workers started", - "workers", workerCount, - "trusted_peers", len(m.allowSet), - ) -} - -// MaybeEnqueue schedules an autosync job if the announcing peer is trusted and -// the CID is not already in flight. Non-blocking: if the queue is full the -// announcement is dropped (a later re-announcement re-triggers), so the DHT -// notification handler is never blocked. -// -// addrInfo is the announcing peer's authenticated identity + addresses from the -// DHT provider record. -func (m *Manager) MaybeEnqueue(ref *corev1.RecordRef, addrInfo peer.AddrInfo) { - if ref == nil { - return - } - - // Authorization: only sync from peers on the allow-set, matched against the - // authenticated peer.ID provided by the DHT provider record. - if _, ok := m.allowSet[addrInfo.ID]; !ok { - return - } - - cid := ref.GetCid() - if cid == "" { - return - } - - // Dedupe: skip if this CID is already queued or being processed. - m.mu.Lock() - if _, ok := m.inFlight[cid]; ok { - m.mu.Unlock() - - return - } - - m.inFlight[cid] = struct{}{} - m.mu.Unlock() - - select { - case m.queue <- job{ref: ref, peer: addrInfo}: - logger.Debug("Autosync job enqueued", "cid", cid, "peer", addrInfo.ID) - default: - // Queue full: release the in-flight marker and drop. - m.clearInFlight(cid) - logger.Warn("Autosync queue full, dropping announcement", "cid", cid, "peer", addrInfo.ID) - } -} - -func (m *Manager) clearInFlight(cid string) { - m.mu.Lock() - delete(m.inFlight, cid) - m.mu.Unlock() -} - -func (m *Manager) worker(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case j := <-m.queue: - m.process(ctx, j) - m.clearInFlight(j.ref.GetCid()) - } - } -} - -// process pulls, verifies, and ingests a single announced record + referrers. -// It is fail-closed: any verification failure aborts ingestion of the offending -// item. -func (m *Manager) process(parentCtx context.Context, j job) { - cid := j.ref.GetCid() - peerID := j.peer.ID - - // Dedupe against local storage: if we already have the record, skip the pull. - if _, err := m.store.Lookup(parentCtx, j.ref); err == nil { - logger.Debug("Record already present locally, skipping autosync", "cid", cid, "peer", peerID) - - return - } - - // Bound the whole job so a slow/unreachable peer cannot tie up the worker. - ctx, cancel := context.WithTimeout(parentCtx, jobTimeout) - defer cancel() - - record, err := m.pullRecord(ctx, peerID, j.ref, j.peer.Addrs) - if err != nil { - logger.Error("Autosync failed to pull record", "cid", cid, "peer", peerID, "error", err) - - return - } - - // Integrity: the pulled content must hash to the announced CID. This prevents - // a peer from substituting different content for an announced CID. - if actual := record.GetCid(); actual != cid { - logger.Warn("Autosync rejected record: CID mismatch", - "announced_cid", cid, "actual_cid", actual, "peer", peerID) - - return - } - - // Validity: run the same OASF schema gate as a normal push before ingest. - valid, validationErrors, err := record.ValidateWith(ctx, m.validator) - if err != nil { - logger.Error("Autosync record validation error", "cid", cid, "peer", peerID, "error", err) - - return - } - - if !valid { - logger.Warn("Autosync rejected record: OASF validation failed", - "cid", cid, "peer", peerID, "errors", validationErrors) - - return - } - - if _, err := m.ingestor.ImportRecord(ctx, record); err != nil { - logger.Error("Autosync failed to ingest record", "cid", cid, "peer", peerID, "error", err) - - return - } - - logger.Info("Autosync ingested record", "cid", cid, "peer", peerID) - - m.syncReferrers(ctx, peerID, j.ref) -} - -// pullRecord pulls the record over libp2p RPC with bounded retries. Addresses -// from the announcement are added to the peerstore first. Between attempts it -// backs off (exponential + jitter) and re-resolves the peer's current addresses -// via the DHT (FindPeer) β€” recovering from stale addresses and giving NAT'd -// peers time for a hole-punch/relay handshake to settle. -func (m *Manager) pullRecord(ctx context.Context, peerID peer.ID, ref *corev1.RecordRef, addrs []ma.Multiaddr) (*corev1.Record, error) { - if len(addrs) > 0 { - m.router.AddAddrs(peerID, addrs) - } else if addrInfo, err := m.router.FindPeer(ctx, peerID); err == nil && len(addrInfo.Addrs) > 0 { - // No addresses from the announcement (e.g. GossipSub-triggered): resolve - // the peer's current addresses via the DHT before the first attempt. - m.router.AddAddrs(peerID, addrInfo.Addrs) - } - - var lastErr error - - for attempt := 1; attempt <= maxPullAttempts; attempt++ { - //nolint:nestif // retry prep (backoff + FindPeer re-resolution) reads clearly inline - if attempt > 1 { - // Back off before retrying, respecting the parent context. - if err := sleepWithBackoff(ctx, attempt); err != nil { - return nil, err - } - - // Re-resolve the peer's current addresses via the DHT. - if addrInfo, err := m.router.FindPeer(ctx, peerID); err == nil { - if len(addrInfo.Addrs) > 0 { - m.router.AddAddrs(peerID, addrInfo.Addrs) - } - } else { - logger.Debug("Autosync FindPeer re-resolution failed", - "peer", peerID, "attempt", attempt, "error", err) - } - } - - attemptCtx, cancel := context.WithTimeout(ctx, pullAttemptTimeout) - record, err := m.transport.Pull(attemptCtx, peerID, ref) - - cancel() - - if err == nil { - return record, nil - } - - lastErr = err - - logger.Debug("Autosync pull attempt failed", - "peer", peerID, "attempt", attempt, "max_attempts", maxPullAttempts, "error", err) - } - - return nil, fmt.Errorf("failed to pull record from peer %s after %d attempts: %w", peerID, maxPullAttempts, lastErr) -} - -// sleepWithBackoff waits for the backoff duration of the given attempt, or -// returns early if the context is cancelled. -func sleepWithBackoff(ctx context.Context, attempt int) error { - timer := time.NewTimer(computeBackoff(attempt)) - defer timer.Stop() - - select { - case <-ctx.Done(): - return fmt.Errorf("autosync backoff interrupted: %w", ctx.Err()) - case <-timer.C: - return nil - } -} - -// computeBackoff returns an exponential backoff (capped at pullBackoffMax) plus -// up to ~50% jitter to avoid synchronized retries. attempt is 1-based; the first -// retry is attempt 2. -func computeBackoff(attempt int) time.Duration { - shift := max(attempt-firstRetryAttempt, 0) - - backoff := pullBackoffBase << shift - if backoff <= 0 || backoff > pullBackoffMax { - backoff = pullBackoffMax - } - - // Pseudo-random jitter (non-crypto; timing quality is irrelevant here). - jitter := time.Duration(time.Now().UnixNano() % (int64(backoff)/2 + 1)) - - return backoff + jitter -} - -// syncReferrers lists, pulls, verifies, and ingests the record's referrers. -// Each referrer is verified independently (belongs-to-record + type allow-list); -// a failing referrer is skipped without affecting the already-ingested record. -func (m *Manager) syncReferrers(ctx context.Context, peerID peer.ID, recordRef *corev1.RecordRef) { - descriptors, err := m.transport.ListReferrers(ctx, peerID, recordRef) - if err != nil { - logger.Warn("Autosync failed to list referrers", "cid", recordRef.GetCid(), "peer", peerID, "error", err) - - return - } - - for _, desc := range descriptors { - if _, ok := allowedReferrerTypes[desc.Type]; !ok { - logger.Warn("Autosync skipped referrer: disallowed type", - "cid", recordRef.GetCid(), "peer", peerID, "type", desc.Type) - - continue - } - - referrer, err := m.transport.PullReferrer(ctx, peerID, recordRef, desc) - if err != nil { - logger.Warn("Autosync failed to pull referrer", - "cid", recordRef.GetCid(), "peer", peerID, "referrer_cid", desc.Cid, "error", err) - - continue - } - - if !m.verifyReferrer(referrer, recordRef.GetCid(), desc) { - logger.Warn("Autosync rejected referrer: verification failed", - "cid", recordRef.GetCid(), "peer", peerID, "referrer_cid", desc.Cid, "type", desc.Type) - - continue - } - - if _, err := m.ingestor.ImportReferrer(ctx, recordRef.GetCid(), referrer); err != nil { - logger.Warn("Autosync failed to ingest referrer", - "cid", recordRef.GetCid(), "peer", peerID, "referrer_cid", desc.Cid, "error", err) - - continue - } - - logger.Debug("Autosync ingested referrer", - "cid", recordRef.GetCid(), "peer", peerID, "referrer_cid", desc.Cid, "type", desc.Type) - } -} - -// verifyReferrer checks that a pulled referrer belongs to the record and is -// self-consistent with the requested descriptor (type + referrer CID). This is -// fail-closed: any mismatch rejects the referrer. -func (m *Manager) verifyReferrer(referrer *corev1.RecordReferrer, recordCID string, desc rpc.ReferrerDescriptor) bool { - if referrer == nil { - return false - } - - // Must belong to the record we are syncing. - if referrer.GetRecordRef().GetCid() != recordCID { - return false - } - - // Must match the requested descriptor (guards against a peer relabeling). - if referrer.GetType() != desc.Type { - return false - } - - if referrer.GetReferrerRef().GetCid() != desc.Cid { - return false - } - - return true -} diff --git a/server/routing/autosync/autosync_test.go b/server/routing/autosync/autosync_test.go deleted file mode 100644 index 0e82ea3dd..000000000 --- a/server/routing/autosync/autosync_test.go +++ /dev/null @@ -1,407 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package autosync - -import ( - "context" - "crypto/rand" - "errors" - "testing" - - typesv1alpha1 "buf.build/gen/go/agntcy/oasf/protocolbuffers/go/agntcy/oasf/types/v1alpha1" - corev1 "github.com/agntcy/dir/api/core/v1" - "github.com/agntcy/dir/server/routing/rpc" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/structpb" -) - -// --- mocks --- - -type fakeTransport struct { - pullRecord *corev1.Record - pullErr error - pullCalls int - pullFailsBefore int // number of initial Pull calls that return a transient error - - descriptors []rpc.ReferrerDescriptor - listErr error - - referrers map[string]*corev1.RecordReferrer - pullReferrerErr error - pullReferrerCalls int -} - -func (f *fakeTransport) Pull(_ context.Context, _ peer.ID, _ *corev1.RecordRef) (*corev1.Record, error) { - f.pullCalls++ - - if f.pullCalls <= f.pullFailsBefore { - return nil, errors.New("transient pull failure") - } - - return f.pullRecord, f.pullErr -} - -func (f *fakeTransport) ListReferrers(_ context.Context, _ peer.ID, _ *corev1.RecordRef) ([]rpc.ReferrerDescriptor, error) { - return f.descriptors, f.listErr -} - -func (f *fakeTransport) PullReferrer(_ context.Context, _ peer.ID, _ *corev1.RecordRef, desc rpc.ReferrerDescriptor) (*corev1.RecordReferrer, error) { - f.pullReferrerCalls++ - - if f.pullReferrerErr != nil { - return nil, f.pullReferrerErr - } - - return f.referrers[desc.Cid], nil -} - -type fakeRouter struct{} - -func (fakeRouter) AddAddrs(peer.ID, []ma.Multiaddr) {} - -func (fakeRouter) FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) { - return peer.AddrInfo{}, errors.New("find peer not available in test") -} - -// countingRouter records FindPeer calls and returns configurable addresses. -type countingRouter struct { - findPeerCalls int - addrs []ma.Multiaddr - findErr error -} - -func (r *countingRouter) AddAddrs(peer.ID, []ma.Multiaddr) {} - -func (r *countingRouter) FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) { - r.findPeerCalls++ - - if r.findErr != nil { - return peer.AddrInfo{}, r.findErr - } - - return peer.AddrInfo{Addrs: r.addrs}, nil -} - -type fakeIngestor struct { - importRecordCalls int - importReferrerCalls int - lastRecord *corev1.Record -} - -func (f *fakeIngestor) ImportRecord(_ context.Context, record *corev1.Record) (*corev1.RecordRef, error) { - f.importRecordCalls++ - f.lastRecord = record - - return &corev1.RecordRef{Cid: record.GetCid()}, nil -} - -func (f *fakeIngestor) ImportReferrer(_ context.Context, _ string, _ *corev1.RecordReferrer) (*corev1.ReferrerRef, error) { - f.importReferrerCalls++ - - return &corev1.ReferrerRef{Cid: "referrer-cid"}, nil -} - -type fakeStore struct { - found bool -} - -func (f *fakeStore) Push(context.Context, *corev1.Record) (*corev1.RecordRef, error) { - return nil, nil //nolint:nilnil -} - -func (f *fakeStore) Pull(context.Context, *corev1.RecordRef) (*corev1.Record, error) { - return nil, nil //nolint:nilnil -} - -func (f *fakeStore) Lookup(_ context.Context, ref *corev1.RecordRef) (*corev1.RecordMeta, error) { - if f.found { - return &corev1.RecordMeta{Cid: ref.GetCid()}, nil - } - - return nil, errors.New("not found") -} - -func (f *fakeStore) Delete(context.Context, *corev1.RecordRef) error { return nil } -func (f *fakeStore) IsReady(context.Context) bool { return true } - -type fakeValidator struct { - valid bool -} - -func (f fakeValidator) ValidateRecord(context.Context, *structpb.Struct) (bool, []string, []string, error) { - if f.valid { - return true, nil, nil, nil - } - - return false, []string{"schema invalid"}, nil, nil -} - -// --- helpers --- - -func randomPeerID(t *testing.T) peer.ID { - t.Helper() - - _, pub, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - - pid, err := peer.IDFromPublicKey(pub) - require.NoError(t, err) - - return pid -} - -func testRecord(t *testing.T) *corev1.Record { - t.Helper() - - rec := corev1.New(&typesv1alpha1.Record{Name: "autosync-test", SchemaVersion: "0.7.0"}) - require.NotEmpty(t, rec.GetCid()) - - return rec -} - -func newTestManager(allow map[peer.ID]struct{}, tr *fakeTransport, ing *fakeIngestor, st *fakeStore, valid bool) *Manager { - return newManager(allow, tr, fakeRouter{}, ing, st, fakeValidator{valid: valid}) -} - -// --- tests --- - -func TestMaybeEnqueue_AllowListGating(t *testing.T) { - trusted := randomPeerID(t) - untrusted := randomPeerID(t) - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, &fakeTransport{}, &fakeIngestor{}, &fakeStore{}, true) - - // Untrusted peer: ignored. - m.MaybeEnqueue(&corev1.RecordRef{Cid: "cid-a"}, peer.AddrInfo{ID: untrusted}) - assert.Empty(t, m.queue) - - // Trusted peer: enqueued. - m.MaybeEnqueue(&corev1.RecordRef{Cid: "cid-a"}, peer.AddrInfo{ID: trusted}) - assert.Len(t, m.queue, 1) -} - -func TestMaybeEnqueue_Dedupe(t *testing.T) { - trusted := randomPeerID(t) - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, &fakeTransport{}, &fakeIngestor{}, &fakeStore{}, true) - - ref := &corev1.RecordRef{Cid: "cid-a"} - m.MaybeEnqueue(ref, peer.AddrInfo{ID: trusted}) - m.MaybeEnqueue(ref, peer.AddrInfo{ID: trusted}) // same CID already in flight - - assert.Len(t, m.queue, 1, "duplicate CID must not be enqueued twice") -} - -func TestProcess_HappyPath(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - cid := rec.GetCid() - - tr := &fakeTransport{ - pullRecord: rec, - descriptors: []rpc.ReferrerDescriptor{{Cid: "ref1", Type: corev1.SignatureReferrerType}}, - referrers: map[string]*corev1.RecordReferrer{ - "ref1": { - Type: corev1.SignatureReferrerType, - RecordRef: &corev1.RecordRef{Cid: cid}, - ReferrerRef: &corev1.ReferrerRef{Cid: "ref1"}, - }, - }, - } - ing := &fakeIngestor{} - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{}, true) - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: cid}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 1, ing.importRecordCalls, "record should be ingested") - assert.Equal(t, cid, ing.lastRecord.GetCid()) - assert.Equal(t, 1, ing.importReferrerCalls, "signature referrer should be ingested") -} - -func TestProcess_AlreadyLocalSkipsPull(t *testing.T) { - trusted := randomPeerID(t) - tr := &fakeTransport{} - ing := &fakeIngestor{} - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{found: true}, true) - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: "cid-a"}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 0, tr.pullCalls, "must not pull a record we already have") - assert.Equal(t, 0, ing.importRecordCalls) -} - -func TestProcess_CIDMismatchRejected(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - ing := &fakeIngestor{} - - tr := &fakeTransport{pullRecord: rec} - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{}, true) - // Announce a CID that does not match the pulled record's content. - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: "different-cid"}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 0, ing.importRecordCalls, "record with mismatched CID must be rejected") -} - -func TestProcess_ValidationFailureRejected(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - ing := &fakeIngestor{} - - tr := &fakeTransport{pullRecord: rec} - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{}, false) // validator rejects - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: rec.GetCid()}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 0, ing.importRecordCalls, "OASF-invalid record must be rejected") -} - -func TestProcess_DisallowedReferrerTypeSkipped(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - cid := rec.GetCid() - ing := &fakeIngestor{} - - tr := &fakeTransport{ - pullRecord: rec, - descriptors: []rpc.ReferrerDescriptor{{Cid: "evil1", Type: "com.attacker.evil"}}, - } - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{}, true) - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: cid}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 1, ing.importRecordCalls, "record still ingested") - assert.Equal(t, 0, tr.pullReferrerCalls, "disallowed referrer type must not be pulled") - assert.Equal(t, 0, ing.importReferrerCalls) -} - -func TestProcess_ReferrerBelongsToOtherRecordRejected(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - cid := rec.GetCid() - ing := &fakeIngestor{} - - tr := &fakeTransport{ - pullRecord: rec, - descriptors: []rpc.ReferrerDescriptor{{Cid: "ref1", Type: corev1.SignatureReferrerType}}, - referrers: map[string]*corev1.RecordReferrer{ - "ref1": { - Type: corev1.SignatureReferrerType, - RecordRef: &corev1.RecordRef{Cid: "some-other-record"}, // belongs elsewhere - ReferrerRef: &corev1.ReferrerRef{Cid: "ref1"}, - }, - }, - } - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{}, true) - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: cid}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 1, ing.importRecordCalls) - assert.Equal(t, 0, ing.importReferrerCalls, "referrer belonging to a different record must be rejected") -} - -func TestPullRecord_RetriesThenSucceeds(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - cid := rec.GetCid() - - tr := &fakeTransport{pullRecord: rec, pullFailsBefore: 1} // fail once, then succeed - ing := &fakeIngestor{} - - m := newTestManager(map[peer.ID]struct{}{trusted: {}}, tr, ing, &fakeStore{}, true) - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: cid}, peer: peer.AddrInfo{ID: trusted}}) - - assert.Equal(t, 2, tr.pullCalls, "should retry after a transient failure") - assert.Equal(t, 1, ing.importRecordCalls, "record ingested after retry") -} - -func TestPullRecord_ResolvesAddressesWhenNoneProvided(t *testing.T) { - trusted := randomPeerID(t) - rec := testRecord(t) - cid := rec.GetCid() - - tr := &fakeTransport{pullRecord: rec} - ing := &fakeIngestor{} - router := &countingRouter{} - - // GossipSub-triggered job: peer has an ID but no addresses. - m := newManager(map[peer.ID]struct{}{trusted: {}}, tr, router, ing, &fakeStore{}, fakeValidator{valid: true}) - m.process(t.Context(), job{ref: &corev1.RecordRef{Cid: cid}, peer: peer.AddrInfo{ID: trusted}}) - - assert.GreaterOrEqual(t, router.findPeerCalls, 1, "must resolve addresses via FindPeer when none are provided") - assert.Equal(t, 1, ing.importRecordCalls, "record should still be ingested") -} - -func TestComputeBackoff(t *testing.T) { - // First retry (attempt 2) is at least the base and below the cap+jitter. - b := computeBackoff(2) - assert.GreaterOrEqual(t, b, pullBackoffBase) - - // Large attempts are capped (plus at most ~50% jitter). - capped := computeBackoff(100) - assert.LessOrEqual(t, capped, pullBackoffMax+pullBackoffMax/2+1) -} - -func TestVerifyReferrer(t *testing.T) { - m := &Manager{} - desc := rpc.ReferrerDescriptor{Cid: "ref1", Type: corev1.SignatureReferrerType} - - tests := []struct { - name string - referrer *corev1.RecordReferrer - want bool - }{ - { - name: "valid", - referrer: &corev1.RecordReferrer{ - Type: corev1.SignatureReferrerType, - RecordRef: &corev1.RecordRef{Cid: "rec"}, - ReferrerRef: &corev1.ReferrerRef{Cid: "ref1"}, - }, - want: true, - }, - { - name: "nil referrer", - referrer: nil, - want: false, - }, - { - name: "wrong record", - referrer: &corev1.RecordReferrer{ - Type: corev1.SignatureReferrerType, - RecordRef: &corev1.RecordRef{Cid: "other"}, - ReferrerRef: &corev1.ReferrerRef{Cid: "ref1"}, - }, - want: false, - }, - { - name: "type mismatch", - referrer: &corev1.RecordReferrer{ - Type: corev1.PublicKeyReferrerType, - RecordRef: &corev1.RecordRef{Cid: "rec"}, - ReferrerRef: &corev1.ReferrerRef{Cid: "ref1"}, - }, - want: false, - }, - { - name: "referrer cid mismatch", - referrer: &corev1.RecordReferrer{ - Type: corev1.SignatureReferrerType, - RecordRef: &corev1.RecordRef{Cid: "rec"}, - ReferrerRef: &corev1.ReferrerRef{Cid: "different"}, - }, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, m.verifyReferrer(tt.referrer, "rec", desc)) - }) - } -} diff --git a/server/routing/cleanup_core_test.go b/server/routing/cleanup_core_test.go deleted file mode 100644 index 0a2eb7b34..000000000 --- a/server/routing/cleanup_core_test.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "context" - "os" - "testing" - "time" - - "github.com/agntcy/dir/server/datastore" - "github.com/agntcy/dir/server/types" - ipfsdatastore "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Test the core cleanup logic without complex server dependencies. -func TestCleanup_CoreLogic(t *testing.T) { - ctx := t.Context() - - dstore, cleanup := setupCleanupCoreTestDatastore(t) - defer cleanup() - - t.Run("cleanup_labels_for_specific_cid", func(t *testing.T) { - testCID := "test-cid-123" - localPeerID := testLocalPeerID - - // Setup test data: record + labels - recordKey := ipfsdatastore.NewKey("/records/" + testCID) - err := dstore.Put(ctx, recordKey, []byte{}) - require.NoError(t, err) - - // Add labels for this CID (local and remote) - testLabels := []struct { - label string - peerID string - shouldBeDeleted bool - }{ - {"/skills/AI", localPeerID, true}, // Local - should be deleted - {"/skills/ML", localPeerID, true}, // Local - should be deleted - {"/skills/AI", "remote-peer", false}, // Remote - should be kept - } - - for _, tl := range testLabels { - enhancedKey := BuildEnhancedLabelKey(types.Label(tl.label), testCID, tl.peerID) - err = dstore.Put(ctx, ipfsdatastore.NewKey(enhancedKey), []byte("metadata")) - require.NoError(t, err) - } - - // Test cleanup logic - success := simulateCleanupLabelsForCID(ctx, dstore, testCID, localPeerID) - assert.True(t, success) - - // Verify record key was deleted - exists, err := dstore.Has(ctx, recordKey) - require.NoError(t, err) - assert.False(t, exists) - - // Verify label cleanup - for _, tl := range testLabels { - enhancedKey := BuildEnhancedLabelKey(types.Label(tl.label), testCID, tl.peerID) - exists, err := dstore.Has(ctx, ipfsdatastore.NewKey(enhancedKey)) - require.NoError(t, err) - - if tl.shouldBeDeleted { - assert.False(t, exists, "Local label should be deleted") - } else { - assert.True(t, exists, "Remote label should be kept") - } - } - }) - - t.Run("stale_label_detection", func(t *testing.T) { - // Test the core logic of stale label detection - now := time.Now() - - testCases := []struct { - name string - metadata *types.LabelMetadata - isStale bool - }{ - { - name: "fresh_label", - metadata: &types.LabelMetadata{ - Timestamp: now.Add(-time.Hour), - LastSeen: now.Add(-time.Hour), - }, - isStale: false, - }, - { - name: "stale_label", - metadata: &types.LabelMetadata{ - Timestamp: now.Add(-MaxLabelAge - time.Hour), - LastSeen: now.Add(-MaxLabelAge - time.Hour), - }, - isStale: true, - }, - { - name: "borderline_fresh", - metadata: &types.LabelMetadata{ - Timestamp: now.Add(-MaxLabelAge + time.Minute), - LastSeen: now.Add(-MaxLabelAge + time.Minute), - }, - isStale: false, - }, - { - name: "borderline_stale", - metadata: &types.LabelMetadata{ - Timestamp: now.Add(-MaxLabelAge - time.Minute), - LastSeen: now.Add(-MaxLabelAge - time.Minute), - }, - isStale: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := tc.metadata.IsStale(MaxLabelAge) - assert.Equal(t, tc.isStale, result) - }) - } - }) - - t.Run("remote_label_filter_logic", func(t *testing.T) { - // Test the remote label filtering logic - localPeerID := testLocalPeerID - - testCases := []struct { - name string - key string - expected bool // true if should be included (is remote) - }{ - { - name: "remote_label", - key: "/skills/AI/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/remote-peer", - expected: true, - }, - { - name: "local_label", - key: "/skills/AI/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/" + testLocalPeerID, - expected: false, - }, - { - name: "malformed_key_treated_as_remote", - key: "/invalid-key", - expected: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Test the core filtering logic - keyPeerID := ExtractPeerIDFromKey(tc.key) - isRemote := (keyPeerID != localPeerID) || (keyPeerID == "") - assert.Equal(t, tc.expected, isRemote) - }) - } - }) - - t.Run("batch_deletion_efficiency", func(t *testing.T) { - // Test that batch operations work correctly - testCID := "batch-test-cid" - localPeerID := testLocalPeerID - - // Setup multiple labels to delete - labelsToDelete := []string{ - "/skills/AI", - "/skills/ML", - "/domains/tech", - "/modules/nlp", - } - - // Store record and labels - recordKey := ipfsdatastore.NewKey("/records/" + testCID) - err := dstore.Put(ctx, recordKey, []byte{}) - require.NoError(t, err) - - for _, label := range labelsToDelete { - enhancedKey := BuildEnhancedLabelKey(types.Label(label), testCID, localPeerID) - err = dstore.Put(ctx, ipfsdatastore.NewKey(enhancedKey), []byte("metadata")) - require.NoError(t, err) - } - - // Count keys before cleanup - allResults, err := dstore.Query(ctx, query.Query{}) - require.NoError(t, err) - - var keysBefore []string - for result := range allResults.Next() { - keysBefore = append(keysBefore, result.Key) - } - - allResults.Close() - - // Run cleanup - success := simulateCleanupLabelsForCID(ctx, dstore, testCID, localPeerID) - assert.True(t, success) - - // Count keys after cleanup - allResults, err = dstore.Query(ctx, query.Query{}) - require.NoError(t, err) - - var keysAfter []string - for result := range allResults.Next() { - keysAfter = append(keysAfter, result.Key) - } - - allResults.Close() - - // Should have deleted record + all labels (5 keys total) - expectedDeleted := 1 + len(labelsToDelete) // 1 record + 4 labels - actualDeleted := len(keysBefore) - len(keysAfter) - assert.Equal(t, expectedDeleted, actualDeleted) - }) -} - -// Simplified cleanup logic for testing (without server dependency). -func simulateCleanupLabelsForCID(ctx context.Context, dstore types.Datastore, cid string, localPeerID string) bool { - batch, err := dstore.Batch(ctx) - if err != nil { - return false - } - - keysDeleted := 0 - - // Remove the /records/ key - recordKey := ipfsdatastore.NewKey("/records/" + cid) - if err := batch.Delete(ctx, recordKey); err == nil { - keysDeleted++ - } - - // Find and remove all label keys for this CID using shared namespace iteration - entries, err := QueryAllNamespaces(ctx, dstore) - if err != nil { - return false - } - - for _, entry := range entries { - // Parse enhanced key - _, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - // Delete malformed keys - if err := batch.Delete(ctx, ipfsdatastore.NewKey(entry.Key)); err == nil { - keysDeleted++ - } - - continue - } - - // Check if this key matches our CID and is from local peer - if keyCID == cid && keyPeerID == localPeerID { - labelKey := ipfsdatastore.NewKey(entry.Key) - if err := batch.Delete(ctx, labelKey); err == nil { - keysDeleted++ - } - } - } - - // Commit the batch deletion - if err := batch.Commit(ctx); err != nil { - return false - } - - return keysDeleted > 0 -} - -// Helper function for cleanup testing. -func setupCleanupCoreTestDatastore(t *testing.T) (types.Datastore, func()) { - t.Helper() - - dsOpts := []datastore.Option{ - datastore.WithFsProvider("/tmp/test-cleanup-core-" + t.Name()), - } - - dstore, err := datastore.New(dsOpts...) - require.NoError(t, err) - - cleanup := func() { - _ = dstore.Close() - _ = os.RemoveAll("/tmp/test-cleanup-core-" + t.Name()) - } - - return dstore, cleanup -} diff --git a/server/routing/cleanup_tasks.go b/server/routing/cleanup_tasks.go deleted file mode 100644 index 14d3d5d61..000000000 --- a/server/routing/cleanup_tasks.go +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "context" - "encoding/json" - "fmt" - "path" - "sync" - "time" - - corev1 "github.com/agntcy/dir/api/core/v1" - "github.com/agntcy/dir/server/routing/internal/p2p" - "github.com/agntcy/dir/server/routing/pubsub" - "github.com/agntcy/dir/server/types" - "github.com/agntcy/dir/utils/logging" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" -) - -var cleanupLogger = logging.Logger("routing/cleanup") - -// remoteLabelFilter identifies remote labels by checking if they lack a corresponding local record. -// Remote labels are those that don't have a matching "/records/CID" key in the datastore. -// -//nolint:containedctx -type remoteLabelFilter struct { - dstore types.Datastore - ctx context.Context - localPeerID string -} - -func (f *remoteLabelFilter) Filter(e query.Entry) bool { - // With enhanced keys, we can check PeerID directly from the key - // Key format: /skills/AI/CID123/Peer1 - keyPeerID := ExtractPeerIDFromKey(e.Key) - if keyPeerID == "" { - // Invalid key format, assume remote to be safe - return true - } - - // It's remote if the PeerID in the key is not our local peer - return keyPeerID != f.localPeerID -} - -// CleanupManager handles all background cleanup and republishing tasks for the routing system. -// This includes CID provider republishing, GossipSub label republishing, stale remote label cleanup, and orphaned record cleanup. -type CleanupManager struct { - dstore types.Datastore - storeAPI types.StoreAPI - server *p2p.Server - publishFunc pubsub.PublishEventHandler // Publishing callback (captures routeRemote state) - republishInterval time.Duration // How often local CID providers are republished -} - -// NewCleanupManager creates a new cleanup manager with the required dependencies. -// The publishFunc is injected from routeRemote.Publish to avoid circular dependencies -// while still providing access to DHT and GossipSub publishing logic. -// -// Parameters: -// - dstore: Datastore for label storage -// - storeAPI: Store API for record operations -// - server: P2P server for DHT operations -// - publishFunc: Callback for publishing (from routeRemote.Publish, see pubsub.PublishEventHandler) -// - republishInterval: how often local CID providers are republished; if <= 0, -// the default RepublishInterval constant (36h) is used. -func NewCleanupManager( - dstore types.Datastore, - storeAPI types.StoreAPI, - server *p2p.Server, - publishFunc pubsub.PublishEventHandler, - republishInterval time.Duration, -) *CleanupManager { - // Guard against zero/negative values: time.NewTicker panics on <= 0, and a - // missing config value should fall back to the default interval. - if republishInterval <= 0 { - republishInterval = RepublishInterval - } - - return &CleanupManager{ - dstore: dstore, - storeAPI: storeAPI, - server: server, - publishFunc: publishFunc, - republishInterval: republishInterval, - } -} - -// StartLabelRepublishTask starts a background task that periodically republishes local -// CID provider announcements to keep content discoverable (provider records expire after ProviderRecordTTL). -// The wg parameter is used to track this goroutine in the parent's WaitGroup. -func (c *CleanupManager) StartLabelRepublishTask(ctx context.Context, wg *sync.WaitGroup) { - ticker := time.NewTicker(c.republishInterval) - - cleanupLogger.Info("Started CID provider republishing task", "interval", c.republishInterval) - - defer func() { - ticker.Stop() - wg.Done() - cleanupLogger.Debug("CID provider republishing task stopped") - }() - - for { - select { - case <-ctx.Done(): - cleanupLogger.Info("CID provider republishing task stopping (context cancelled)") - - return - case <-ticker.C: - c.republishLocalProviders(ctx) - } - } -} - -// StartRemoteLabelCleanupTask starts a background task that periodically cleans up stale remote labels. -// This is critical for the pull-based architecture to remove cached labels from offline or deleted remote content. -// The wg parameter is used to track this goroutine in the parent's WaitGroup. -func (c *CleanupManager) StartRemoteLabelCleanupTask(ctx context.Context, wg *sync.WaitGroup) { - ticker := time.NewTicker(CleanupInterval) - - cleanupLogger.Info("Starting remote label cleanup task", "interval", CleanupInterval) - - defer func() { - ticker.Stop() - wg.Done() - cleanupLogger.Debug("Remote label cleanup task stopped") - }() - - for { - select { - case <-ctx.Done(): - cleanupLogger.Info("Remote label cleanup task stopping (context cancelled)") - - return - case <-ticker.C: - if err := c.cleanupStaleRemoteLabels(ctx); err != nil { - cleanupLogger.Error("Failed to cleanup stale remote labels", "error", err) - } - } - } -} - -// republishLocalProviders republishes all local CID provider announcements and labels -// to ensure they remain discoverable. This maintains both DHT provider records and -// GossipSub label announcements for optimal network propagation. -func (c *CleanupManager) republishLocalProviders(ctx context.Context) { - cleanupLogger.Info("Starting CID provider and label republishing cycle") - - // Query all local records from the datastore - results, err := c.dstore.Query(ctx, query.Query{ - Prefix: "/records/", - }) - if err != nil { - cleanupLogger.Error("Failed to query local records for republishing", "error", err) - - return - } - defer results.Close() - - republishedCount := 0 - labelRepublishedCount := 0 - errorCount := 0 - - var orphanedCIDs []string - - for result := range results.Next() { - if result.Error != nil { - cleanupLogger.Warn("Error reading local record for republishing", "error", result.Error) - - continue - } - - // Extract CID from record key: /records/CID123 β†’ CID123 - cidStr := path.Base(result.Key) - if cidStr == "" { - continue - } - - // Verify the record still exists in storage - ref := &corev1.RecordRef{Cid: cidStr} - - _, err := c.storeAPI.Lookup(ctx, ref) - if err != nil { - cleanupLogger.Warn("Record no longer exists in storage, marking as orphaned", "cid", cidStr, "error", err) - orphanedCIDs = append(orphanedCIDs, cidStr) - errorCount++ - - continue - } - - // Pull the record from storage for republishing - record, err := c.storeAPI.Pull(ctx, ref) - if err != nil { - cleanupLogger.Warn("Failed to pull record for republishing", - "cid", cidStr, - "error", err) - - errorCount++ - - continue - } - - // Wrap record with adapter for interface-based publishing - adapter, err := record.Decode() - if err != nil { - cleanupLogger.Warn("Failed to create record adapter for republishing", - "cid", cidStr, - "error", err) - - errorCount++ - - continue - } - - // Use injected publishing function (handles both DHT and GossipSub) - // This reuses routeRemote.Publish logic without circular dependency - if err := c.publishFunc(ctx, adapter); err != nil { - cleanupLogger.Warn("Failed to republish record to network", - "cid", cidStr, - "error", err) - - errorCount++ - - continue - } - - cleanupLogger.Debug("Successfully republished record to network", "cid", cidStr) - - republishedCount++ - labelRepublishedCount++ // Count label republishing (done inside publishFunc) - } - - // Clean up orphaned local records and their labels - if len(orphanedCIDs) > 0 { - cleanedCount := c.cleanupOrphanedLocalLabels(ctx, orphanedCIDs) - cleanupLogger.Info("Cleaned up orphaned local records", "count", cleanedCount) - } - - cleanupLogger.Info("Completed republishing cycle", - "dhtRepublished", republishedCount, - "gossipSubRepublished", labelRepublishedCount, - "errors", errorCount, - "orphaned", len(orphanedCIDs)) -} - -// cleanupStaleRemoteLabels removes remote labels that haven't been seen recently. -func (c *CleanupManager) cleanupStaleRemoteLabels(ctx context.Context) error { - localPeerID := c.server.Host().ID().String() - - cleanupLogger.Debug("Starting stale remote label cleanup") - - // Query all label keys with remote filter - // We'll query each namespace separately and combine results - var allResults []query.Result - - for _, namespace := range types.AllLabelTypes() { - nsResults, err := c.dstore.Query(ctx, query.Query{ - Prefix: namespace.Prefix(), - Filters: []query.Filter{ - &remoteLabelFilter{ - dstore: c.dstore, - ctx: ctx, - localPeerID: localPeerID, - }, - }, - }) - if err != nil { - cleanupLogger.Warn("Failed to query namespace", "namespace", namespace, "error", err) - - continue - } - - // Collect results from this namespace - for result := range nsResults.Next() { - allResults = append(allResults, result) - } - - nsResults.Close() - } - - var staleKeys []datastore.Key - - // Check each remote label for staleness - for _, result := range allResults { - if result.Error != nil { - cleanupLogger.Warn("Error reading label entry", "key", result.Key, "error", result.Error) - - continue - } - - // Parse enhanced key to get peer information - _, _, keyPeerID, err := ParseEnhancedLabelKey(result.Key) - if err != nil { - cleanupLogger.Warn("Failed to parse enhanced label key, marking for deletion", - "key", result.Key, "error", err) - - staleKeys = append(staleKeys, datastore.NewKey(result.Key)) - - continue - } - - var metadata types.LabelMetadata - if err := json.Unmarshal(result.Value, &metadata); err != nil { - cleanupLogger.Warn("Failed to parse label metadata, marking for deletion", - "key", result.Key, "error", err) - - staleKeys = append(staleKeys, datastore.NewKey(result.Key)) - - continue - } - - // Validate metadata before checking staleness - if err := metadata.Validate(); err != nil { - cleanupLogger.Warn("Invalid label metadata found during cleanup, marking for deletion", - "key", result.Key, "error", err) - - staleKeys = append(staleKeys, datastore.NewKey(result.Key)) - - continue - } - - // Check if label is stale using the IsStale method - if metadata.IsStale(MaxLabelAge) { - cleanupLogger.Debug("Found stale remote label", - "key", result.Key, "age", metadata.Age(), "peer", keyPeerID) - - staleKeys = append(staleKeys, datastore.NewKey(result.Key)) - } - } - - // Delete stale labels in batch - if len(staleKeys) > 0 { - batch, err := c.dstore.Batch(ctx) - if err != nil { - return fmt.Errorf("failed to create batch for cleanup: %w", err) - } - - for _, key := range staleKeys { - if err := batch.Delete(ctx, key); err != nil { - cleanupLogger.Warn("Failed to delete stale label", "key", key.String(), "error", err) - } - } - - if err := batch.Commit(ctx); err != nil { - return fmt.Errorf("failed to commit stale label cleanup: %w", err) - } - - cleanupLogger.Info("Cleaned up stale remote labels", "count", len(staleKeys)) - } else { - cleanupLogger.Debug("No stale remote labels found") - } - - return nil -} - -// cleanupOrphanedLocalLabels removes local records and labels for CIDs that no longer exist in storage. -func (c *CleanupManager) cleanupOrphanedLocalLabels(ctx context.Context, orphanedCIDs []string) int { - cleanedCount := 0 - - for _, cid := range orphanedCIDs { - if c.cleanupLabelsForCID(ctx, cid) { - cleanedCount++ - } - } - - return cleanedCount -} - -// cleanupLabelsForCID removes all local records and labels associated with a specific CID. -func (c *CleanupManager) cleanupLabelsForCID(ctx context.Context, cid string) bool { - batch, err := c.dstore.Batch(ctx) - if err != nil { - cleanupLogger.Error("Failed to create cleanup batch", "cid", cid, "error", err) - - return false - } - - keysDeleted := 0 - - // Remove the /records/ key - recordKey := datastore.NewKey("/records/" + cid) - if err := batch.Delete(ctx, recordKey); err != nil { - cleanupLogger.Warn("Failed to delete record key", "key", recordKey.String(), "error", err) - } else { - keysDeleted++ - } - - // Find and remove all label keys for this CID across all namespaces - localPeerID := c.server.Host().ID().String() - - for _, namespace := range types.AllLabelTypes() { - // Query labels in this namespace that match our CID - labelResults, err := c.dstore.Query(ctx, query.Query{ - Prefix: namespace.Prefix(), - }) - if err != nil { - cleanupLogger.Warn("Failed to query labels for cleanup", "namespace", namespace, "cid", cid, "error", err) - - continue - } - - defer labelResults.Close() - - for result := range labelResults.Next() { - // Parse enhanced key to get CID and PeerID - _, keyCID, keyPeerID, err := ParseEnhancedLabelKey(result.Key) - if err != nil { - cleanupLogger.Warn("Failed to parse enhanced label key during cleanup, deleting", - "key", result.Key, "error", err) - // Delete malformed keys - if err := batch.Delete(ctx, datastore.NewKey(result.Key)); err == nil { - keysDeleted++ - } - - continue - } - - // Check if this key matches our CID and is from local peer - if keyCID == cid && keyPeerID == localPeerID { - // Delete this local label - labelKey := datastore.NewKey(result.Key) - if err := batch.Delete(ctx, labelKey); err != nil { - cleanupLogger.Warn("Failed to delete label key", "key", labelKey.String(), "error", err) - } else { - keysDeleted++ - - cleanupLogger.Debug("Scheduled orphaned label for deletion", "key", result.Key) - } - } - } - } - - // Commit the batch deletion - if err := batch.Commit(ctx); err != nil { - cleanupLogger.Error("Failed to commit orphaned label cleanup", "cid", cid, "error", err) - - return false - } - - if keysDeleted > 0 { - cleanupLogger.Debug("Successfully cleaned up orphaned labels", "cid", cid, "keysDeleted", keysDeleted) - } - - return keysDeleted > 0 -} diff --git a/server/routing/config/config.go b/server/routing/config/config.go index 74a1baa2f..70eba4edc 100644 --- a/server/routing/config/config.go +++ b/server/routing/config/config.go @@ -4,10 +4,7 @@ package config import ( - "fmt" "time" - - "github.com/libp2p/go-libp2p/core/peer" ) var ( @@ -16,12 +13,6 @@ var ( // TODO: once we deploy our bootstrap nodes, we should update this } - // GossipSub default (only enable/disable is configurable). - DefaultGossipSubEnabled = true - - // Autosync default (disabled by default; deny-by-default policy). - DefaultAutosyncEnabled = false - // RelayService default (disabled; enable only on publicly-reachable nodes). DefaultRelayServiceEnabled = false @@ -55,7 +46,8 @@ type Config struct { // Path to the routing datastore. // If empty, the routing data will be stored in memory. - // If not empty, this dir will be used to store the routing data on disk. + // If not empty, this dir will be used to persist the DHT's state on disk: + // its routing table and the provider records it holds for other peers. DatastoreDir string `json:"datastore_dir,omitempty" mapstructure:"datastore_dir"` // Refresh interval for DHT routing tables. @@ -63,9 +55,10 @@ type Config struct { // This is primarily used for testing with faster intervals. RefreshInterval time.Duration `json:"refresh_interval,omitempty" mapstructure:"refresh_interval"` - // RepublishInterval controls how often local CID provider announcements are - // republished to keep content discoverable (DHT provider records + GossipSub - // labels). If not set or zero, uses the default RepublishInterval constant (36h). + // RepublishInterval controls how often the records this node has published + // are re-advertised, by CID and by label, so their provider records do not + // expire. They are also advertised once at startup regardless. + // If not set or zero, uses the default RepublishInterval constant (36h). // Lower values let newly joined nodes converge on existing content sooner, at // the cost of more frequent announcement traffic. RepublishInterval time.Duration `json:"republish_interval,omitempty" mapstructure:"republish_interval"` @@ -95,90 +88,4 @@ type Config struct { // reservations. Enable only on genuinely public nodes. Mutually exclusive // with ForceReachabilityPrivate. ForceReachabilityPublic bool `json:"force_reachability_public,omitempty" mapstructure:"force_reachability_public"` - - // GossipSub configuration for label announcements - GossipSub GossipSubConfig `json:"gossipsub" mapstructure:"gossipsub"` - - // Autosync configuration for DHT-based record + referrer synchronization - Autosync AutosyncConfig `json:"autosync" mapstructure:"autosync"` -} - -// GossipSubConfig configures GossipSub-based label announcements. -// Protocol parameters (topic name, message size limits) are NOT configurable -// and are defined in server/routing/pubsub/constants.go to ensure network-wide -// compatibility. Only the enable/disable flag is configurable. -// -// Benefits when enabled: -// - Reaches ALL subscribed peers (not just k-closest in DHT) -// - Minimal bandwidth (~100B vs KB-MB for full record) -// - Fast propagation (~5-20ms vs ~100-500ms for DHT) -// - High cache hit rate (90%+ vs 30% with pull-based) -type GossipSubConfig struct { - // Enabled controls whether GossipSub label announcements are used. - // When true: Labels are announced via GossipSub (efficient, wide propagation) - // When false: Falls back to DHT+Pull mechanism (existing behavior) - // Default: true (recommended for production) - // - // Note: Protocol parameters (topic, message size) are hardcoded in - // server/routing/pubsub/constants.go for network compatibility. - Enabled bool `json:"enabled,omitempty" mapstructure:"enabled"` -} - -// AutosyncConfig configures DHT-based record + referrer synchronization. -// -// When enabled, DHT provider announcements originating from a peer in PeerList -// trigger the node to pull the announced record (and its referrers) from that -// peer over libp2p RPC and ingest them locally with full parity to a normal -// push (content store + search index + referrer state). -// -// The policy is deny-by-default: only peers explicitly listed in PeerList are -// ever synced from. Allow-list matching is performed against libp2p's -// authenticated peer.ID (never a self-reported/payload field). -type AutosyncConfig struct { - // Enabled controls whether DHT-based autosync is active. - // Default: false (deny-by-default). - Enabled bool `json:"enabled,omitempty" mapstructure:"enabled"` - - // PeerList is the allow-list of trusted source peers to auto-sync from. - // - // Note: this is a list of objects (not bare peer-ID strings) so that - // per-peer policy fields (e.g. a future "republish" flag) can be added - // without a breaking config change. Because it is a list of structs, it is - // configured via config file/YAML only (not via a single environment - // variable). - PeerList []AutosyncPeer `json:"peerlist,omitempty" mapstructure:"peerlist"` -} - -// AutosyncPeer identifies a single trusted source peer in the autosync -// allow-list. -type AutosyncPeer struct { - // Peer is the libp2p peer ID of the trusted source peer - // (e.g. "12D3KooW..."). - Peer string `json:"peer" mapstructure:"peer"` - - // NOTE: A "Republish" flag is intentionally deferred to a future iteration. - // Keeping this a struct (rather than a bare string) makes adding it later a - // non-breaking change. -} - -// AllowSet parses the configured PeerList into a set of libp2p peer IDs for -// O(1) membership checks by the autosync manager. -// -// It fails fast: an invalid peer ID returns an error identifying the offending -// entry, rather than being silently skipped. This is deliberate for a security -// allow-list β€” a typo in a trusted peer ID should be surfaced at startup, not -// silently ignored (which could otherwise cause a trusted peer to never sync). -func (c AutosyncConfig) AllowSet() (map[peer.ID]struct{}, error) { - allowSet := make(map[peer.ID]struct{}, len(c.PeerList)) - - for i, p := range c.PeerList { - pid, err := peer.Decode(p.Peer) - if err != nil { - return nil, fmt.Errorf("invalid autosync peer ID at peerlist[%d] (%q): %w", i, p.Peer, err) - } - - allowSet[pid] = struct{}{} - } - - return allowSet, nil } diff --git a/server/routing/config/config_test.go b/server/routing/config/config_test.go deleted file mode 100644 index 0d0b9d5f8..000000000 --- a/server/routing/config/config_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package config - -import ( - "crypto/rand" - "testing" - - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// newTestPeerID generates a valid libp2p peer ID for use in tests. -func newTestPeerID(t *testing.T) peer.ID { - t.Helper() - - _, pub, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - - pid, err := peer.IDFromPublicKey(pub) - require.NoError(t, err) - - return pid -} - -func TestDefaultAutosyncDisabled(t *testing.T) { - // Deny-by-default: autosync must be off unless explicitly enabled. - assert.False(t, DefaultAutosyncEnabled) - - var cfg AutosyncConfig - assert.False(t, cfg.Enabled) - assert.Empty(t, cfg.PeerList) -} - -func TestAutosyncConfig_AllowSet_Valid(t *testing.T) { - p1 := newTestPeerID(t) - p2 := newTestPeerID(t) - - cfg := AutosyncConfig{ - Enabled: true, - PeerList: []AutosyncPeer{ - {Peer: p1.String()}, - {Peer: p2.String()}, - }, - } - - allowSet, err := cfg.AllowSet() - require.NoError(t, err) - assert.Len(t, allowSet, 2) - assert.Contains(t, allowSet, p1) - assert.Contains(t, allowSet, p2) -} - -func TestAutosyncConfig_AllowSet_Empty(t *testing.T) { - cfg := AutosyncConfig{Enabled: true} - - allowSet, err := cfg.AllowSet() - require.NoError(t, err) - assert.Empty(t, allowSet) -} - -func TestAutosyncConfig_AllowSet_Deduplicates(t *testing.T) { - p1 := newTestPeerID(t) - - cfg := AutosyncConfig{ - PeerList: []AutosyncPeer{ - {Peer: p1.String()}, - {Peer: p1.String()}, - }, - } - - allowSet, err := cfg.AllowSet() - require.NoError(t, err) - assert.Len(t, allowSet, 1) - assert.Contains(t, allowSet, p1) -} - -func TestAutosyncConfig_AllowSet_InvalidPeerID(t *testing.T) { - valid := newTestPeerID(t) - - cfg := AutosyncConfig{ - PeerList: []AutosyncPeer{ - {Peer: valid.String()}, - {Peer: "not-a-valid-peer-id"}, - }, - } - - allowSet, err := cfg.AllowSet() - require.Error(t, err) - assert.Nil(t, allowSet) - // Error should identify the offending entry (index + value) for fast triage. - assert.Contains(t, err.Error(), "peerlist[1]") - assert.Contains(t, err.Error(), "not-a-valid-peer-id") -} diff --git a/server/routing/constants.go b/server/routing/constants.go index 0406661fc..bad046365 100644 --- a/server/routing/constants.go +++ b/server/routing/constants.go @@ -12,26 +12,51 @@ const ( // This is configured via dht.MaxRecordAge() and affects all PutValue operations. // Default DHT TTL is 36h, but we use 48h for better network resilience. RecordTTL = 48 * time.Hour - // RepublishInterval defines how often we republish CID provider announcements to prevent expiration. - // Provider records typically expire after 24h, but we use a longer interval for robustness. - // This ensures our content remains discoverable by triggering pull-based label caching. + // RepublishInterval defines how often published records are re-advertised + // so their provider records, for both CIDs and labels, stay alive. RepublishInterval = 36 * time.Hour - // CleanupInterval defines how often we clean up stale announcements. - // This should match DHTRecordTTL to stay consistent with DHT behavior and prevent - // our local cache from having stale entries that no longer exist in the DHT. - CleanupInterval = 48 * time.Hour + // advertisePollInterval is how often a node with an empty routing table + // rechecks for a peer to advertise to. + advertisePollInterval = 5 * time.Second // RefreshInterval defines how often DHT routing tables are refreshed. // This is a shorter interval for maintaining network connectivity. RefreshInterval = 30 * time.Second + // ProviderCountTimeout bounds a single provider-count lookup. The caller + // iterates every local record, so this is a latency budget per CID rather + // than a correctness knob: cutting a lookup short undercounts providers. + ProviderCountTimeout = 10 * time.Second +) + +// Budgets for a remote search. They overlap rather than run in sequence: the +// provider lookup and the peer queries it feeds are concurrent, and every one +// of them is capped by the overall search budget. +// +// Results stream, so these bound latency rather than correctness β€” an expiring +// budget costs recall. +const ( + // SearchTimeout bounds a whole remote search, discovery and peer queries. + SearchTimeout = 30 * time.Second + + // SearchDiscoveryTimeout bounds the DHT provider lookup. It runs to + // completion rather than stopping at a target count, so it needs a deadline + // of its own; peers already found keep being queried after it expires. + SearchDiscoveryTimeout = 15 * time.Second + + // SearchPeerTimeout bounds one peer's query. Provider records outlive the + // peers that wrote them, so some of these will always time out. + SearchPeerTimeout = 10 * time.Second ) // Protocol constants for libp2p DHT and discovery. const ( // ProtocolPrefix is the prefix used for DHT protocol identification. - ProtocolPrefix = "dir" + // The DHT appends "/kad/1.0.0", so peers speak "/dir/2/kad/1.0.0". + // v2 nodes must not share a routing table with v1: they advertise label + // provider records that a v1 node would misread as content CIDs. + ProtocolPrefix = "/dir/2" // ProtocolRendezvous is the rendezvous string used for peer discovery. - ProtocolRendezvous = "dir/connect" + ProtocolRendezvous = "dir/2/connect" ) // Validation rules and limits. @@ -39,12 +64,30 @@ const ( // MaxHops defines the maximum number of hops allowed in distributed queries. MaxHops = 20 - // NotificationChannelSize defines the buffer size for announcement notifications. - NotificationChannelSize = 1000 + // advertiseConcurrency bounds how many DHT Provide calls run at once. + // Each one is a full Kademlia lookup followed by K AddProvider sends, so a + // record carrying several deep skills would otherwise serialise into tens + // of seconds on a synchronous publish. + advertiseConcurrency = 8 + + // searchPeerWorkers bounds how many providers are queried at once. + searchPeerWorkers = 8 + + // searchProviderBuffer decouples the provider lookup from those workers. + // The lookup stalls while nobody reads its channel, so discovery has to be + // able to run ahead of the queries it feeds. + searchProviderBuffer = 64 + + // advertisePageSize bounds how many published records are loaded at a time when + // enumerating what to advertise. GetRecords applies no LIMIT when the limit + // is zero, so an unpaged call would read the whole corpus into memory. + advertisePageSize = 500 - // MaxLabelAge defines when remote label announcements are considered stale. - // Labels older than this will be cleaned up during periodic cleanup cycles. - MaxLabelAge = 72 * time.Hour + // maxListCandidates caps how many records one query of a multi-query list + // pulls before the results are intersected. The requested limit cannot be + // pushed into those queries without dropping valid matches, so this bounds + // the work instead. + maxListCandidates = 10000 // DefaultMinMatchScore defines the minimum allowed match score for production safety. // Per proto specification: "If not set, it will return records that match at least one query". diff --git a/server/routing/handler.go b/server/routing/handler.go deleted file mode 100644 index 00a6c50f8..000000000 --- a/server/routing/handler.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "context" - "fmt" - - corev1 "github.com/agntcy/dir/api/core/v1" - "github.com/agntcy/dir/utils/logging" - "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p-kad-dht/records" - "github.com/libp2p/go-libp2p/core/peer" - mh "github.com/multiformats/go-multihash" -) - -var ( - _ records.ProviderStore = &handler{} - handlerLogger = logging.Logger("routing/handler") -) - -type handler struct { - *records.ProviderManager - hostID string - notifyCh chan<- *handlerSync -} - -type handlerSync struct { - Ref *corev1.RecordRef - Peer peer.AddrInfo -} - -func (h *handler) AddProvider(ctx context.Context, key []byte, prov peer.AddrInfo) error { - if err := h.handleAnnounce(ctx, key, prov); err != nil { - // log this error only - handlerLogger.Error("Failed to handle announce", "error", err) - } - - if err := h.ProviderManager.AddProvider(ctx, key, prov); err != nil { - return fmt.Errorf("failed to add provider: %w", err) - } - - return nil -} - -func (h *handler) GetProviders(ctx context.Context, key []byte) ([]peer.AddrInfo, error) { - providers, err := h.ProviderManager.GetProviders(ctx, key) - if err != nil { - return nil, fmt.Errorf("failed to get providers: %w", err) - } - - return providers, nil -} - -// handleAnnounce tries to parse the data from provider in order to update the local routing data -// about the content and peer. -// nolint:unparam -func (h *handler) handleAnnounce(ctx context.Context, key []byte, prov peer.AddrInfo) error { - keyStr := string(key) - handlerLogger.Debug("Received announcement event", "key", keyStr, "provider", prov) - - // validate if the provider is not the same as the host - if peer.ID(h.hostID) == prov.ID { - handlerLogger.Info("Ignoring announcement event from self", "provider", prov) - - return nil - } - - // All announcements are now treated as CID provider announcements - // Labels are discovered via pull-based mechanism when content is fetched - return h.handleCIDProviderAnnouncement(ctx, key, prov) -} - -// handleCIDProviderAnnouncement handles CID provider announcements (existing logic). -func (h *handler) handleCIDProviderAnnouncement(_ context.Context, key []byte, prov peer.AddrInfo) error { - // get ref cid from request - // if this fails, it may mean that it's not DIR-constructed CID - cast, err := mh.Cast(key) - if err != nil { - handlerLogger.Error("Failed to cast key to multihash", "error", err) - - return nil - } - - // create CID from multihash - ref := &corev1.RecordRef{ - Cid: cid.NewCidV1(1, cast).String(), - } - - // Validate that we have a non-empty CID - if ref.GetCid() == "" { - handlerLogger.Info("Ignoring announcement event for empty CID") - - return nil - } - - handlerLogger.Info("CID provider announcement event", "ref", ref, "provider", prov, "host", h.hostID) - - // notify the channel - h.notifyCh <- &handlerSync{ - Ref: ref, - Peer: prov, - } - - return nil -} diff --git a/server/routing/handler_test.go b/server/routing/handler_test.go deleted file mode 100644 index e2592a25c..000000000 --- a/server/routing/handler_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -//nolint:testifylint -package routing - -import ( - "testing" - "time" - - typesv1 "buf.build/gen/go/agntcy/oasf/protocolbuffers/go/agntcy/oasf/types/v1" - corev1 "github.com/agntcy/dir/api/core/v1" - "github.com/ipfs/go-cid" - "github.com/stretchr/testify/assert" -) - -// Testing 2 nodes, A -> B -// stores and announces an record. -// A discovers it retrieves the key metadata from B. -func TestHandler(t *testing.T) { - // Test data - testRecord := corev1.New(&typesv1.Record{ - Name: "test-handler-agent", - SchemaVersion: "1.0.0", - Skills: []*typesv1.Skill{ - {Name: "test_skill", Id: 1}, - }, - Locators: []*typesv1.Locator{ - {Type: "type1", Urls: []string{"url1"}}, - }, - }) - testRef := &corev1.RecordRef{Cid: testRecord.GetCid()} - - // create demo network - firstNode := newTestServer(t, t.Context(), nil) - secondNode := newTestServer(t, t.Context(), firstNode.remote.server.P2pAddrs()) - - // wait for connection - time.Sleep(2 * time.Second) - <-firstNode.remote.server.DHT().RefreshRoutingTable() - <-secondNode.remote.server.DHT().RefreshRoutingTable() - - // publish the key on second node and wait on the first - cidStr := testRef.GetCid() - decodedCID, err := cid.Decode(cidStr) - assert.NoError(t, err) - - // push the data - _, err = secondNode.remote.storeAPI.Push(t.Context(), testRecord) - assert.NoError(t, err) - - // announce the key - err = secondNode.remote.server.DHT().Provide(t.Context(), decodedCID, true) - assert.NoError(t, err) - - // wait for sync - time.Sleep(2 * time.Second) - <-firstNode.remote.server.DHT().RefreshRoutingTable() - <-secondNode.remote.server.DHT().RefreshRoutingTable() - - // check on first - found := false - - peerCh := firstNode.remote.server.DHT().FindProvidersAsync(t.Context(), decodedCID, 1) - for peer := range peerCh { - if peer.ID == secondNode.remote.server.Host().ID() { - found = true - - break - } - } - - assert.True(t, found) -} diff --git a/server/routing/internal/p2p/constants.go b/server/routing/internal/p2p/constants.go index 7d3db9594..15455a3c8 100644 --- a/server/routing/internal/p2p/constants.go +++ b/server/routing/internal/p2p/constants.go @@ -10,7 +10,7 @@ import "time" const ( // ConnMgrLowWater is the minimum number of connections to maintain. // Below this, the connection manager will not prune any peers. - // Value accounts for: DHT routing table (~20) + GossipSub mesh (~10) + buffer (~20). + // Value accounts for: DHT routing table (~20) + buffer. ConnMgrLowWater = 50 // ConnMgrHighWater is the maximum number of connections before pruning starts. @@ -23,21 +23,10 @@ const ( ConnMgrGracePeriod = 2 * time.Minute ) -// Peer priority constants for Connection Manager tagging. -// Higher values indicate higher priority and are less likely to be pruned. -const ( - // PeerPriorityBootstrap is the priority for bootstrap peers. - // Bootstrap peers are also protected (never pruned) in addition to this high priority. - PeerPriorityBootstrap = 100 - - // PeerPriorityGossipSubMesh is the priority for GossipSub mesh peers. - // Mesh peers are critical for fast label propagation and should be kept. - PeerPriorityGossipSubMesh = 50 -) - -// MeshPeerTaggingInterval defines how often GossipSub mesh peers are re-tagged -// to protect them from Connection Manager pruning as mesh topology changes. -const MeshPeerTaggingInterval = 30 * time.Second +// PeerPriorityBootstrap is the Connection Manager priority for bootstrap peers. +// Higher values are less likely to be pruned; bootstrap peers are also +// protected outright. +const PeerPriorityBootstrap = 100 // mDNS service name for local network peer discovery. // This is used to identify DIR peers on the same LAN. diff --git a/server/routing/internal/p2p/host.go b/server/routing/internal/p2p/host.go index 4939780d9..885331021 100644 --- a/server/routing/internal/p2p/host.go +++ b/server/routing/internal/p2p/host.go @@ -123,7 +123,7 @@ func newHost(listenAddr, dirAPIAddr, ociAddr string, key crypto.PrivKey, enableR // Create connection manager to limit and manage peer connections. // This prevents resource exhaustion and enables smart peer pruning based on priority. connMgr, err := connmgr.NewConnManager( - ConnMgrLowWater, // Minimum connections (DHT + GossipSub + buffer) + ConnMgrLowWater, // Minimum connections (DHT + buffer) ConnMgrHighWater, // Maximum connections (prevents resource exhaustion) connmgr.WithGracePeriod(ConnMgrGracePeriod), // Protect new connections ) diff --git a/server/routing/internal/p2p/mockrpc/rpc.go b/server/routing/internal/p2p/mockrpc/rpc.go deleted file mode 100644 index 788749e0b..000000000 --- a/server/routing/internal/p2p/mockrpc/rpc.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package mockrpc - -import ( - "context" - "fmt" - "time" - - "github.com/agntcy/dir/utils/logging" - rpc "github.com/libp2p/go-libp2p-gorpc" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/protocol" -) - -const ( - EchoService = "EchoRPCAPI" - EchoServiceFuncEcho = "Echo" -) - -var logger = logging.Logger("mockrpc") - -type EchoRPCAPI struct { - service *Service -} - -type Envelope struct { - Message string -} - -func (r *EchoRPCAPI) Echo(_ context.Context, in Envelope, out *Envelope) error { - *out = r.service.ReceiveEcho(in) - - return nil -} - -type Service struct { - rpcServer *rpc.Server - rpcClient *rpc.Client - host host.Host - protocol protocol.ID - listenCh chan<- string - ignored peerMap -} - -func Start(ctx context.Context, host host.Host, protocol protocol.ID, listenCh chan<- string, ignored []peer.AddrInfo) error { - service := &Service{ - host: host, - protocol: protocol, - listenCh: listenCh, - ignored: newPeerMap(append(peer.AddrInfosToIDs(ignored), host.ID())), - } - - err := service.SetupRPC() - if err != nil { - return err - } - - // send dummy message - go service.StartMessaging(ctx) - - return nil -} - -func (s *Service) StartMessaging(ctx context.Context) { - ticker := time.NewTicker(time.Second * 1) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.Echo(ctx, "Message: Hello from "+s.host.ID().String()) - } - } -} - -func (s *Service) SetupRPC() error { - echoRPCAPI := EchoRPCAPI{service: s} - - s.rpcServer = rpc.NewServer(s.host, s.protocol) - - err := s.rpcServer.Register(&echoRPCAPI) - if err != nil { - return err //nolint:wrapcheck - } - - s.rpcClient = rpc.NewClientWithServer(s.host, s.protocol, s.rpcServer) - - return nil -} - -func (s *Service) Echo(ctx context.Context, message string) { - peers := filterPeers(s.host.Peerstore().Peers(), s.ignored) - replies := make([]*Envelope, len(peers)) - - // Send message to all peers - errs := s.rpcClient.MultiCall( - newCtxsN(ctx, len(peers)), - peers, - EchoService, - EchoServiceFuncEcho, - Envelope{Message: message}, - copyEnvelopesToIfaces(replies), - ) - - // Check responses from peers - for i, err := range errs { - if err != nil { - logger.Error("Error calling Echo", "peer", peers[i].String(), "error", err) - } else { - logger.Info("Echoed", "peer", peers[i].String(), "message", replies[i].Message) - } - } -} - -func (s *Service) ReceiveEcho(e Envelope) Envelope { - msg := fmt.Sprintf("Peer %s echoing: %s", s.host.ID(), e.Message) - s.listenCh <- msg - - return Envelope{Message: msg} -} - -func newCtxsN(ctx context.Context, n int) []context.Context { - ctxs := make([]context.Context, 0, n) - for range n { - ctxs = append(ctxs, ctx) - } - - return ctxs -} - -func copyEnvelopesToIfaces(in []*Envelope) []any { - ifaces := make([]any, len(in)) - - for i := range in { - in[i] = &Envelope{} - ifaces[i] = in[i] - } - - return ifaces -} - -type peerMap map[peer.ID]struct{} - -func newPeerMap(peers peer.IDSlice) peerMap { - peerMap := peerMap{} - for _, peer := range peers { - peerMap[peer] = struct{}{} - } - - return peerMap -} - -func filterPeers(peers peer.IDSlice, ignored peerMap) peer.IDSlice { - var filtered peer.IDSlice - - for _, p := range peers { - if _, exists := ignored[p]; !exists { - filtered = append(filtered, p) - } - } - - return filtered -} diff --git a/server/routing/internal/p2p/mockstream/stream.go b/server/routing/internal/p2p/mockstream/stream.go deleted file mode 100644 index 265085039..000000000 --- a/server/routing/internal/p2p/mockstream/stream.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package mockstream - -import ( - "bufio" - "context" - - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/protocol" -) - -func HandleStream(ctx context.Context, listenCh chan<- string) func(s network.Stream) { - return func(s network.Stream) { - // Create a buffer stream for non-blocking read and write. - rw := bufio.NewReadWriter(bufio.NewReader(s), bufio.NewWriter(s)) - - go readData(ctx, rw, listenCh, s.Close) - go writeData(ctx, rw) - - go func() { - <-ctx.Done() - s.Close() - }() - } -} - -func StartDataStream(ctx context.Context, h host.Host, protoc string, listenCh chan<- string) { - for { - select { - case <-ctx.Done(): - return - - default: - for _, p := range h.Peerstore().Peers() { - s, err := h.NewStream(ctx, p, protocol.ID(protoc)) - if err != nil { - continue - } - - rw := bufio.NewReadWriter(bufio.NewReader(s), bufio.NewWriter(s)) - go readData(ctx, rw, listenCh, s.Close) - go writeData(ctx, rw) - } - } - } -} - -func readData(ctx context.Context, rw *bufio.ReadWriter, listenCh chan<- string, closeFn func() error) { - for { - select { - case <-ctx.Done(): - return - default: - str, _ := rw.ReadString('\n') - listenCh <- str - - _ = closeFn() - - return - } - } -} - -func writeData(ctx context.Context, rw *bufio.ReadWriter) { - for { - select { - case <-ctx.Done(): - return - default: - _, _ = rw.WriteString("hello world\n") - _ = rw.Flush() - } - } -} diff --git a/server/routing/internal/p2p/options.go b/server/routing/internal/p2p/options.go index ef4493fe2..640c9e9a3 100644 --- a/server/routing/internal/p2p/options.go +++ b/server/routing/internal/p2p/options.go @@ -12,15 +12,12 @@ import ( "time" dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p-kad-dht/records" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "golang.org/x/crypto/ssh" ) -type APIRegistrer func(host.Host) error - type options struct { Key crypto.PrivKey ListenAddress string @@ -29,8 +26,6 @@ type options struct { BootstrapPeers []peer.AddrInfo RefreshInterval time.Duration Randevous string - APIRegistrer APIRegistrer - ProviderStore records.ProviderStore DHTCustomOpts func(host.Host) ([]dht.Option, error) RelayService bool StaticRelays []peer.AddrInfo @@ -157,15 +152,6 @@ func WithRefreshInterval(period time.Duration) Option { } } -// API can only be registreded for non-bootstrap nodes. -func WithAPIRegistrer(reg APIRegistrer) Option { - return func(opts *options) error { - opts.APIRegistrer = reg - - return nil - } -} - // WithCustomDHTOpts sets custom config for DHT. // NOTE: this is app-specific, be careful when using! func WithCustomDHTOpts(dhtOptFactory func(host.Host) ([]dht.Option, error)) Option { diff --git a/server/routing/internal/p2p/server.go b/server/routing/internal/p2p/server.go index 797fe20e1..15ad6b9e4 100644 --- a/server/routing/internal/p2p/server.go +++ b/server/routing/internal/p2p/server.go @@ -173,7 +173,6 @@ func start(ctx context.Context, opts *options) <-chan status { // Peer discovery is now handled automatically by: // - DHT: Bootstrap() connects to bootstrap peers at startup // - DHT: RoutingTableRefreshPeriod() maintains routing table (every 30s) - // - GossipSub: Mesh maintenance with peer exchange (if enabled) // - Connection Manager: Maintains healthy connection count (50-200) // // The custom discover() polling loop has been removed as it was redundant @@ -191,16 +190,6 @@ func start(ctx context.Context, opts *options) <-chan status { "rendezvous", opts.Randevous) } } - // Register services. Only available on non-bootstrap nodes. - if opts.APIRegistrer != nil && len(opts.BootstrapPeers) > 0 { - err := opts.APIRegistrer(host) - if err != nil { - statusCh <- status{Err: err} - - return - } - } - // Run until context expiry logger.Debug("Host and DHT created, running routing services", "host", host.ID(), "addresses", host.Addrs()) diff --git a/server/routing/label_keys.go b/server/routing/label_keys.go new file mode 100644 index 000000000..c4e154a2b --- /dev/null +++ b/server/routing/label_keys.go @@ -0,0 +1,141 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "errors" + "fmt" + "strings" + + "github.com/agntcy/dir/server/types" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +var ( + errEmptyLabel = errors.New("cannot derive a DHT key from an empty label") + errBareNamespace = errors.New("cannot derive a DHT key from a bare namespace") +) + +// labelKey maps a label to the DHT key it is advertised under. +// +// Every node derives the same key from the same label string, which is what +// lets a searcher find providers of a label it has never seen announced. Only +// the multihash reaches the DHT (Provide hashes the CID), so the codec is +// cosmetic; what matters is that publisher and searcher hash identically. +func labelKey(label types.Label) (cid.Cid, error) { + normalized := normalizeLabel(label) + if normalized == "" { + return cid.Undef, errEmptyLabel + } + + if isBareNamespace(normalized) { + return cid.Undef, fmt.Errorf("%w: %q", errBareNamespace, normalized) + } + + hash, err := mh.Sum([]byte(normalized), mh.SHA2_256, -1) + if err != nil { + return cid.Undef, fmt.Errorf("failed to hash label %q: %w", normalized, err) + } + + return cid.NewCidV1(cid.Raw, hash), nil +} + +// expandLabel returns a label together with its ancestors, closest first: +// "/skills/A/B/C" yields "/skills/A/B/C", "/skills/A/B", "/skills/A". +// +// Ancestors are what make prefix search work: a record tagged "/skills/A/B" is +// only findable under "/skills/A" if its holder also advertises that key. +// +// The bare namespace ("/skills") is deliberately excluded. Every node holding +// any skill at all would provide it, so it selects nothing while attracting +// every provider record in the network onto one set of custodians. +// +// Labels outside the known namespaces are returned as-is; their structure is +// not ours to interpret. +func expandLabel(label types.Label) []types.Label { + normalized := types.Label(normalizeLabel(label)) + if normalized == "" || isBareNamespace(normalized.String()) { + return nil + } + + namespace := normalized.Namespace() + if namespace == "" { + return []types.Label{normalized} + } + + segments := splitNonEmpty(normalized.Value()) + if len(segments) == 0 { + return nil + } + + expanded := make([]types.Label, 0, len(segments)) + + for i := len(segments); i > 0; i-- { + expanded = append(expanded, types.Label(namespace+strings.Join(segments[:i], "/"))) + } + + return expanded +} + +// expandLabels returns the distinct union of expandLabel over every label, +// preserving first-seen order. Records routinely share ancestors, so the +// deduplicated set is far smaller than the sum of the individual expansions. +func expandLabels(labels []types.Label) []types.Label { + seen := make(map[types.Label]struct{}, len(labels)) + distinct := make([]types.Label, 0, len(labels)) + + for _, label := range labels { + for _, expanded := range expandLabel(label) { + if _, ok := seen[expanded]; ok { + continue + } + + seen[expanded] = struct{}{} + + distinct = append(distinct, expanded) + } + } + + return distinct +} + +// isBareNamespace reports whether a normalized label is just a namespace root +// such as "/skills". Such a key matches every record that carries any label of +// that kind, so it discriminates nothing while drawing the entire network's +// provider records onto a single set of custodians. +func isBareNamespace(normalized string) bool { + for _, labelType := range types.AllLabelTypes() { + if normalized == "/"+labelType.String() { + return true + } + } + + return false +} + +// normalizeLabel trims surrounding whitespace and any trailing slash, so +// "/skills/A" and "/skills/A/" resolve to the same DHT key. +func normalizeLabel(label types.Label) string { + trimmed := strings.TrimSpace(label.String()) + if trimmed == "/" { + return "" + } + + return strings.TrimSuffix(trimmed, "/") +} + +// splitNonEmpty splits on "/" and drops empty segments, so a label containing +// a doubled slash does not produce a key with an empty path component. +func splitNonEmpty(value string) []string { + segments := make([]string, 0, strings.Count(value, "/")+1) + + for segment := range strings.SplitSeq(value, "/") { + if segment != "" { + segments = append(segments, segment) + } + } + + return segments +} diff --git a/server/routing/label_keys_test.go b/server/routing/label_keys_test.go new file mode 100644 index 000000000..e4e92dba7 --- /dev/null +++ b/server/routing/label_keys_test.go @@ -0,0 +1,167 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "testing" + + "github.com/agntcy/dir/server/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLabelKeyIsDeterministic(t *testing.T) { + t.Parallel() + + first, err := labelKey(types.Label("/skills/AI/ML")) + require.NoError(t, err) + + second, err := labelKey(types.Label("/skills/AI/ML")) + require.NoError(t, err) + + assert.Equal(t, first, second, "the same label must always resolve to the same DHT key") +} + +func TestLabelKeyIgnoresTrailingSlashAndWhitespace(t *testing.T) { + t.Parallel() + + canonical, err := labelKey(types.Label("/skills/AI/ML")) + require.NoError(t, err) + + for _, variant := range []string{"/skills/AI/ML/", " /skills/AI/ML ", "\t/skills/AI/ML/\n"} { + key, err := labelKey(types.Label(variant)) + require.NoError(t, err, variant) + assert.Equal(t, canonical, key, "variant %q must match the canonical key", variant) + } +} + +func TestLabelKeySeparatesDistinctLabels(t *testing.T) { + t.Parallel() + + skill, err := labelKey(types.Label("/skills/AI")) + require.NoError(t, err) + + domain, err := labelKey(types.Label("/domains/AI")) + require.NoError(t, err) + + assert.NotEqual(t, skill, domain, "the same value in different namespaces must not collide") +} + +func TestLabelKeyRejectsEmptyLabel(t *testing.T) { + t.Parallel() + + for _, empty := range []string{"", " ", "/"} { + _, err := labelKey(types.Label(empty)) + require.ErrorIs(t, err, errEmptyLabel, "expected an error for %q", empty) + } +} + +// A bare namespace matches everything, so it is not a usable key on either the +// publish or the search side. +func TestLabelKeyRejectsBareNamespace(t *testing.T) { + t.Parallel() + + for _, namespace := range []string{"/skills", "/skills/", "/domains", "/modules", "/locators"} { + _, err := labelKey(types.Label(namespace)) + require.ErrorIs(t, err, errBareNamespace, "expected an error for %q", namespace) + } +} + +func TestExpandLabel(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + label string + want []types.Label + }{ + { + name: "nested skill yields ancestors, closest first", + label: "/skills/AI/ML/NLP", + want: []types.Label{"/skills/AI/ML/NLP", "/skills/AI/ML", "/skills/AI"}, + }, + { + name: "single segment has no ancestors", + label: "/skills/AI", + want: []types.Label{"/skills/AI"}, + }, + { + name: "bare namespace expands to nothing", + label: "/skills", + want: nil, + }, + { + name: "namespace with trailing slash expands to nothing", + label: "/skills/", + want: nil, + }, + { + name: "locators expand like any other namespace", + label: "/locators/docker-image", + want: []types.Label{"/locators/docker-image"}, + }, + { + name: "doubled slashes do not produce empty segments", + label: "/domains/finance//banking", + want: []types.Label{"/domains/finance/banking", "/domains/finance"}, + }, + { + name: "unknown namespace is returned untouched", + label: "/something/else/entirely", + want: []types.Label{"/something/else/entirely"}, + }, + { + name: "empty label expands to nothing", + label: "", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, expandLabel(types.Label(tt.label))) + }) + } +} + +// Ancestor expansion must stop below the namespace root, however deep the +// label is. +func TestExpandLabelStopsBelowNamespaceRoot(t *testing.T) { + t.Parallel() + + roots := []types.Label{"/skills", "/domains", "/modules", "/locators"} + + for _, label := range []string{"/skills/a/b/c/d", "/domains/finance/banking", "/modules/a/b"} { + for _, got := range expandLabel(types.Label(label)) { + assert.NotContains(t, roots, got, "expansion of %q must not reach a namespace root", label) + } + } +} + +func TestExpandLabelsDeduplicatesSharedAncestors(t *testing.T) { + t.Parallel() + + expanded := expandLabels([]types.Label{ + "/skills/AI/ML", + "/skills/AI/NLP", + "/skills/AI", + "/domains/finance", + }) + + assert.Equal(t, []types.Label{ + "/skills/AI/ML", + "/skills/AI", + "/skills/AI/NLP", + "/domains/finance", + }, expanded) +} + +func TestExpandLabelsHandlesEmptyInput(t *testing.T) { + t.Parallel() + + assert.Empty(t, expandLabels(nil)) + assert.Empty(t, expandLabels([]types.Label{"", "/"})) +} diff --git a/server/routing/label_utils.go b/server/routing/label_utils.go deleted file mode 100644 index 757ff4aca..000000000 --- a/server/routing/label_utils.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "errors" - "fmt" - "strings" - - "github.com/agntcy/dir/server/types" -) - -// Key manipulation utilities for routing operations. -// These functions handle the enhanced label key format: /namespace/value/CID/PeerID - -// Example: Label("/skills/AI/ML") β†’ "/skills/AI/ML/CID123/Peer1". -func BuildEnhancedLabelKey(label types.Label, cid, peerID string) string { - return fmt.Sprintf("%s/%s/%s", label.String(), cid, peerID) -} - -// Example: "/skills/AI/ML/CID123/Peer1" β†’ (Label("/skills/AI/ML"), "CID123", "Peer1", nil). -func ParseEnhancedLabelKey(key string) (types.Label, string, string, error) { - labelStr, cid, peerID, err := parseEnhancedLabelKeyInternal(key) - if err != nil { - return types.Label(""), "", "", err - } - - return types.Label(labelStr), cid, peerID, nil -} - -// parseEnhancedLabelKeyInternal contains the actual parsing logic. -// This is used internally by ParseEnhancedLabelKey. -func parseEnhancedLabelKeyInternal(key string) (string, string, string, error) { - if !strings.HasPrefix(key, "/") { - return "", "", "", errors.New("key must start with /") - } - - parts := strings.Split(key, "/") - if len(parts) < types.MinLabelKeyParts { - return "", "", "", errors.New("key must have at least namespace/path/CID/PeerID") - } - - // Extract PeerID (last part) and CID (second to last part) - peerID := parts[len(parts)-1] - cid := parts[len(parts)-2] - - // Extract label (everything except the last two parts) - labelParts := parts[1 : len(parts)-2] // Skip empty first part and last two parts - label := "/" + strings.Join(labelParts, "/") - - return label, cid, peerID, nil -} - -// ExtractPeerIDFromKey extracts just the PeerID from a self-descriptive key. -func ExtractPeerIDFromKey(key string) string { - parts := strings.Split(key, "/") - if len(parts) < types.MinLabelKeyParts { - return "" - } - - return parts[len(parts)-1] -} - -// IsValidLabelKey checks if a key starts with any valid label type prefix. -// Returns true if the key starts with /skills/, /domains/, /features/, or /locators/. -func IsValidLabelKey(key string) bool { - for _, labelType := range types.AllLabelTypes() { - if strings.HasPrefix(key, labelType.Prefix()) { - return true - } - } - - return false -} - -// GetLabelTypeFromKey extracts the label type from a key. -// Returns the label type and true if found, or LabelTypeUnknown and false if not found. -func GetLabelTypeFromKey(key string) (types.LabelType, bool) { - for _, labelType := range types.AllLabelTypes() { - if strings.HasPrefix(key, labelType.Prefix()) { - return labelType, true - } - } - - return types.LabelTypeUnknown, false -} diff --git a/server/routing/label_utils_test.go b/server/routing/label_utils_test.go deleted file mode 100644 index 8d801acee..000000000 --- a/server/routing/label_utils_test.go +++ /dev/null @@ -1,420 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "testing" - - "github.com/agntcy/dir/server/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBuildEnhancedLabelKey(t *testing.T) { - testCases := []struct { - name string - label types.Label - cid string - peerID string - expected string - }{ - { - name: "skill_label", - label: types.Label("/skills/AI/ML"), - cid: "CID123", - peerID: "Peer1", - expected: "/skills/AI/ML/CID123/Peer1", - }, - { - name: "domain_label", - label: types.Label("/domains/research"), - cid: "CID456", - peerID: "Peer2", - expected: "/domains/research/CID456/Peer2", - }, - { - name: "module_label", - label: types.Label("/modules/runtime/model"), - cid: "CID789", - peerID: "Peer3", - expected: "/modules/runtime/model/CID789/Peer3", - }, - { - name: "locator_label", - label: types.Label("/locators/docker-image"), - cid: "CID999", - peerID: "Peer4", - expected: "/locators/docker-image/CID999/Peer4", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := BuildEnhancedLabelKey(tc.label, tc.cid, tc.peerID) - assert.Equal(t, tc.expected, result) - }) - } -} - -func TestParseEnhancedLabelKey(t *testing.T) { - testCases := []struct { - name string - key string - expectedLabel types.Label - expectedCID string - expectedPeer string - expectError bool - errorMsg string - }{ - { - name: "valid_skill_key", - key: "/skills/AI/ML/CID123/Peer1", - expectedLabel: types.Label("/skills/AI/ML"), - expectedCID: "CID123", - expectedPeer: "Peer1", - expectError: false, - }, - { - name: "valid_domain_key", - key: "/domains/research/healthcare/CID456/Peer2", - expectedLabel: types.Label("/domains/research/healthcare"), - expectedCID: "CID456", - expectedPeer: "Peer2", - expectError: false, - }, - { - name: "valid_module_key", - key: "/modules/runtime/model/CID789/Peer3", - expectedLabel: types.Label("/modules/runtime/model"), - expectedCID: "CID789", - expectedPeer: "Peer3", - expectError: false, - }, - { - name: "invalid_no_leading_slash", - key: "skills/AI/ML/CID123/Peer1", - expectError: true, - errorMsg: "key must start with /", - }, - { - name: "invalid_too_few_parts", - key: "/skills/AI", - expectError: true, - errorMsg: "key must have at least namespace/path/CID/PeerID", - }, - { - name: "minimal_valid_key", - key: "/skills/AI/CID123/Peer1", - expectedLabel: types.Label("/skills/AI"), - expectedCID: "CID123", - expectedPeer: "Peer1", - expectError: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - label, cid, peerID, err := ParseEnhancedLabelKey(tc.key) - - if tc.expectError { - require.Error(t, err) - assert.Contains(t, err.Error(), tc.errorMsg) - assert.Equal(t, types.Label(""), label) - assert.Empty(t, cid) - assert.Empty(t, peerID) - } else { - require.NoError(t, err) - assert.Equal(t, tc.expectedLabel, label) - assert.Equal(t, tc.expectedCID, cid) - assert.Equal(t, tc.expectedPeer, peerID) - } - }) - } -} - -func TestExtractPeerIDFromKey(t *testing.T) { - testCases := []struct { - name string - key string - expectedPeer string - }{ - { - name: "valid_key", - key: "/skills/AI/ML/CID123/Peer1", - expectedPeer: "Peer1", - }, - { - name: "complex_label", - key: "/domains/research/healthcare/informatics/CID456/Peer2", - expectedPeer: "Peer2", - }, - { - name: "too_few_parts", - key: "/skills/AI", - expectedPeer: "", - }, - { - name: "empty_key", - key: "", - expectedPeer: "", - }, - { - name: "single_slash", - key: "/", - expectedPeer: "", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := ExtractPeerIDFromKey(tc.key) - assert.Equal(t, tc.expectedPeer, result) - }) - } -} - -func TestIsValidLabelKey(t *testing.T) { - testCases := []struct { - name string - key string - expected bool - }{ - // Valid keys - { - name: "valid_skill_key", - key: "/skills/AI/ML/CID123/Peer1", - expected: true, - }, - { - name: "valid_domain_key", - key: "/domains/research/CID123/Peer1", - expected: true, - }, - { - name: "valid_module_key", - key: "/modules/runtime/CID123/Peer1", - expected: true, - }, - { - name: "valid_locator_key", - key: "/locators/docker-image/CID123/Peer1", - expected: true, - }, - // Invalid keys - { - name: "invalid_namespace", - key: "/invalid/test/CID123/Peer1", - expected: false, - }, - { - name: "records_namespace", - key: "/records/CID123", - expected: false, - }, - { - name: "no_leading_slash", - key: "skills/AI/CID123/Peer1", - expected: false, - }, - { - name: "empty_key", - key: "", - expected: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := IsValidLabelKey(tc.key) - assert.Equal(t, tc.expected, result) - }) - } -} - -func TestGetLabelTypeFromKey(t *testing.T) { - testCases := []struct { - name string - key string - expectedType types.LabelType - expectedOK bool - }{ - { - name: "skill_key", - key: "/skills/AI/ML/CID123/Peer1", - expectedType: types.LabelTypeSkill, - expectedOK: true, - }, - { - name: "domain_key", - key: "/domains/research/CID123/Peer1", - expectedType: types.LabelTypeDomain, - expectedOK: true, - }, - { - name: "module_key", - key: "/modules/runtime/CID123/Peer1", - expectedType: types.LabelTypeModule, - expectedOK: true, - }, - { - name: "locator_key", - key: "/locators/docker-image/CID123/Peer1", - expectedType: types.LabelTypeLocator, - expectedOK: true, - }, - { - name: "invalid_key", - key: "/invalid/test/CID123/Peer1", - expectedType: types.LabelTypeUnknown, - expectedOK: false, - }, - { - name: "records_key", - key: "/records/CID123", - expectedType: types.LabelTypeUnknown, - expectedOK: false, - }, - { - name: "empty_key", - key: "", - expectedType: types.LabelTypeUnknown, - expectedOK: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - labelType, ok := GetLabelTypeFromKey(tc.key) - assert.Equal(t, tc.expectedType, labelType) - assert.Equal(t, tc.expectedOK, ok) - }) - } -} - -func TestParseEnhancedLabelKeyInternal(t *testing.T) { - testCases := []struct { - name string - key string - expectedLabel string - expectedCID string - expectedPeer string - expectError bool - errorMsg string - }{ - { - name: "valid_simple_key", - key: "/skills/AI/CID123/Peer1", - expectedLabel: "/skills/AI", - expectedCID: "CID123", - expectedPeer: "Peer1", - expectError: false, - }, - { - name: "valid_complex_key", - key: "/modules/runtime/model/security/CID456/Peer2", - expectedLabel: "/modules/runtime/model/security", - expectedCID: "CID456", - expectedPeer: "Peer2", - expectError: false, - }, - { - name: "no_leading_slash", - key: "skills/AI/CID123/Peer1", - expectError: true, - errorMsg: "key must start with /", - }, - { - name: "too_few_parts", - key: "/skills/AI", - expectError: true, - errorMsg: "key must have at least namespace/path/CID/PeerID", - }, - { - name: "exactly_min_parts", - key: "/skills/AI/CID123/Peer1", - expectedLabel: "/skills/AI", - expectedCID: "CID123", - expectedPeer: "Peer1", - expectError: false, - }, - { - name: "empty_key", - key: "", - expectError: true, - errorMsg: "key must start with /", - }, - { - name: "only_slash", - key: "/", - expectError: true, - errorMsg: "key must have at least namespace/path/CID/PeerID", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - label, cid, peerID, err := parseEnhancedLabelKeyInternal(tc.key) - - if tc.expectError { - require.Error(t, err) - assert.Contains(t, err.Error(), tc.errorMsg) - assert.Empty(t, label) - assert.Empty(t, cid) - assert.Empty(t, peerID) - } else { - require.NoError(t, err) - assert.Equal(t, tc.expectedLabel, label) - assert.Equal(t, tc.expectedCID, cid) - assert.Equal(t, tc.expectedPeer, peerID) - } - }) - } -} - -func TestParseEnhancedLabelKey_RoundTrip(t *testing.T) { - // Test that BuildEnhancedLabelKey and ParseEnhancedLabelKey are inverse operations - testCases := []struct { - label types.Label - cid string - peerID string - }{ - {types.Label("/skills/AI/ML"), "CID123", "Peer1"}, - {types.Label("/domains/research"), "CID456", "Peer2"}, - {types.Label("/modules/runtime/model/security"), "CID789", "Peer3"}, - {types.Label("/locators/docker-image"), "CID999", "Peer4"}, - } - - for _, tc := range testCases { - t.Run(tc.label.String(), func(t *testing.T) { - // Build key - key := BuildEnhancedLabelKey(tc.label, tc.cid, tc.peerID) - - // Parse it back - parsedLabel, parsedCID, parsedPeer, err := ParseEnhancedLabelKey(key) - - require.NoError(t, err) - assert.Equal(t, tc.label, parsedLabel) - assert.Equal(t, tc.cid, parsedCID) - assert.Equal(t, tc.peerID, parsedPeer) - }) - } -} - -func BenchmarkBuildEnhancedLabelKey(b *testing.B) { - label := types.Label("/skills/AI/ML") - cid := "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" - peerID := "12D3KooWBhvJH9k6u7S5Q8Z8u7S5Q8Z8u7S5Q8Z8u7S5Q8Z8u7S5Q8" - - for b.Loop() { - _ = BuildEnhancedLabelKey(label, cid, peerID) - } -} - -func BenchmarkParseEnhancedLabelKey(b *testing.B) { - key := "/skills/AI/ML/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/12D3KooWBhvJH9k6u7S5Q8Z8u7S5Q8Z8u7S5Q8Z8u7S5Q8Z8u7S5Q8" - - for b.Loop() { - _, _, _, _ = ParseEnhancedLabelKey(key) - } -} diff --git a/server/routing/metrics.go b/server/routing/metrics.go deleted file mode 100644 index c06be11f7..000000000 --- a/server/routing/metrics.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -// Package routing provides label frequency metrics for operational monitoring. -// -// The Metrics system tracks how many records are associated with each label -// (skills, domains, features) on the local peer. This data is persisted to -// the datastore and can be used for: -// -// - Operational monitoring and dashboards -// - Debugging label distribution issues -// - Future query optimization features -// - Administrative APIs and tooling -// -// Metrics are automatically maintained during Publish/Unpublish operations -// and stored at the "/metrics" datastore key in JSON format. -package routing - -import ( - "context" - "encoding/json" - "errors" - "fmt" - - "github.com/agntcy/dir/server/types" - "github.com/ipfs/go-datastore" -) - -// LabelMetric represents the frequency count for a specific label. -type LabelMetric struct { - Name string `json:"name"` // Full label name (e.g., "/skills/AI/ML", "/domains/research") - Total uint64 `json:"total"` // Number of local records that have this label -} - -// Metrics tracks label frequency distribution for operational monitoring. -// This provides visibility into what types of records this peer is providing -// and can be used for debugging, monitoring, and future optimization features. -type Metrics struct { - Data map[string]LabelMetric `json:"data"` // Map of label name β†’ frequency count -} - -func (m *Metrics) increment(label types.Label) { - labelStr := label.String() - if _, ok := m.Data[labelStr]; !ok { - m.Data[labelStr] = LabelMetric{ - Name: labelStr, - Total: 0, - } - } - - m.Data[labelStr] = LabelMetric{ - Name: labelStr, - Total: m.Data[labelStr].Total + 1, - } -} - -func (m *Metrics) decrement(label types.Label) { - labelStr := label.String() - if _, ok := m.Data[labelStr]; !ok { - return - } - - currentTotal := m.Data[labelStr].Total - if currentTotal > 0 { - m.Data[labelStr] = LabelMetric{ - Name: labelStr, - Total: currentTotal - 1, - } - } - - // Remove the label from the map if the total is zero. - if m.Data[labelStr].Total == 0 { - delete(m.Data, labelStr) - } -} - -// NOTE: counts() method removed as it's no longer used in the new List API -// The new ListResponse doesn't include label_counts field for simplicity - -// NOTE: labels() method removed as it's no longer used in the new List API -// The new List API doesn't return peer statistics for empty requests - -func (m *Metrics) update(ctx context.Context, dstore types.Datastore) error { - data, err := json.Marshal(m) - if err != nil { - return fmt.Errorf("failed to marshal metrics data: %w", err) - } - - err = dstore.Put(ctx, datastore.NewKey("/metrics"), data) - if err != nil { - return fmt.Errorf("failed to update metrics data: %w", err) - } - - return nil -} - -func loadMetrics(ctx context.Context, dstore types.Datastore) (*Metrics, error) { - // Fetch metrics data - data, err := dstore.Get(ctx, datastore.NewKey("/metrics")) - if err != nil { - if errors.Is(err, datastore.ErrNotFound) { - return &Metrics{ - Data: make(map[string]LabelMetric), - }, nil - } - - return nil, fmt.Errorf("failed to update metrics data: %w", err) - } - - // Parse existing metrics data - var metrics Metrics - if err := json.Unmarshal(data, &metrics); err != nil { - return nil, fmt.Errorf("failed to unmarshal metrics data: %w", err) - } - - return &metrics, nil -} diff --git a/server/routing/pubsub/constants.go b/server/routing/pubsub/constants.go deleted file mode 100644 index 1244f230d..000000000 --- a/server/routing/pubsub/constants.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package pubsub - -// Protocol constants for GossipSub label announcements. -// These values are INTENTIONALLY NOT CONFIGURABLE to ensure network-wide compatibility. -// All peers must use the same values to communicate properly. -// -// Rationale: -// - Different topics β†’ peers can't discover each other's labels -// - Different message sizes β†’ messages may be rejected -// - Different label limits β†’ validation inconsistencies -// -// If protocol changes are needed, increment the topic version (e.g., "dir/labels/v2") -// and coordinate the upgrade across all peers. -const ( - // TopicLabels is the GossipSub topic for label announcements. - // All peers must subscribe to the same topic to communicate. - // Versioned to allow future protocol changes (e.g., "dir/labels/v2"). - TopicLabels = "dir/labels/v1" - - // MaxMessageSize is the maximum size of label announcement messages. - // This prevents abuse and ensures all peers can process messages. - // 10KB allows ~100 labels with reasonable overhead. - MaxMessageSize = 10 * 1024 // 10KB - - // MaxLabelsPerAnnouncement is the maximum number of labels per announcement. - // This prevents abuse from malicious peers. - // 100 labels is generous for typical records. - MaxLabelsPerAnnouncement = 100 -) diff --git a/server/routing/pubsub/events.go b/server/routing/pubsub/events.go deleted file mode 100644 index c7276e295..000000000 --- a/server/routing/pubsub/events.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package pubsub - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "time" - - coretypes "github.com/agntcy/dir/api/core/types" -) - -// PublishEventHandler is a callback function type for handling record publication events. -// This is used for dependency injection to allow components to trigger publishing -// operations without creating circular dependencies. -// -// The handler should: -// - Accept a types.Record interface (caller must wrap concrete types with adapters) -// - Announce the record to DHT -// - Publish the record's labels via GossipSub -// - Handle errors appropriately -// -// Example usage: -// -// // In routing_remote.go: -// cleanupManager := NewCleanupManager(..., routeAPI.Publish) -// -// // In cleanup_tasks.go: -// type CleanupManager struct { -// publishFunc pubsub.PublishEventHandler -// } -type PublishEventHandler func(context.Context, coretypes.Record) error - -// RecordPublishEvent is the wire format for record publication announcements via GossipSub. -// This is a minimal structure optimized for network efficiency. -// -// Protocol parameters: See constants.go for TopicLabels, MaxMessageSize, etc. -// These are intentionally NOT configurable to ensure network-wide compatibility. -// -// Security Note: -// - PeerID is NOT included in the wire format to prevent spoofing -// - Instead, the authenticated sender (msg.ReceivedFrom) is passed separately to handlers -// - This ensures only cryptographically verified peer IDs are used for storage -// -// Conversion to storage format: -// - Wire: RecordPublishEvent with []string labels -// - Handler receives: authenticated PeerID from libp2p transport -// - Storage: Enhanced keys (/skills/AI/CID/PeerID) with types.LabelMetadata -// -// Example wire format: -// -// { -// "cid": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", -// "labels": ["/skills/AI/ML", "/domains/research", "/modules/tensorflow"], -// "timestamp": "2025-10-01T10:00:00Z" -// } -type RecordPublishEvent struct { - // CID is the content identifier of the record. - // This uniquely identifies the record being announced. - CID string `json:"cid"` - - // Labels is the list of label strings extracted from the record. - // Format: namespace-prefixed paths (e.g., "/skills/AI/ML") - // These will be converted to types.Label type upon receipt. - Labels []string `json:"labels"` - - // Timestamp is when this announcement was created. - // This becomes the types.LabelMetadata.Timestamp field. - Timestamp time.Time `json:"timestamp"` -} - -// Validate checks if the event is well-formed and safe to process. -// This prevents malformed or malicious events from being processed. -// -// Note: PeerID validation is intentionally omitted as it's provided -// separately by the authenticated libp2p transport layer (msg.ReceivedFrom). -func (e *RecordPublishEvent) Validate() error { - if e.CID == "" { - return errors.New("missing CID") - } - - if len(e.Labels) == 0 { - return errors.New("no labels provided") - } - - if len(e.Labels) > MaxLabelsPerAnnouncement { - return errors.New("too many labels") - } - - if e.Timestamp.IsZero() { - return errors.New("missing timestamp") - } - - return nil -} - -// Marshal serializes the event to JSON for network transmission. -func (e *RecordPublishEvent) Marshal() ([]byte, error) { - data, err := json.Marshal(e) - if err != nil { - return nil, fmt.Errorf("failed to marshal record publish event: %w", err) - } - - // Validate size to prevent oversized messages - if len(data) > MaxMessageSize { - return nil, errors.New("event exceeds maximum size") - } - - return data, nil -} - -// UnmarshalRecordPublishEvent deserializes and validates a record publish event. -// This is the entry point for processing received GossipSub messages. -func UnmarshalRecordPublishEvent(data []byte) (*RecordPublishEvent, error) { - // Check size before unmarshaling to prevent resource exhaustion - if len(data) > MaxMessageSize { - return nil, errors.New("event exceeds maximum size") - } - - var event RecordPublishEvent - if err := json.Unmarshal(data, &event); err != nil { - return nil, fmt.Errorf("failed to unmarshal record publish event: %w", err) - } - - // Validate after unmarshaling to ensure well-formed data - if err := event.Validate(); err != nil { - return nil, err - } - - return &event, nil -} diff --git a/server/routing/pubsub/manager.go b/server/routing/pubsub/manager.go deleted file mode 100644 index 608d8b8a9..000000000 --- a/server/routing/pubsub/manager.go +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package pubsub - -import ( - "context" - "errors" - "fmt" - "time" - - coretypes "github.com/agntcy/dir/api/core/types" - "github.com/agntcy/dir/server/routing/internal/p2p" - "github.com/agntcy/dir/server/types" - "github.com/agntcy/dir/utils/logging" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/libp2p/go-libp2p/core/host" -) - -var logger = logging.Logger("routing/pubsub") - -// Manager handles GossipSub operations for label announcements. -// It provides efficient label propagation across the network without -// requiring peers to pull entire records. -// -// Architecture: -// - Publisher: Announces labels when storing records -// - Subscriber: Receives and caches labels from remote peers -// - Integration: Works alongside DHT for resilient discovery -// -// Performance: -// - Propagation: ~5-20ms (vs DHT's ~100-500ms) -// - Bandwidth: ~100B per announcement (vs KB-MB for full record pull) -// - Reach: ALL subscribed peers (vs DHT's k-closest peers) -type Manager struct { - ctx context.Context //nolint:containedctx // Needed for long-running message handler goroutine - host host.Host - pubsub *pubsub.PubSub - topic *pubsub.Topic - sub *pubsub.Subscription - localPeerID string - topicName string // Topic name (protocol constant) - - // Callback invoked when record publish event is received. - // Parameters: - // - context.Context: Operation context - // - string: Authenticated peer ID (from msg.ReceivedFrom, cryptographically verified) - // - *RecordPublishEvent: The announcement payload - onRecordPublishEvent func(context.Context, string, *RecordPublishEvent) -} - -// New creates a new GossipSub manager for label announcements. -// This initializes the GossipSub router, joins the labels topic, and -// starts the message handler goroutine. -// -// Protocol parameters (TopicLabels, MaxMessageSize) are defined in constants.go -// and are intentionally NOT configurable to ensure network-wide compatibility. -// -// Parameters: -// - ctx: Context for lifecycle management -// - h: libp2p host for network operations -// -// Returns: -// - *Manager: Initialized manager ready for use -// - error: If GossipSub setup fails -func New(ctx context.Context, h host.Host) (*Manager, error) { - // Create GossipSub with protocol-defined settings - ps, err := pubsub.NewGossipSub( - ctx, - h, - // Enable peer exchange for better peer discovery - pubsub.WithPeerExchange(true), - // Limit message size to protocol-defined maximum - pubsub.WithMaxMessageSize(MaxMessageSize), - ) - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub: %w", err) - } - - // Join the protocol-defined topic - topic, err := ps.Join(TopicLabels) - if err != nil { - return nil, fmt.Errorf("failed to join labels topic %q: %w", TopicLabels, err) - } - - // Subscribe to receive label announcements - sub, err := topic.Subscribe() - if err != nil { - return nil, fmt.Errorf("failed to subscribe to labels topic %q: %w", TopicLabels, err) - } - - manager := &Manager{ - ctx: ctx, - host: h, - pubsub: ps, - topic: topic, - sub: sub, - localPeerID: h.ID().String(), - topicName: TopicLabels, - } - - // Start message handler goroutine - go manager.handleMessages() - - logger.Info("GossipSub manager initialized", - "topic", TopicLabels, - "maxMessageSize", MaxMessageSize, - "peerID", manager.localPeerID) - - return manager, nil -} - -// PublishRecord announces a record's labels to the network. -// This is called when a record is stored locally and should be -// discoverable by remote peers. -// -// Flow: -// 1. Extract CID and labels from record -// 2. Convert types.Label to wire format ([]string) -// 3. Create and validate RecordPublishEvent -// 4. Publish to GossipSub topic -// 5. GossipSub mesh propagates to all subscribed peers -// -// Parameters: -// - ctx: Context for operation timeout/cancellation -// - record: The record interface (caller must wrap concrete types with adapter) -// -// Returns: -// - error: If validation or publishing fails -// -// Note: This is non-blocking. GossipSub handles propagation asynchronously. -func (m *Manager) PublishRecord(ctx context.Context, record coretypes.Record) error { - if record == nil { - return errors.New("record is nil") - } - - // Extract CID from record - cid := record.GetCid() - if cid == "" { - return errors.New("record has no CID") - } - - // Extract labels from record (uses shared label extraction logic) - labelList := types.GetLabelsFromRecord(record) - if len(labelList) == 0 { - // No labels to publish (not an error, just nothing to do) - logger.Debug("Record has no labels, skipping GossipSub announcement", "cid", cid) - - return nil - } - - // Convert types.Label to strings for wire format - labelStrings := make([]string, len(labelList)) - for i, label := range labelList { - labelStrings[i] = label.String() - } - - // Create announcement with current timestamp - // Note: PeerID is not included in the wire format - recipients use - // the authenticated msg.ReceivedFrom from libp2p transport layer - announcement := &RecordPublishEvent{ - CID: cid, - Labels: labelStrings, - Timestamp: time.Now(), - } - - // Validate before publishing to catch issues early - if err := announcement.Validate(); err != nil { - return fmt.Errorf("invalid announcement: %w", err) - } - - // Serialize to JSON - data, err := announcement.Marshal() - if err != nil { - return fmt.Errorf("failed to marshal announcement: %w", err) - } - - // Publish to GossipSub topic - if err := m.topic.Publish(ctx, data); err != nil { - return fmt.Errorf("failed to publish announcement: %w", err) - } - - logger.Info("Published record announcement", - "cid", cid, - "labels", len(labelList), - "topicPeers", len(m.topic.ListPeers()), - "size", len(data)) - - return nil -} - -// SetOnRecordPublishEvent sets the callback for received record publication events. -// This callback is invoked for each valid announcement received from remote peers. -// -// The callback receives: -// - ctx: Operation context -// - authenticatedPeerID: The peer's ID from msg.ReceivedFrom (cryptographically verified) -// - event: The announcement payload -// -// The callback should: -// - Convert wire format ([]string) to labels.Label -// - Build enhanced keys using BuildEnhancedLabelKey() with authenticatedPeerID -// - Store labels.LabelMetadata in datastore -// -// Security Note: Always use authenticatedPeerID (not any ID from the event payload) -// as it's verified by libp2p's cryptographic transport layer. -// -// Example: -// -// manager.SetOnRecordPublishEvent(func(ctx context.Context, authenticatedPeerID string, event *RecordPublishEvent) { -// for _, labelStr := range event.Labels { -// label := labels.Label(labelStr) -// key := BuildEnhancedLabelKey(label, event.CID, authenticatedPeerID) -// // ... store in datastore ... -// } -// }) -func (m *Manager) SetOnRecordPublishEvent(fn func(context.Context, string, *RecordPublishEvent)) { - m.onRecordPublishEvent = fn -} - -// handleMessages is the main message processing loop. -// It runs in a goroutine and processes all incoming label announcements. -// -// Flow: -// 1. Wait for next message from subscription -// 2. Skip own messages (already cached locally) -// 3. Unmarshal and validate announcement -// 4. Invoke callback for processing -// -// Error handling: -// - Context cancellation: Normal shutdown, exit loop -// - Subscription cancelled: Normal shutdown, exit loop -// - Invalid messages: Log warning, continue processing -// - Unmarshal errors: Log warning, continue processing -// -// This goroutine runs for the lifetime of the Manager. -func (m *Manager) handleMessages() { - for { - msg, err := m.sub.Next(m.ctx) - if err != nil { - // Check if context was cancelled (normal shutdown) - if m.ctx.Err() != nil { - logger.Debug("Message handler stopping", "reason", "context_cancelled") - - return - } - - // Check if subscription was cancelled (happens during shutdown) - // This prevents error spam during graceful shutdown - if errors.Is(err, context.Canceled) || err.Error() == "subscription cancelled" { - logger.Debug("Message handler stopping", "reason", "subscription_cancelled") - - return - } - - // Log error but continue processing - logger.Error("Error reading from labels topic", "error", err) - - continue - } - - // Skip our own messages (we already cached labels locally). - // Use the signed author (GetFrom), not the forwarder (ReceivedFrom). - if msg.GetFrom() == m.host.ID() { - continue - } - - // Parse and validate announcement - announcement, err := UnmarshalRecordPublishEvent(msg.Data) - if err != nil { - logger.Warn("Received invalid label announcement", - "from", msg.GetFrom(), - "error", err, - "size", len(msg.Data)) - - continue - } - - // Extract the authenticated ORIGINAL author from the signed message - // (GetFrom is verified via GossipSub message signing). This is the record - // publisher, not the mesh forwarder (ReceivedFrom) β€” important so that - // label attribution and autosync allow-list checks use the real origin. - authenticatedPeerID := msg.GetFrom().String() - - logger.Debug("Received label announcement", - "from", authenticatedPeerID, - "cid", announcement.CID, - "labels", len(announcement.Labels)) - - // Invoke callback with authenticated peer ID - if m.onRecordPublishEvent != nil { - // Pass authenticated peer ID as separate parameter for security - m.onRecordPublishEvent(m.ctx, authenticatedPeerID, announcement) - } - } -} - -// GetTopicPeers returns the list of peers subscribed to the labels topic. -// This is useful for monitoring network connectivity and debugging. -// -// Returns: -// - []string: List of peer IDs (as strings) -func (m *Manager) GetTopicPeers() []string { - peers := m.topic.ListPeers() - peerIDs := make([]string, len(peers)) - - for i, p := range peers { - peerIDs[i] = p.String() - } - - return peerIDs -} - -// GetMeshPeerCount returns the number of peers in the GossipSub mesh. -// This is used for readiness checks to ensure the mesh is formed. -func (m *Manager) GetMeshPeerCount() int { - return len(m.topic.ListPeers()) -} - -// Close stops the GossipSub manager and releases resources. -// This should be called during shutdown to clean up gracefully. -// -// Flow: -// 1. Cancel subscription (stops handleMessages goroutine) -// 2. Leave topic -// 3. Release resources -// -// Returns: -// - error: If cleanup fails (rare) -func (m *Manager) Close() error { - m.sub.Cancel() - - if err := m.topic.Close(); err != nil { - return fmt.Errorf("failed to close gossipsub topic: %w", err) - } - - return nil -} - -// TagMeshPeers tags all current GossipSub mesh peers with high priority -// to prevent them from being pruned by the Connection Manager. -// -// Mesh peers are critical for fast label propagation (5-20ms delivery). -// If mesh peers are pruned, the mesh must rebuild, causing temporary -// degradation in GossipSub performance. -// -// This method should be called: -// - After GossipSub initialization -// - Periodically (e.g., every 30 seconds) as mesh changes -// - Or in response to mesh events (advanced) -// -// Priority: 50 points (high, but below bootstrap's 100) -// -// Safety: -// - Safe to call even if Connection Manager is nil (no-op) -// - Safe to call when mesh is empty (no-op) -// - Safe to call multiple times (re-tagging is harmless) -func (m *Manager) TagMeshPeers() { - if m == nil || m.host.ConnManager() == nil { - return // No-op if manager or connection manager not available - } - - peers := m.topic.ListPeers() - - if len(peers) == 0 { - logger.Debug("No mesh peers to tag") - - return - } - - for _, p := range peers { - m.host.ConnManager().TagPeer(p, "gossipsub-mesh", p2p.PeerPriorityGossipSubMesh) - } - - logger.Debug("Tagged GossipSub mesh peers", - "count", len(peers), - "priority", p2p.PeerPriorityGossipSubMesh, - "topic", m.topicName) -} diff --git a/server/routing/query_matching.go b/server/routing/query_matching.go index 17757f449..661d0e626 100644 --- a/server/routing/query_matching.go +++ b/server/routing/query_matching.go @@ -4,7 +4,6 @@ package routing import ( - "context" "strings" routingv1 "github.com/agntcy/dir/api/routing/v1" @@ -14,44 +13,6 @@ import ( var queryLogger = logging.Logger("routing/query") -// LabelRetriever function type for injecting different label retrieval strategies. -// This allows us to use the same query matching logic for both local and remote scenarios -// while keeping the label retrieval implementation separate. -type LabelRetriever func(ctx context.Context, cid string) []types.Label - -// MatchesAllQueries checks if a record matches ALL provided queries using injected label retrieval. -// This implements AND logic - all queries must match for the record to be considered a match. -// -// Parameters: -// - ctx: Context for the operation -// - cid: The CID of the record to check -// - queries: List of queries that must ALL match (AND relationship) -// - labelRetriever: Function to retrieve labels for the given CID -// -// Returns true if all queries match, false otherwise. -func MatchesAllQueries( - ctx context.Context, - cid string, - queries []*routingv1.RecordQuery, - labelRetriever LabelRetriever, -) bool { - if len(queries) == 0 { - return true // No filters = match everything - } - - // Use the injected label retrieval strategy - labels := labelRetriever(ctx, cid) - - // ALL queries must match (AND relationship) - for _, query := range queries { - if !QueryMatchesLabels(query, labels) { - return false - } - } - - return true -} - // QueryMatchesLabels checks if a single query matches against a list of labels. // This function contains the unified logic for all query types, resolving the // differences between local and remote implementations. @@ -160,26 +121,3 @@ func QueryMatchesLabels(query *routingv1.RecordQuery, labelList []types.Label) b return false } } - -// GetMatchingQueries returns the queries that match against a specific label key. -// This is used primarily for calculating match scores in Search operations. -func GetMatchingQueries(labelKey string, queries []*routingv1.RecordQuery) []*routingv1.RecordQuery { - var matchingQueries []*routingv1.RecordQuery - - // Extract label from the enhanced key - label, _, _, err := ParseEnhancedLabelKey(labelKey) - if err != nil { - queryLogger.Warn("Failed to parse enhanced label key for query matching", "key", labelKey, "error", err) - - return matchingQueries - } - - // Check which queries this label satisfies - for _, query := range queries { - if QueryMatchesLabels(query, []types.Label{label}) { - matchingQueries = append(matchingQueries, query) - } - } - - return matchingQueries -} diff --git a/server/routing/query_matching_test.go b/server/routing/query_matching_test.go index 7edaf436d..5f0fd3248 100644 --- a/server/routing/query_matching_test.go +++ b/server/routing/query_matching_test.go @@ -4,7 +4,6 @@ package routing import ( - "context" "testing" routingv1 "github.com/agntcy/dir/api/routing/v1" @@ -202,239 +201,6 @@ func TestQueryMatchesLabels(t *testing.T) { } } -func TestMatchesAllQueries(t *testing.T) { - ctx := t.Context() - testCID := "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi" - - // Mock label retriever that returns predefined labels - mockLabelRetriever := func(_ context.Context, cid string) []types.Label { - if cid == testCID { - return []types.Label{ - types.Label("/skills/AI"), - types.Label("/skills/AI/ML"), - types.Label("/domains/technology"), - types.Label("/modules/runtime/model"), - types.Label("/locators/docker-image"), - } - } - - return []types.Label{} - } - - testCases := []struct { - name string - cid string - queries []*routingv1.RecordQuery - expected bool - }{ - { - name: "no_queries_matches_all", - cid: testCID, - queries: []*routingv1.RecordQuery{}, - expected: true, - }, - { - name: "single_matching_query", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - }, - expected: true, - }, - { - name: "single_non_matching_query", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "blockchain", - }, - }, - expected: false, - }, - { - name: "multiple_matching_queries_and_logic", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR, - Value: "docker-image", - }, - }, - expected: true, - }, - { - name: "mixed_matching_and_non_matching_queries", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", // matches - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "blockchain", // doesn't match - }, - }, - expected: false, // AND logic - all must match - }, - { - name: "domain_query_matches", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, - Value: "technology", - }, - }, - expected: true, - }, - { - name: "module_query_matches", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE, - Value: "runtime/model", - }, - }, - expected: true, - }, - { - name: "all_query_types_match", - cid: testCID, - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, - Value: "technology", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE, - Value: "runtime/model", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR, - Value: "docker-image", - }, - }, - expected: true, // All should match - }, - { - name: "unknown_cid", - cid: "unknown-cid", - queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - }, - expected: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := MatchesAllQueries(ctx, tc.cid, tc.queries, mockLabelRetriever) - assert.Equal(t, tc.expected, result) - }) - } -} - -func TestGetMatchingQueries(t *testing.T) { - testQueries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "web-development", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, - Value: "healthcare", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE, - Value: "runtime/model", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR, - Value: "docker-image", - }, - } - - testCases := []struct { - name string - labelKey string - expectedMatches int - expectedQueryType routingv1.RecordQueryType - }{ - { - name: "skill_ai_matches", - labelKey: "/skills/AI/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/peer1", - expectedMatches: 1, - expectedQueryType: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - }, - { - name: "skill_web_dev_matches", - labelKey: "/skills/web-development/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/peer1", - expectedMatches: 1, - expectedQueryType: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - }, - { - name: "locator_matches", - labelKey: "/locators/docker-image/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/peer1", - expectedMatches: 1, - expectedQueryType: routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR, - }, - { - name: "domain_matches", - labelKey: "/domains/healthcare/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/peer1", - expectedMatches: 1, - expectedQueryType: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, - }, - { - name: "module_matches", - labelKey: "/modules/runtime/model/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/peer1", - expectedMatches: 1, - expectedQueryType: routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE, - }, - { - name: "no_matches", - labelKey: "/skills/blockchain/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi/peer1", - expectedMatches: 0, - }, - { - name: "malformed_key", - labelKey: "/invalid-key", - expectedMatches: 0, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - matches := GetMatchingQueries(tc.labelKey, testQueries) - assert.Len(t, matches, tc.expectedMatches) - - if tc.expectedMatches > 0 { - assert.Equal(t, tc.expectedQueryType, matches[0].GetType()) - } - }) - } -} - func TestQueryMatchingEdgeCases(t *testing.T) { t.Run("nil_query", func(t *testing.T) { // This should not panic @@ -469,68 +235,3 @@ func TestQueryMatchingEdgeCases(t *testing.T) { assert.False(t, result) }) } - -// Test the integration between MatchesAllQueries and QueryMatchesLabels. -func TestQueryMatchingIntegration(t *testing.T) { - ctx := t.Context() - - // Test with a more complex label retriever - complexLabelRetriever := func(_ context.Context, cid string) []types.Label { - switch cid { - case "ai-record": - return []types.Label{ - types.Label("/skills/AI"), - types.Label("/skills/AI/ML"), - types.Label("/skills/AI/NLP"), - } - case "web-record": - return []types.Label{ - types.Label("/skills/web-development"), - types.Label("/skills/javascript"), - types.Label("/locators/git-repo"), - } - case "mixed-record": - return []types.Label{ - types.Label("/skills/AI"), - types.Label("/skills/web-development"), - types.Label("/domains/healthcare"), - types.Label("/modules/runtime/model"), - types.Label("/locators/docker-image"), - } - default: - return []types.Label{} - } - } - - t.Run("complex_and_logic_test", func(t *testing.T) { - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "web-development", - }, - } - - // Only mixed-record should match both queries - assert.True(t, MatchesAllQueries(ctx, "mixed-record", queries, complexLabelRetriever)) - assert.False(t, MatchesAllQueries(ctx, "ai-record", queries, complexLabelRetriever)) - assert.False(t, MatchesAllQueries(ctx, "web-record", queries, complexLabelRetriever)) - }) - - t.Run("hierarchical_skill_matching", func(t *testing.T) { - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI/ML", - }, - } - - // Should match records with AI/ML or more specific skills - assert.True(t, MatchesAllQueries(ctx, "ai-record", queries, complexLabelRetriever)) - assert.False(t, MatchesAllQueries(ctx, "web-record", queries, complexLabelRetriever)) - assert.False(t, MatchesAllQueries(ctx, "mixed-record", queries, complexLabelRetriever)) // Only has /skills/AI, not AI/ML - }) -} diff --git a/server/routing/routing.go b/server/routing/routing.go index 1e8ad8a6e..25d36cf4d 100644 --- a/server/routing/routing.go +++ b/server/routing/routing.go @@ -1,15 +1,14 @@ // Copyright AGNTCY Contributors (https://github.com/agntcy) // SPDX-License-Identifier: Apache-2.0 -// Package routing provides distributed content routing capabilities for the dir system. -// It implements both local and remote routing strategies with automatic cleanup of stale data. +// Package routing makes the records a node has published discoverable across +// the network, and answers discovery queries from other nodes. // -// The routing system consists of: -// - Local routing: Fast queries against local datastore -// - Remote routing: DHT-based discovery across the network -// - Cleanup service: Automatic removal of stale labels and orphaned records -// -// Label metadata is stored in JSON format with timestamps for lifecycle management. +// Holding a record and publishing it are distinct: a held record is served to +// anyone who knows its CID, while publishing advertises its CID and labels as +// DHT provider keys so it can be found without one. Listing what this node has +// published is a local SQL query; searching the network is a DHT provider +// lookup followed by a query against the peers it names. package routing import ( @@ -17,18 +16,20 @@ import ( "fmt" coretypes "github.com/agntcy/dir/api/core/types" - corev1 "github.com/agntcy/dir/api/core/v1" routingv1 "github.com/agntcy/dir/api/routing/v1" "github.com/agntcy/dir/server/datastore" "github.com/agntcy/dir/server/events" - "github.com/agntcy/dir/server/ingest" "github.com/agntcy/dir/server/types" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) type route struct { local *routeLocal remote *routeRemote + db types.DatabaseAPI eventBus *events.SafeEventBus peerID string } @@ -43,13 +44,15 @@ func (r *route) hasPeersInRoutingTable() bool { return r.remote.server.DHT().RoutingTable().Size() > 0 } -func New(ctx context.Context, store types.StoreAPI, ingestor ingest.Ingestor, validator corev1.Validator, opts types.APIOptions) (types.RoutingAPI, error) { +func New(ctx context.Context, store types.StoreAPI, db types.DatabaseAPI, opts types.APIOptions) (types.RoutingAPI, error) { // Create main router mainRounter := &route{ + db: db, eventBus: opts.EventBus(), } - // Create routing datastore + // Datastore for the DHT's own state: its routing table and the provider + // records it holds for other peers. Nothing else writes to it. var dsOpts []datastore.Option if dstoreDir := opts.Config().Routing.DatastoreDir; dstoreDir != "" { dsOpts = append(dsOpts, datastore.WithFsProvider(dstoreDir)) @@ -61,7 +64,7 @@ func New(ctx context.Context, store types.StoreAPI, ingestor ingest.Ingestor, va } // Create remote router first to get the peer ID - mainRounter.remote, err = newRemote(ctx, store, ingestor, validator, dstore, opts) + mainRounter.remote, err = newRemote(ctx, store, db, dstore, opts) if err != nil { return nil, fmt.Errorf("failed to create remote routing: %w", err) } @@ -69,29 +72,35 @@ func New(ctx context.Context, store types.StoreAPI, ingestor ingest.Ingestor, va // Get local peer ID from the remote server host mainRounter.peerID = mainRounter.remote.server.Host().ID().String() - // Create local router with peer ID - mainRounter.local = newLocal(store, dstore, mainRounter.peerID) + mainRounter.local = newLocal(db) return mainRounter, nil } +// Publish marks a record as one this node announces, and announces it. +// +// Pushing a record only makes it servable to whoever already knows its CID. +// Publishing is what makes it discoverable, and the flag is what makes that +// durable: the reprovide cycle enumerates published records, so a node with no +// peers yet loses nothing by skipping the announcement here β€” it will announce +// as soon as a peer arrives. func (r *route) Publish(ctx context.Context, record coretypes.Record) error { - // Always publish data locally for archival/querying - err := r.local.Publish(ctx, record) - if err != nil { - st := status.Convert(err) + if record == nil { + return status.Error(codes.InvalidArgument, "record is required") //nolint:wrapcheck + } - return status.Errorf(st.Code(), "failed to publish locally: %s", st.Message()) + if err := r.setPublished(record.GetCid(), true); err != nil { + return err } - // Only publish to network if peers are available if r.hasPeersInRoutingTable() { - err = r.remote.Publish(ctx, record) - if err != nil { + if err := r.remote.Publish(ctx, record); err != nil { st := status.Convert(err) return status.Errorf(st.Code(), "failed to publish to the network: %s", st.Message()) } + } else { + localLogger.Info("No DHT peers yet; record will be advertised once one is available", "cid", record.GetCid()) } // Emit RECORD_PUBLISHED event after successful publication @@ -113,32 +122,61 @@ func (r *route) List(ctx context.Context, req *routingv1.ListRequest) (<-chan *r return r.local.List(ctx, req) } +// Search returns records held by other peers. It asks the DHT which peers +// provide a queried label and then asks those peers what they hold, so results +// are best-effort: a peer unreachable within the search budget is missed. +// +// Records held by this node are not included; List covers those. func (r *route) Search(ctx context.Context, req *routingv1.SearchRequest) (<-chan *routingv1.SearchResponse, error) { - // Search is always remote-only - it returns records from other peers using cached announcements - // This operation queries locally cached remote announcements from DHT return r.remote.Search(ctx, req) } -func (r *route) Unpublish(ctx context.Context, record coretypes.Record) error { - err := r.local.Unpublish(ctx, record) - if err != nil { - st := status.Convert(err) +// Unpublish stops advertising a record. The node still holds it and still +// serves it to anyone who knows the CID; it just stops being discoverable. +// +// Nothing is withdrawn from the network, because Kademlia has no retraction: +// the provider records already at their custodians stay until they expire, up +// to RecordTTL. What this does is drop the record from every future reprovide +// cycle, which is the only durable way to stop announcing it. +// +// Labels need no special handling. The cycle recomputes the distinct label set +// from the advertised records each time, so a label stops being announced +// exactly when the last record carrying it does, with no refcount to maintain. +func (r *route) Unpublish(_ context.Context, record coretypes.Record) error { + if record == nil { + return status.Error(codes.InvalidArgument, "record is required") //nolint:wrapcheck + } - return status.Errorf(st.Code(), "failed to unpublish locally: %s", st.Message()) + if err := r.setPublished(record.GetCid(), false); err != nil { + return err } - // Emit RECORD_UNPUBLISHED event after successful unpublication r.eventBus.RecordUnpublished(record.GetCid()) - // no need to explicitly handle unpublishing from the network - // TODO clarify if network sync trigger is needed here + return nil +} + +// setPublished records whether the reprovide cycle should announce this CID. +func (r *route) setPublished(recordCID string, published bool) error { + if recordCID == "" { + return status.Error(codes.InvalidArgument, "record has no CID") //nolint:wrapcheck + } + + if r.db == nil { + return status.Error(codes.Unavailable, "routing has no database to record the published flag in") //nolint:wrapcheck + } + + if err := r.db.SetRecordPublished(recordCID, published); err != nil { + return status.Errorf(codes.Internal, "failed to set published flag for %s: %v", recordCID, err) + } + return nil } // Stop stops the routing services and releases resources. // This should be called during server shutdown to clean up gracefully. func (r *route) Stop() error { - // Stop remote routing (includes GossipSub and p2p server) + // Stop remote routing (includes the p2p server) if r.remote != nil { if err := r.remote.Stop(); err != nil { return fmt.Errorf("failed to stop remote routing: %w", err) @@ -183,25 +221,35 @@ func (r *route) GetPeerID() string { } // GetProviderCount returns the number of distinct peers (including the local -// node) currently announcing the given CID, by counting unique peerIDs in the -// routing datastore. -func (r *route) GetProviderCount(ctx context.Context, cid string) (int, error) { - entries, err := QueryAllNamespaces(ctx, r.remote.dstore) +// node) currently providing the given CID, via a DHT provider lookup. +// +// The local node is counted because Provide registers self in the local +// provider store, which FindProvidersAsync drains before going to the network. +// That also makes the count meaningful with an empty routing table. +// +// Best-effort: the result is whatever the lookup gathers within +// ProviderCountTimeout. Providers are deduplicated by peer ID because a peer +// can be emitted twice if the first sighting carried no addresses. +func (r *route) GetProviderCount(ctx context.Context, recordCID string) (int, error) { + if r.remote == nil || r.remote.server == nil { + return 0, fmt.Errorf("remote routing is not available") + } + + decoded, err := cid.Decode(recordCID) if err != nil { - return 0, fmt.Errorf("failed to query routing datastore for %s: %w", cid, err) + return 0, fmt.Errorf("invalid CID %s: %w", recordCID, err) } - seen := make(map[string]struct{}) + lookupCtx, cancel := context.WithTimeout(ctx, ProviderCountTimeout) + defer cancel() - for _, entry := range entries { - _, keyCID, keyPeerID, parseErr := ParseEnhancedLabelKey(entry.Key) - if parseErr != nil { - continue - } + seen := make(map[peer.ID]struct{}) - if keyCID == cid { - seen[keyPeerID] = struct{}{} - } + // count=0 selects findAll. Any non-zero count lets the local provider store + // short-circuit the network lookup once it holds that many entries, which + // would make the count depend on what this node happens to have cached. + for provider := range r.remote.server.DHT().FindProvidersAsync(lookupCtx, decoded, 0) { + seen[provider.ID] = struct{}{} } return len(seen), nil diff --git a/server/routing/routing_local.go b/server/routing/routing_local.go index 1cab7cffe..6010293b3 100644 --- a/server/routing/routing_local.go +++ b/server/routing/routing_local.go @@ -5,305 +5,240 @@ package routing import ( "context" - "encoding/json" - "strings" - "time" + "errors" + "fmt" - coretypes "github.com/agntcy/dir/api/core/types" corev1 "github.com/agntcy/dir/api/core/v1" routingv1 "github.com/agntcy/dir/api/routing/v1" "github.com/agntcy/dir/server/types" "github.com/agntcy/dir/utils/logging" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) var localLogger = logging.Logger("routing/local") -// operations performed locally. -type routeLocal struct { - store types.StoreAPI - dstore types.Datastore - localPeerID string // Cached local peer ID for efficient filtering -} +// errNoDatabase reports that routing was constructed without the SQL index it +// needs to answer what this node holds. +var errNoDatabase = errors.New("routing has no database") -func newLocal(store types.StoreAPI, dstore types.Datastore, localPeerID string) *routeLocal { - return &routeLocal{ - store: store, - dstore: dstore, - localPeerID: localPeerID, - } +// routeLocal answers questions about the records this node holds. +// +// Held records and their labels come from the SQL index, which the ingest path +// maintains for content arriving from any source and which SearchService +// queries too. +type routeLocal struct { + db types.DatabaseAPI } -func (r *routeLocal) Publish(ctx context.Context, record coretypes.Record) error { - if record == nil { - return status.Error(codes.InvalidArgument, "record is required") //nolint:wrapcheck // Mock should return exact error without wrapping - } - - cid := record.GetCid() - if cid == "" { - return status.Error(codes.InvalidArgument, "record has no CID") //nolint:wrapcheck - } - - localLogger.Debug("Called local routing's Publish method", "cid", cid) - - metrics, err := loadMetrics(ctx, r.dstore) - if err != nil { - return status.Errorf(codes.Internal, "failed to load metrics: %v", err) - } - - batch, err := r.dstore.Batch(ctx) - if err != nil { - return status.Errorf(codes.Internal, "failed to create batch: %v", err) - } - - // the key where we will save the record - recordKey := datastore.NewKey("/records/" + cid) - - // check if we have the record already - // this is useful to avoid updating metrics and running the same operation multiple times - recordExists, err := r.dstore.Has(ctx, recordKey) - if err != nil { - return status.Errorf(codes.Internal, "failed to check if record exists: %v", err) - } - - if recordExists { - localLogger.Info("Skipping republish as record was already published", "cid", cid) - - return nil - } - - // store record for later lookup - if err := batch.Put(ctx, recordKey, nil); err != nil { - return status.Errorf(codes.Internal, "failed to put record key: %v", err) - } - - // Update metrics for all record labels and store them locally for queries - // Note: This handles ALL local storage for both local-only and network scenarios - // Network announcements are handled separately by routing_remote when peers are available - labelList := types.GetLabelsFromRecord(record) - for _, label := range labelList { - // Create minimal metadata (PeerID and CID now in key) - metadata := &types.LabelMetadata{ - Timestamp: time.Now(), - LastSeen: time.Now(), - } - - // Serialize metadata to JSON - metadataBytes, err := json.Marshal(metadata) - if err != nil { - return status.Errorf(codes.Internal, "failed to serialize label metadata: %v", err) - } - - // Store with enhanced self-descriptive key: /skills/AI/CID123/Peer1 - enhancedKey := BuildEnhancedLabelKey(label, cid, r.localPeerID) - - labelKey := datastore.NewKey(enhancedKey) - if err := batch.Put(ctx, labelKey, metadataBytes); err != nil { - return status.Errorf(codes.Internal, "failed to put label key: %v", err) - } - - metrics.increment(label) - } - - err = batch.Commit(ctx) - if err != nil { - return status.Errorf(codes.Internal, "failed to commit batch: %v", err) - } - - // sync metrics - err = metrics.update(ctx, r.dstore) - if err != nil { - return status.Errorf(codes.Internal, "failed to update metrics: %v", err) - } - - localLogger.Info("Successfully published record", "cid", cid) - - return nil +func newLocal(db types.DatabaseAPI) *routeLocal { + return &routeLocal{db: db} } -//nolint:cyclop +// List returns the records this node has published, filtered by the request's +// queries. Records that are merely held are absent: nothing announces them. +// +// Queries AND together: a record is returned only if it satisfies every one. func (r *routeLocal) List(ctx context.Context, req *routingv1.ListRequest) (<-chan *routingv1.ListResponse, error) { localLogger.Debug("Called local routing's List method", "req", req) - // βœ… DEFENSIVE: Deduplicate queries for consistent behavior (same as remote Search) + if r.db == nil { + return nil, status.Error(codes.Unavailable, "local routing has no database to list from") //nolint:wrapcheck + } + + // Duplicate queries would otherwise cost a redundant round trip each. originalQueries := req.GetQueries() - deduplicatedQueries := deduplicateQueries(originalQueries) + queries := deduplicateQueries(originalQueries) - if len(originalQueries) != len(deduplicatedQueries) { + if len(originalQueries) != len(queries) { localLogger.Info("Deduplicated list queries for consistent filtering", - "originalCount", len(originalQueries), "deduplicatedCount", len(deduplicatedQueries)) + "originalCount", len(originalQueries), "deduplicatedCount", len(queries)) } - // Output channel for results outCh := make(chan *routingv1.ListResponse) - // Process in background with deduplicated queries go func() { defer close(outCh) - r.listLocalRecords(ctx, deduplicatedQueries, req.GetLimit(), outCh) + r.listLocalRecords(ctx, queries, req.GetLimit(), outCh) }() return outCh, nil } -// listLocalRecords lists all local records with optional query filtering. -// Uses the simple and efficient approach: start with /records/ index, then filter by queries. +// listLocalRecords resolves the query to a CID set, loads each record's labels +// and streams the results. func (r *routeLocal) listLocalRecords(ctx context.Context, queries []*routingv1.RecordQuery, limit uint32, outCh chan<- *routingv1.ListResponse) { - processedCount := 0 - limitInt := int(limit) + cids, err := r.matchingCIDs(queries, int(limit)) + if err != nil { + localLogger.Error("Failed to list local records", "error", err) - // Step 1: Get all local record CIDs from /records/ index - recordResults, err := r.dstore.Query(ctx, query.Query{ - Prefix: "/records/", - }) + return + } + + if len(cids) == 0 { + localLogger.Debug("Completed List operation", "processed", 0, "queries", len(queries)) + + return + } + + labels, err := r.db.GetRecordLabels(cids) if err != nil { - localLogger.Error("Failed to query local records", "error", err) + localLogger.Error("Failed to load labels for local records", "error", err) return } - defer recordResults.Close() - // Step 2: For each local record, check if it matches ALL queries - for result := range recordResults.Next() { - if result.Error != nil { - localLogger.Warn("Error reading record entry", "key", result.Key, "error", result.Error) + sent := 0 - continue - } + for _, cid := range cids { + recordLabels := labels[cid] - // Extract CID from record key: /records/CID123 β†’ CID123 - cid := strings.TrimPrefix(result.Key, "/records/") - if cid == "" { - continue + asStrings := make([]string, len(recordLabels)) + for i, label := range recordLabels { + asStrings[i] = label.String() } - // Check if this record matches all queries (AND relationship) - if r.matchesAllQueries(ctx, cid, queries) { - // Get labels for this record - internalLabels := r.getRecordLabelsEfficiently(ctx, cid) - - // Convert []Label to []string for gRPC API boundary - apiLabels := make([]string, len(internalLabels)) - for i, label := range internalLabels { - apiLabels[i] = label.String() - } - - // Send the response - outCh <- &routingv1.ListResponse{ - RecordRef: &corev1.RecordRef{Cid: cid}, - Labels: apiLabels, - } - - processedCount++ - if limitInt > 0 && processedCount >= limitInt { - break - } + select { + case outCh <- &routingv1.ListResponse{ + RecordRef: &corev1.RecordRef{Cid: cid}, + Labels: asStrings, + }: + sent++ + case <-ctx.Done(): + localLogger.Debug("List cancelled", "sent", sent) + + return } } - localLogger.Debug("Completed List operation", "processed", processedCount, "queries", len(queries)) + localLogger.Debug("Completed List operation", "processed", sent, "queries", len(queries)) } -// matchesAllQueries checks if a record matches ALL provided queries (AND relationship). -// Uses shared query matching logic with local label retrieval strategy. -func (r *routeLocal) matchesAllQueries(ctx context.Context, cid string, queries []*routingv1.RecordQuery) bool { - // Inject local label retrieval strategy into shared query matching logic - return MatchesAllQueries(ctx, cid, queries, r.getRecordLabelsEfficiently) -} - -// getRecordLabelsEfficiently gets labels for a record by extracting them from datastore keys. -// This completely avoids expensive Pull operations by using the fact that labels are stored as keys. -// This function is designed to be resilient - it never returns an error, only logs warnings. -func (r *routeLocal) getRecordLabelsEfficiently(ctx context.Context, cid string) []types.Label { - var labelList []types.Label +// matchingCIDs returns the CIDs of held records satisfying every query. +// +// Each query runs on its own and the results are intersected, because the +// filter API can only AND across label kinds: two skill queries in one filter +// set would OR together and wrongly widen the result. +func (r *routeLocal) matchingCIDs(queries []*routingv1.RecordQuery, limit int) ([]string, error) { + if len(queries) <= 1 { + filters, err := listFilters(queries, limit) + if err != nil { + return nil, err + } - // Use shared namespace iteration function - entries, err := QueryAllNamespaces(ctx, r.dstore) - if err != nil { - localLogger.Error("Failed to get namespace entries for labels", "cid", cid, "error", err) + cids, err := r.db.GetRecordCIDs(filters...) + if err != nil { + return nil, fmt.Errorf("failed to query records: %w", err) + } - return labelList + return cids, nil } - // Find keys for this CID and local peer: "/skills/AI/ML/CID123/Peer1" - for _, entry := range entries { - // Parse the enhanced key to get components - label, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) + var matched []string + + for i, query := range queries { + // The limit cannot be pushed down here: a record ranked past it in one + // query could still belong in the intersection. + filters, err := listFilters([]*routingv1.RecordQuery{query}, maxListCandidates) + if err != nil { + return nil, err + } + + cids, err := r.db.GetRecordCIDs(filters...) if err != nil { - localLogger.Warn("Failed to parse enhanced label key", "key", entry.Key, "error", err) + return nil, fmt.Errorf("failed to query records: %w", err) + } + + if i == 0 { + matched = cids continue } - // Check if this key matches our CID and is from local peer - if keyCID == cid && keyPeerID == r.localPeerID { - labelList = append(labelList, label) + matched = intersect(matched, cids) + if len(matched) == 0 { + return nil, nil } } - return labelList -} - -func (r *routeLocal) Unpublish(ctx context.Context, record coretypes.Record) error { - if record == nil { - return status.Error(codes.InvalidArgument, "record is required") //nolint:wrapcheck // Mock should return exact error without wrapping + if limit > 0 && len(matched) > limit { + matched = matched[:limit] } - cid := record.GetCid() - if cid == "" { - return status.Error(codes.InvalidArgument, "record has no CID") //nolint:wrapcheck - } + return matched, nil +} - localLogger.Debug("Called local routing's Unpublish method", "cid", cid) +// baseListFilters apply to every List, whatever the request asks for. +// Unpublished records are excluded because List reports what this node is +// providing, not everything it holds. +var baseListFilters = []types.FilterOption{types.WithPublished(true)} - // load metrics for the client - metrics, err := loadMetrics(ctx, r.dstore) - if err != nil { - return status.Errorf(codes.Internal, "failed to load metrics: %v", err) - } +// listFilters translates the queries into database filters. An empty query set +// selects every published record, which is what an unfiltered List asks for. +func listFilters(queries []*routingv1.RecordQuery, limit int) ([]types.FilterOption, error) { + filters := make([]types.FilterOption, 0, len(queries)+len(baseListFilters)+1) + filters = append(filters, baseListFilters...) - batch, err := r.dstore.Batch(ctx) - if err != nil { - return status.Errorf(codes.Internal, "failed to create batch: %v", err) - } + for _, query := range queries { + filter, err := listFilter(query) + if err != nil { + return nil, err + } - // get record key and remove record - recordKey := datastore.NewKey("/records/" + cid) - if err := batch.Delete(ctx, recordKey); err != nil { - return status.Errorf(codes.Internal, "failed to delete record key: %v", err) + filters = append(filters, filter) } - // keep track of all record labels - labelList := types.GetLabelsFromRecord(record) - - for _, label := range labelList { - // Delete enhanced key with CID and PeerID - enhancedKey := BuildEnhancedLabelKey(label, cid, r.localPeerID) + if limit > 0 { + filters = append(filters, types.WithLimit(limit)) + } - labelKey := datastore.NewKey(enhancedKey) - if err := batch.Delete(ctx, labelKey); err != nil { - return status.Errorf(codes.Internal, "failed to delete label key: %v", err) - } + return filters, nil +} - metrics.decrement(label) +// listFilter translates one query into a database filter. +// +// Hierarchical namespaces match the value or any descendant, so a query for +// "AI" finds "AI/ML". Locators are flat and match exactly. +func listFilter(query *routingv1.RecordQuery) (types.FilterOption, error) { + value := query.GetValue() + if value == "" { + return nil, fmt.Errorf("query of type %s has no value", query.GetType()) + } + + descendants := value + "/*" + + switch query.GetType() { + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL: + return types.WithSkillNames(value, descendants), nil + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN: + return types.WithDomainNames(value, descendants), nil + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE: + return types.WithModuleNames(value, descendants), nil + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR: + return types.WithLocatorTypes(value), nil + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_UNSPECIFIED: + return nil, fmt.Errorf("query type is unspecified") + default: + return nil, fmt.Errorf("unknown query type %s", query.GetType()) } +} - err = batch.Commit(ctx) - if err != nil { - return status.Errorf(codes.Internal, "failed to commit batch: %v", err) +// intersect returns the members of left that also appear in right, preserving +// left's ordering. +func intersect(left, right []string) []string { + set := make(map[string]struct{}, len(right)) + for _, cid := range right { + set[cid] = struct{}{} } - // sync metrics - err = metrics.update(ctx, r.dstore) - if err != nil { - return status.Errorf(codes.Internal, "failed to update metrics: %v", err) - } + kept := left[:0] - localLogger.Info("Successfully unpublished record", "cid", cid) + for _, cid := range left { + if _, ok := set[cid]; ok { + kept = append(kept, cid) + } + } - return nil + return kept } diff --git a/server/routing/routing_local_test.go b/server/routing/routing_local_test.go index 16e320999..16bc27402 100644 --- a/server/routing/routing_local_test.go +++ b/server/routing/routing_local_test.go @@ -5,99 +5,37 @@ package routing import ( - "context" - "errors" - "log/slog" - "os" "strings" "testing" "time" typesv1alpha1 "buf.build/gen/go/agntcy/oasf/protocolbuffers/go/agntcy/oasf/types/v1alpha1" - coretypes "github.com/agntcy/dir/api/core/types" corev1 "github.com/agntcy/dir/api/core/v1" routingv1 "github.com/agntcy/dir/api/routing/v1" - "github.com/agntcy/dir/server/datastore" "github.com/agntcy/dir/server/types" - "github.com/agntcy/dir/utils/logging" - ipfsdatastore "github.com/ipfs/go-datastore" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -const testPeerID = "test-peer-id" +// publishRecord indexes a record and announces it, which is what a completed +// push followed by a publish leaves behind. +func publishRecord(t *testing.T, r *route, db types.DatabaseAPI, record *corev1.Record) { + t.Helper() -func TestPublish_InvalidObject(t *testing.T) { - r := &routeLocal{localPeerID: testPeerID} + adapter, err := record.Decode() + require.NoError(t, err) - t.Run("nil record", func(t *testing.T) { - err := r.Publish(t.Context(), nil) - - assert.Error(t, err) - assert.ErrorContains(t, err, "record is required") - }) - - t.Run("record with no CID", func(t *testing.T) { - err := r.Publish(t.Context(), &mockRecord{}) - assert.Error(t, err) - assert.ErrorContains(t, err, "record has no CID") - }) + require.NoError(t, db.AddRecord(adapter)) + require.NoError(t, r.Publish(t.Context(), adapter)) } -type mockRecord struct { - coretypes.Record -} +func TestPublish_NilRecord(t *testing.T) { + r := &route{} -func (mockRecord) GetCid() string { - return "" -} + err := r.Publish(t.Context(), nil) -type mockStore struct { - data map[string]*corev1.Record -} - -func newMockStore() *mockStore { - return &mockStore{ - data: make(map[string]*corev1.Record), - } -} - -func (m *mockStore) Push(_ context.Context, record *corev1.Record) (*corev1.RecordRef, error) { - cid := record.GetCid() - if cid == "" { - return nil, errors.New("record CID is required") - } - - m.data[cid] = record - - return &corev1.RecordRef{Cid: cid}, nil -} - -func (m *mockStore) Lookup(_ context.Context, ref *corev1.RecordRef) (*corev1.RecordMeta, error) { - if _, exists := m.data[ref.GetCid()]; exists { - return &corev1.RecordMeta{ - Cid: ref.GetCid(), - }, nil - } - - return nil, errors.New("test object not found") -} - -func (m *mockStore) Pull(_ context.Context, ref *corev1.RecordRef) (*corev1.Record, error) { - if record, exists := m.data[ref.GetCid()]; exists { - return record, nil - } - - return nil, errors.New("test object not found") -} - -func (m *mockStore) Delete(_ context.Context, ref *corev1.RecordRef) error { - delete(m.data, ref.GetCid()) - - return nil -} - -func (m *mockStore) IsReady(_ context.Context) bool { - return true + assert.Error(t, err) + assert.ErrorContains(t, err, "record is required") } func TestPublishList_ValidSingleSkillQuery(t *testing.T) { @@ -135,34 +73,16 @@ func TestPublishList_ValidSingleSkillQuery(t *testing.T) { ) // create demo network - mainNode := newTestServer(t, t.Context(), nil) - r := newTestServer(t, t.Context(), mainNode.remote.server.P2pAddrs()) + db := newTestDatabase(t) + mainNode := newTestServer(t, t.Context(), nil, nil) + r := newTestServer(t, t.Context(), mainNode.remote.server.P2pAddrs(), db) // wait for connection <-mainNode.remote.server.DHT().RefreshRoutingTable() time.Sleep(1 * time.Second) - // Mock store - mockstore := newMockStore() - r.local.store = mockstore - - _, err := r.local.store.Push(t.Context(), testRecord) - assert.NoError(t, err) - - _, err = r.local.store.Push(t.Context(), testRecord2) - assert.NoError(t, err) - - // Publish first record - adapter, err := testRecord.Decode() - assert.NoError(t, err) - err = r.Publish(t.Context(), adapter) - assert.NoError(t, err) - - // Publish second record - adapter2, err := testRecord2.Decode() - assert.NoError(t, err) - err = r.Publish(t.Context(), adapter2) - assert.NoError(t, err) + publishRecord(t, r, db, testRecord) + publishRecord(t, r, db, testRecord2) for k, v := range validQueriesWithExpectedObjectRef { t.Run("Valid query: "+k, func(t *testing.T) { @@ -214,6 +134,7 @@ func TestPublishList_ValidSingleSkillQuery(t *testing.T) { assert.NoError(t, err) err = r.Unpublish(t.Context(), adapterUnpub) assert.NoError(t, err) + assert.NoError(t, db.RemoveRecord(testRecord2.GetCid())) // Try to list second record using RecordQuery refsChan, err := r.List(t.Context(), &routingv1.ListRequest{ @@ -251,25 +172,15 @@ func TestPublishList_ValidMultiSkillQuery(t *testing.T) { ) // create demo network - mainNode := newTestServer(t, t.Context(), nil) - r := newTestServer(t, t.Context(), mainNode.remote.server.P2pAddrs()) + db := newTestDatabase(t) + mainNode := newTestServer(t, t.Context(), nil, nil) + r := newTestServer(t, t.Context(), mainNode.remote.server.P2pAddrs(), db) // wait for connection <-mainNode.remote.server.DHT().RefreshRoutingTable() time.Sleep(1 * time.Second) - // Mock store - mockstore := newMockStore() - r.local.store = mockstore - - _, err := r.local.store.Push(t.Context(), testRecord) - assert.NoError(t, err) - - // Publish first record - adapter, err := testRecord.Decode() - assert.NoError(t, err) - err = r.Publish(t.Context(), adapter) - assert.NoError(t, err) + publishRecord(t, r, db, testRecord) t.Run("Valid multi skill query", func(t *testing.T) { // list with multiple RecordQueries (AND logic) @@ -301,116 +212,84 @@ func TestPublishList_ValidMultiSkillQuery(t *testing.T) { }) } -func newBadgerDatastore(b *testing.B) types.Datastore { - b.Helper() - - dsOpts := []datastore.Option{ - datastore.WithFsProvider("/tmp/test-datastore"), // Use a temporary directory - } - - dstore, err := datastore.New(dsOpts...) - if err != nil { - b.Fatalf("failed to create badger datastore: %v", err) - } - - b.Cleanup(func() { - _ = dstore.Close() - _ = os.RemoveAll("/tmp/test-datastore") - }) - - return dstore -} - -func newInMemoryDatastore(b *testing.B) types.Datastore { - b.Helper() - - dstore, err := datastore.New() - if err != nil { - b.Fatalf("failed to create in-memory datastore: %v", err) - } - - return dstore -} - -func Benchmark_RouteLocal(b *testing.B) { - store := newMockStore() - badgerDatastore := newBadgerDatastore(b) - inMemoryDatastore := newInMemoryDatastore(b) - localLogger = slog.New(slog.DiscardHandler) - - badgerRouter := newLocal(store, badgerDatastore, testPeerID) - inMemoryRouter := newLocal(store, inMemoryDatastore, testPeerID) - - record := corev1.New(&typesv1alpha1.Record{ - Name: "bench-agent", +// TestLocalList covers the parts of List that are not about a single skill +// filter: unfiltered listing, the limit, the returned label set, and the AND +// across queries. +func TestLocalList(t *testing.T) { + bothSkills := corev1.New(&typesv1alpha1.Record{ + Name: "both-skills", SchemaVersion: "0.7.0", Skills: []*typesv1alpha1.Skill{ {Name: "category1/class1"}, + {Name: "category2/class2"}, }, }) + oneSkill := corev1.New(&typesv1alpha1.Record{ + Name: "one-skill", + SchemaVersion: "0.7.0", + Skills: []*typesv1alpha1.Skill{{Name: "category1/class1"}}, + }) + + db := newTestDatabase(t) + node := newTestServer(t, t.Context(), nil, db) + + publishRecord(t, node, db, bothSkills) + publishRecord(t, node, db, oneSkill) - _, err := store.Push(b.Context(), record) - assert.NoError(b, err) + list := func(t *testing.T, req *routingv1.ListRequest) []*routingv1.ListResponse { + t.Helper() - b.Run("Badger DB Publish and Unpublish", func(b *testing.B) { - adapter, err := record.Decode() - assert.NoError(b, err) + responses, err := node.List(t.Context(), req) + require.NoError(t, err) - for b.Loop() { - _ = badgerRouter.Publish(b.Context(), adapter) - err := badgerRouter.Unpublish(b.Context(), adapter) - assert.NoError(b, err) + var collected []*routingv1.ListResponse + for response := range responses { + collected = append(collected, response) } + + return collected + } + + t.Run("no queries returns everything held", func(t *testing.T) { + assert.Len(t, list(t, &routingv1.ListRequest{}), 2) }) - b.Run("Badger DB List", func(b *testing.B) { - adapter, err := record.Decode() - assert.NoError(b, err) - - _ = badgerRouter.Publish(b.Context(), adapter) - for b.Loop() { - _, err := badgerRouter.List(b.Context(), &routingv1.ListRequest{ - Queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "category1/class1", - }, - }, - }) - assert.NoError(b, err) - } + t.Run("limit caps the results", func(t *testing.T) { + assert.Len(t, list(t, &routingv1.ListRequest{Limit: new(uint32(1))}), 1) }) - b.Run("In memory DB Publish and Unpublish", func(b *testing.B) { - adapter, err := record.Decode() - assert.NoError(b, err) + t.Run("queries AND rather than OR", func(t *testing.T) { + // oneSkill satisfies the first query only, so a union would wrongly + // return it alongside bothSkills. + responses := list(t, &routingv1.ListRequest{ + Queries: []*routingv1.RecordQuery{ + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "category1/class1"}, + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "category2/class2"}, + }, + }) - for b.Loop() { - _ = inMemoryRouter.Publish(b.Context(), adapter) - err := inMemoryRouter.Unpublish(b.Context(), adapter) - assert.NoError(b, err) - } + require.Len(t, responses, 1) + assert.Equal(t, bothSkills.GetCid(), responses[0].GetRecordRef().GetCid()) }) - b.Run("In memory DB List", func(b *testing.B) { - adapter, err := record.Decode() - assert.NoError(b, err) - - _ = inMemoryRouter.Publish(b.Context(), adapter) - for b.Loop() { - _, err := inMemoryRouter.List(b.Context(), &routingv1.ListRequest{ - Queries: []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "category1/class1", - }, - }, - }) - assert.NoError(b, err) - } + t.Run("responses carry the full label set", func(t *testing.T) { + responses := list(t, &routingv1.ListRequest{ + Queries: []*routingv1.RecordQuery{ + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "category2"}, + }, + }) + + require.Len(t, responses, 1) + assert.ElementsMatch(t, + []string{"/skills/category1/class1", "/skills/category2/class2"}, + responses[0].GetLabels()) }) - _ = badgerDatastore.Delete(b.Context(), ipfsdatastore.NewKey("/")) // Delete all keys - _ = inMemoryDatastore.Delete(b.Context(), ipfsdatastore.NewKey("/")) // Delete all keys - localLogger = logging.Logger("routing/local") + t.Run("unmatched query returns nothing", func(t *testing.T) { + assert.Empty(t, list(t, &routingv1.ListRequest{ + Queries: []*routingv1.RecordQuery{ + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "category3"}, + }, + })) + }) } diff --git a/server/routing/routing_remote.go b/server/routing/routing_remote.go index db90615b1..bebc8a33e 100644 --- a/server/routing/routing_remote.go +++ b/server/routing/routing_remote.go @@ -5,30 +5,19 @@ package routing import ( "context" - "encoding/json" "fmt" "sync" "time" coretypes "github.com/agntcy/dir/api/core/types" - corev1 "github.com/agntcy/dir/api/core/v1" routingv1 "github.com/agntcy/dir/api/routing/v1" - "github.com/agntcy/dir/server/ingest" - "github.com/agntcy/dir/server/routing/autosync" "github.com/agntcy/dir/server/routing/internal/p2p" - "github.com/agntcy/dir/server/routing/pubsub" "github.com/agntcy/dir/server/routing/rpc" - validators "github.com/agntcy/dir/server/routing/validators" "github.com/agntcy/dir/server/types" "github.com/agntcy/dir/utils/logging" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p-kad-dht/records" - record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" "google.golang.org/grpc/codes" @@ -37,83 +26,29 @@ import ( var remoteLogger = logging.Logger("routing/remote") -// NamespaceEntry contains processed namespace query data. -// This is used by namespace iteration functions for routing operations. -type NamespaceEntry struct { - Namespace string - Key string - Value []byte -} - -// QueryAllNamespaces queries all supported label namespaces and returns processed entries. -// This centralizes namespace iteration and datastore querying, eliminating code duplication -// between local and remote routing operations. All resource management is handled internally. -func QueryAllNamespaces(ctx context.Context, dstore types.Datastore) ([]NamespaceEntry, error) { - var entries []NamespaceEntry - - // Query all label namespaces - namespaces := []string{ - types.LabelTypeSkill.Prefix(), - types.LabelTypeDomain.Prefix(), - types.LabelTypeModule.Prefix(), - types.LabelTypeLocator.Prefix(), - } - - for _, namespace := range namespaces { - // Check for context cancellation - select { - case <-ctx.Done(): - return nil, fmt.Errorf("namespace query canceled: %w", ctx.Err()) - default: - } - - results, err := dstore.Query(ctx, query.Query{Prefix: namespace}) - if err != nil { - remoteLogger.Warn("Failed to query namespace", "namespace", namespace, "error", err) - - continue - } +// routeRemote handles routing across the network. Records and their labels are +// advertised as DHT provider keys, and discovery is a DHT lookup followed by a +// query against the peers it names. +type routeRemote struct { + storeAPI types.StoreAPI - // Process results and handle cleanup - func() { - defer results.Close() - - for result := range results.Next() { - if result.Error != nil { - continue - } - - entries = append(entries, NamespaceEntry{ - Namespace: namespace, - Key: result.Key, - Value: result.Value, - }) - } - }() - } + server *p2p.Server + service *rpc.Service - return entries, nil -} + // dstore persists the DHT's own state; nothing in this package reads it. + dstore types.Datastore -// routeRemote handles routing across the network with hybrid label discovery. -// It uses both GossipSub (efficient, wide propagation) and DHT+Pull (fallback). -type routeRemote struct { - storeAPI types.StoreAPI + // db is the authority on what this node holds, and so on what it advertises. + db types.DatabaseAPI - // autosyncMgr pulls+ingests records from trusted peers on DHT announcements. - // It is nil when autosync is disabled (deny-by-default). - autosyncMgr *autosync.Manager + isBootstrapNode bool // True if this node is a bootstrap node (no bootstrap peers configured) - server *p2p.Server - service *rpc.Service - notifyCh chan *handlerSync - dstore types.Datastore - cleanupManager *CleanupManager - pubsubManager *pubsub.Manager // GossipSub manager for label announcements (nil if disabled) - isBootstrapNode bool // True if this node is a bootstrap node (no bootstrap peers configured) + // reprovideInterval is how often published records are re-advertised so + // their DHT provider records do not expire. + reprovideInterval time.Duration // Lifecycle management - //nolint:containedctx // Context needed for managing lifecycle of multiple long-running goroutines (handleNotify, cleanup tasks) + //nolint:containedctx // Context needed for managing lifecycle of long-running cleanup tasks ctx context.Context // Routing subsystem context cancel context.CancelFunc // Cancel function for graceful shutdown wg sync.WaitGroup // Tracks all background goroutines @@ -121,8 +56,7 @@ type routeRemote struct { func newRemote(parentCtx context.Context, storeAPI types.StoreAPI, - ingestor ingest.Ingestor, - validator corev1.Validator, + db types.DatabaseAPI, dstore types.Datastore, opts types.APIOptions, ) (*routeRemote, error) { @@ -132,43 +66,20 @@ func newRemote(parentCtx context.Context, // Determine if this is a bootstrap node (no bootstrap peers configured) isBootstrapNode := len(opts.Config().Routing.BootstrapPeers) == 0 - // Resolve autosync policy up front so a bad peer ID fails fast at startup - // (deny-by-default: the allow-set is empty unless autosync is enabled). - autosyncCfg := opts.Config().Routing.Autosync - - var autosyncPeers map[peer.ID]struct{} - - if autosyncCfg.Enabled { - allowSet, err := autosyncCfg.AllowSet() - if err != nil { - cancel() - - return nil, fmt.Errorf("invalid autosync configuration: %w", err) - } - - if ingestor == nil { - cancel() - - return nil, fmt.Errorf("autosync is enabled but no ingestion service was provided") - } - - if validator == nil { - cancel() - - return nil, fmt.Errorf("autosync is enabled but no record validator was provided") - } - - autosyncPeers = allowSet + // Create routing + reprovideInterval := opts.Config().Routing.RepublishInterval + if reprovideInterval <= 0 { + reprovideInterval = RepublishInterval } - // Create routing routeAPI := &routeRemote{ - storeAPI: storeAPI, - notifyCh: make(chan *handlerSync, NotificationChannelSize), - dstore: dstore, - ctx: routingCtx, - cancel: cancel, - isBootstrapNode: isBootstrapNode, + storeAPI: storeAPI, + dstore: dstore, + db: db, + ctx: routingCtx, + cancel: cancel, + isBootstrapNode: isBootstrapNode, + reprovideInterval: reprovideInterval, } refreshInterval := RefreshInterval @@ -190,30 +101,12 @@ func newRemote(parentCtx context.Context, p2p.WithForceReachabilityPrivate(opts.Config().Routing.ForceReachabilityPrivate), p2p.WithForceReachabilityPublic(opts.Config().Routing.ForceReachabilityPublic), p2p.WithCustomDHTOpts( - func(h host.Host) ([]dht.Option, error) { - providerMgr, err := records.NewProviderManager(parentCtx, h.ID(), h.Peerstore(), dstore) - if err != nil { - return nil, fmt.Errorf("failed to create provider manager: %w", err) - } - - labelValidators := validators.CreateLabelValidators() - validator := record.NamespacedValidator{ - types.LabelTypeSkill.String(): labelValidators[types.LabelTypeSkill.String()], - types.LabelTypeDomain.String(): labelValidators[types.LabelTypeDomain.String()], - types.LabelTypeModule.String(): labelValidators[types.LabelTypeModule.String()], - } - + func(_ host.Host) ([]dht.Option, error) { return []dht.Option{ dht.Datastore(dstore), // custom DHT datastore dht.ProtocolPrefix(protocol.ID(ProtocolPrefix)), // custom DHT protocol prefix - dht.Validator(validator), // custom validators for label namespaces dht.MaxRecordAge(RecordTTL), // set consistent TTL for all DHT records dht.Mode(dht.ModeServer), - dht.ProviderStore(&handler{ - ProviderManager: providerMgr, - hostID: h.ID().String(), - notifyCh: routeAPI.notifyCh, - }), }, nil }, ), @@ -224,7 +117,7 @@ func newRemote(parentCtx context.Context, routeAPI.server = server - rpcService, err := rpc.New(server.Host(), storeAPI) + rpcService, err := rpc.New(server.Host(), storeAPI, db) if err != nil { defer server.Close() @@ -233,70 +126,19 @@ func newRemote(parentCtx context.Context, routeAPI.service = rpcService - // Initialize GossipSub manager if enabled - // Protocol parameters (topic, message size) are defined in pubsub.constants - // and are NOT configurable to ensure network-wide compatibility - if opts.Config().Routing.GossipSub.Enabled { - // Use parent context for GossipSub (should live as long as the server) - pubsubManager, err := pubsub.New(parentCtx, server.Host()) - if err != nil { - defer server.Close() - - return nil, fmt.Errorf("failed to create pubsub manager: %w", err) - } - - routeAPI.pubsubManager = pubsubManager - - // Set callback for received label announcements - pubsubManager.SetOnRecordPublishEvent(routeAPI.handleRecordPublishEvent) - - // Start periodic mesh peer tagging to protect them from Connection Manager pruning - routeAPI.startMeshPeerTagging() - - remoteLogger.Info("GossipSub label announcements enabled") - } else { - remoteLogger.Info("GossipSub disabled, using DHT+Pull fallback only") - } - - // Pass Publish as callback to avoid circular dependency - // The method value captures routeAPI's state (server, pubsubManager) - routeAPI.cleanupManager = NewCleanupManager(dstore, storeAPI, server, routeAPI.Publish, opts.Config().Routing.RepublishInterval) - - // Initialize DHT autosync if enabled (deny-by-default). The manager pulls and - // ingests records/referrers announced by trusted peers, off the notification - // handler goroutine. - if autosyncCfg.Enabled { - routeAPI.autosyncMgr = autosync.NewManager(autosyncPeers, rpcService, server, ingestor, storeAPI, validator) - //nolint:contextcheck // Intentionally passing routing context to worker goroutines for lifecycle management - routeAPI.autosyncMgr.Start(routeAPI.ctx, &routeAPI.wg) - - remoteLogger.Info("DHT autosync enabled", "trusted_peers", len(autosyncPeers)) - } - - // Start all background goroutines with routing context - routeAPI.wg.Add(1) - - go routeAPI.handleNotify() - - routeAPI.wg.Add(1) //nolint:contextcheck // Intentionally passing routing context to child goroutine for lifecycle management - go routeAPI.cleanupManager.StartLabelRepublishTask(routeAPI.ctx, &routeAPI.wg) - - routeAPI.wg.Add(1) - //nolint:contextcheck // Intentionally passing routing context to child goroutine for lifecycle management - go routeAPI.cleanupManager.StartRemoteLabelCleanupTask(routeAPI.ctx, &routeAPI.wg) + routeAPI.startAdvertiseTask() return routeAPI, nil } -// Publish announces a record to the network via DHT and GossipSub. -// This method is part of the RoutingAPI interface and is also used -// by CleanupManager for republishing via method value injection. +// Publish announces a record to the DHT: the CID so the record can be fetched, +// and its labels so it can be found. // // Flow: // 1. Validate and extract CID from record // 2. Announce CID to DHT (critical - returns error if fails) -// 3. Publish record via GossipSub (best-effort - logs warning if fails) +// 3. Announce the record's labels and their ancestors (best-effort) // // Parameters: // - ctx: Operation context @@ -330,33 +172,27 @@ func (r *routeRemote) Publish(ctx context.Context, record coretypes.Record) erro return status.Errorf(codes.Internal, "failed to announce CID to DHT: %v", err) } - // 2. Publish record via GossipSub (if enabled) - // This provides efficient label propagation to ALL subscribed peers - if r.pubsubManager != nil { - if err := r.pubsubManager.PublishRecord(ctx, record); err != nil { - // Log warning but don't fail - DHT announcement already succeeded - // Remote peers can still discover via DHT+Pull fallback - remoteLogger.Warn("Failed to publish record via GossipSub", - "cid", cidStr, - "error", err, - "fallback", "DHT+Pull will handle discovery") - } else { - remoteLogger.Debug("Successfully published record via GossipSub", - "cid", cidStr, - "topicPeers", len(r.pubsubManager.GetTopicPeers())) - } + // 2. Announce the record's labels, and their ancestors, as DHT keys. + // This is what lets a peer searching for /skills/A reach a record tagged + // /skills/A/B without any node holding a global index. + labels := expandLabels(types.GetLabelsFromRecord(record)) + if failed := r.provideLabels(ctx, labels); failed > 0 { + remoteLogger.Warn("Some label announcements failed", + "cid", cidStr, + "labels", len(labels), + "failed", failed) } remoteLogger.Debug("Successfully announced record to network", "cid", cidStr, - "dhtPeers", r.server.DHT().RoutingTable().Size(), - "gossipSubEnabled", r.pubsubManager != nil) + "labelKeys", len(labels), + "dhtPeers", r.server.DHT().RoutingTable().Size()) return nil } -// Search queries remote records using cached labels with OR logic and minimum threshold. -// Records are returned if they match at least minMatchScore queries (OR relationship). +// Search normalises the request and streams the matches on a channel. Queries +// are OR'd: a record is returned once it matches minMatchScore of them. func (r *routeRemote) Search(ctx context.Context, req *routingv1.SearchRequest) (<-chan *routingv1.SearchResponse, error) { remoteLogger.Debug("Called remote routing's Search method", "req", req) @@ -388,245 +224,6 @@ func (r *routeRemote) Search(ctx context.Context, req *routingv1.SearchRequest) return outCh, nil } -// searchRemoteRecords searches for remote records using cached labels with OR logic. -// Records are returned if they match at least minMatchScore queries. -// -//nolint:gocognit // Core search algorithm requires complex logic for namespace iteration, filtering, and scoring -func (r *routeRemote) searchRemoteRecords(ctx context.Context, queries []*routingv1.RecordQuery, limit uint32, minMatchScore uint32, outCh chan<- *routingv1.SearchResponse) { - localPeerID := r.server.Host().ID().String() - processedCIDs := make(map[string]bool) // Avoid duplicates - processedCount := 0 - limitInt := int(limit) - - remoteLogger.Debug("Starting remote search with OR logic and minimum threshold", "queries", len(queries), "minMatchScore", minMatchScore, "localPeerID", localPeerID) - - // Query all namespaces to find remote records - entries, err := QueryAllNamespaces(ctx, r.dstore) - if err != nil { - remoteLogger.Error("Failed to get namespace entries for search", "error", err) - - return - } - - for _, entry := range entries { - if limitInt > 0 && processedCount >= limitInt { - break - } - - _, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - remoteLogger.Warn("Failed to parse enhanced label key", "key", entry.Key, "error", err) - - continue - } - - // Filter for remote records only (exclude local records) - if keyPeerID == localPeerID { - continue // Skip local records - } - - // Avoid duplicate CIDs (same record might have multiple matching labels) - if processedCIDs[keyCID] { - continue - } - - // Calculate match score using OR logic (how many queries match this record) - matchQueries, score := r.calculateMatchScore(ctx, keyCID, queries, keyPeerID) - - remoteLogger.Debug("Calculated match score for remote record", "cid", keyCID, "score", score, "minMatchScore", minMatchScore, "matchingQueries", len(matchQueries)) - - // Apply minimum match score filter (record included if score β‰₯ threshold) - if score >= minMatchScore { - peer := r.createPeerInfo(ctx, keyPeerID) - - outCh <- &routingv1.SearchResponse{ - RecordRef: &corev1.RecordRef{Cid: keyCID}, - Peer: peer, - MatchQueries: matchQueries, - MatchScore: score, - } - - processedCIDs[keyCID] = true - processedCount++ - - remoteLogger.Debug("Record meets minimum threshold, including in results", "cid", keyCID, "score", score) - - if limitInt > 0 && processedCount >= limitInt { - break - } - } else { - remoteLogger.Debug("Record does not meet minimum threshold, excluding from results", "cid", keyCID, "score", score, "minMatchScore", minMatchScore) - } - } - - remoteLogger.Debug("Completed Search operation", "processed", processedCount, "queries", len(queries)) -} - -// calculateMatchScore calculates how many queries match a remote record (OR logic). -// Returns the matching queries and the match score for minimum threshold filtering. -func (r *routeRemote) calculateMatchScore(ctx context.Context, cid string, queries []*routingv1.RecordQuery, peerID string) ([]*routingv1.RecordQuery, uint32) { - if len(queries) == 0 { - return nil, 0 - } - - labels := r.getRemoteRecordLabels(ctx, cid, peerID) - if len(labels) == 0 { - return nil, 0 - } - - var matchingQueries []*routingv1.RecordQuery - - // Check each query against all labels - any match counts toward the score (OR logic) - for _, query := range queries { - if QueryMatchesLabels(query, labels) { - matchingQueries = append(matchingQueries, query) - } - } - - score := safeIntToUint32(len(matchingQueries)) - - remoteLogger.Debug("OR logic match score calculated", "cid", cid, "total_queries", len(queries), "matching_queries", len(matchingQueries), "score", score) - - return matchingQueries, score -} - -// getRemoteRecordLabels gets labels for a remote record by finding all enhanced keys for this CID/PeerID. -func (r *routeRemote) getRemoteRecordLabels(ctx context.Context, cid, peerID string) []types.Label { - var labelList []types.Label - - entries, err := QueryAllNamespaces(ctx, r.dstore) - if err != nil { - remoteLogger.Error("Failed to get namespace entries for labels", "error", err) - - return nil - } - - for _, entry := range entries { - label, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - continue - } - - if keyCID == cid && keyPeerID == peerID { - labelList = append(labelList, label) - } - } - - return labelList -} - -// createPeerInfo creates a Peer message from a PeerID string, advertising the -// peer's Directory API (/dir/) and OCI registry (/oci/) endpoints when known. -// Addresses are returned in prefixed multiaddr form (e.g. "/dir/host:443", -// "/oci/host:5000") so the consumer can tell them apart. Missing endpoints are -// omitted (the slice may be empty). -func (r *routeRemote) createPeerInfo(ctx context.Context, peerID string) *routingv1.Peer { - addrs := make([]string, 0, 2) //nolint:mnd // dir + oci - - if v := r.getPeerProtocolAddress(ctx, peerID, p2p.DirProtocolCode); v != "" { - addrs = append(addrs, "/"+p2p.DirProtocol+"/"+v) - } - - if v := r.getPeerProtocolAddress(ctx, peerID, p2p.OciProtocolCode); v != "" { - addrs = append(addrs, "/"+p2p.OciProtocol+"/"+v) - } - - return &routingv1.Peer{ - Id: peerID, - Addrs: addrs, - } -} - -// getPeerProtocolAddress returns the value of the given custom multiaddr -// protocol (e.g. DirProtocolCode, OciProtocolCode) advertised by the peer, -// checking the datastore cache first and falling back to the live peerstore. -// Returns "" if the peer advertises no such address. -func (r *routeRemote) getPeerProtocolAddress(ctx context.Context, peerID string, code int) string { - // Try datastore cache first (fast path) - if addr := r.getPeerProtocolAddressFromDatastore(ctx, peerID, code); addr != "" { - return addr - } - - // Fallback: Try live peerstore (handles mDNS and DHT without addresses) - pid, err := peer.Decode(peerID) - if err != nil { - remoteLogger.Error("Failed to decode peer ID", "peerID", peerID, "error", err) - - return "" - } - - peerstoreAddrs := r.server.Host().Peerstore().Addrs(pid) - if len(peerstoreAddrs) == 0 { - return "" - } - - return extractProtocolValue(peerstoreAddrs, code) -} - -// getPeerProtocolAddressFromDatastore checks the datastore cache for the peer's -// advertised value of the given multiaddr protocol code. -func (r *routeRemote) getPeerProtocolAddressFromDatastore(ctx context.Context, peerID string, code int) string { - key := datastore.NewKey("peer_addrs/" + peerID) - - addresses, err := r.dstore.Get(ctx, key) - if err != nil { - return "" - } - - var multiaddrs []ma.Multiaddr - if err := json.Unmarshal(addresses, &multiaddrs); err != nil { - remoteLogger.Error("Failed to unmarshal peer addresses", "error", err) - - return "" - } - - return extractProtocolValue(multiaddrs, code) -} - -// storePeerAddresses stores peer addresses in datastore for later retrieval. -// Tries DHT notification addresses first, falls back to peerstore if empty. -func (r *routeRemote) storePeerAddresses(ctx context.Context, peerIDStr string, peerID peer.ID, notifAddrs []ma.Multiaddr, cid string) { - // Try DHT notification addresses first - peerAddrs := notifAddrs - if len(peerAddrs) == 0 { - // Fallback: get addresses from libp2p peerstore - peerAddrs = r.server.Host().Peerstore().Addrs(peerID) - remoteLogger.Debug("DHT notification had no addresses, using peerstore", - "peerID", peerIDStr, - "peerstoreAddrs", len(peerAddrs)) - } - - if len(peerAddrs) == 0 { - remoteLogger.Warn("No peer addresses available from DHT or peerstore", - "peerID", peerIDStr, - "cid", cid) - - return - } - - // Check if already stored - key := datastore.NewKey("peer_addrs/" + peerIDStr) - if _, err := r.dstore.Get(ctx, key); err == nil { - return // Already have addresses - } - - // Marshal and store - addresses, err := json.Marshal(peerAddrs) - if err != nil { - remoteLogger.Error("Failed to marshal peer addresses", "error", err) - - return - } - - if err := r.dstore.Put(ctx, key, addresses); err != nil { - remoteLogger.Error("Failed to store peer addresses", "error", err) - - return - } - - remoteLogger.Debug("Stored peer addresses", "peerID", peerIDStr, "count", len(peerAddrs)) -} - // extractProtocolValue returns the value of the given multiaddr protocol code // from the first address that carries it, or "" if none do. The stored value is // percent-encoded (see p2p.EncodeAppAddr), so it is decoded back to its original @@ -641,394 +238,18 @@ func extractProtocolValue(multiaddrs []ma.Multiaddr, code int) string { return "" } -func (r *routeRemote) handleNotify() { - defer r.wg.Done() - - cleanupLogger.Debug("Started DHT provider notification handler") - - // Process DHT provider notifications and handle pull-based label discovery - for { - select { - case <-r.ctx.Done(): - cleanupLogger.Debug("DHT provider notification handler stopped") - - return - case notif := <-r.notifyCh: - // All announcements are now CID provider announcements - // Labels are discovered via pull-based mechanism - r.handleCIDProviderNotification(r.ctx, notif) - } - } -} - -// startMeshPeerTagging starts a background goroutine that periodically tags -// GossipSub mesh peers to protect them from Connection Manager pruning. -// -// GossipSub mesh changes over time as peers join/leave and mesh prunes/grafts. -// This periodic tagging ensures current mesh peers are always protected with -// high priority (50 points), preventing the Connection Manager from disconnecting -// them when connection limits are reached. -// -// The goroutine: -// - Tags mesh peers immediately (initial protection) -// - Re-tags every 30 seconds (maintain protection as mesh changes) -// - Stops when routing context is cancelled (clean shutdown) -// -// This method should only be called when GossipSub is enabled. -func (r *routeRemote) startMeshPeerTagging() { - if r.pubsubManager == nil { - return // Safety check: only run if GossipSub is enabled - } - - // Tag mesh peers initially - r.pubsubManager.TagMeshPeers() - - // Start periodic tagging goroutine - r.wg.Add(1) - - r.wg.Go(func() { - defer r.wg.Done() - - ticker := time.NewTicker(p2p.MeshPeerTaggingInterval) - defer ticker.Stop() - - remoteLogger.Info("Started periodic GossipSub mesh peer tagging", - "interval", p2p.MeshPeerTaggingInterval) - - for { - select { - case <-r.ctx.Done(): - remoteLogger.Debug("Stopping mesh peer tagging") - - return - case <-ticker.C: - r.pubsubManager.TagMeshPeers() - } - } - }) -} - -// handleCIDProviderNotification implements fallback label discovery via DHT+Pull. -// This is the secondary mechanism when GossipSub labels haven't arrived yet. -// -// Flow: -// 1. Check if labels already cached (from GossipSub) β†’ Update timestamps, skip pull -// 2. If not cached β†’ FALLBACK: Pull record, extract labels, cache -// -// Timing scenarios: -// - 90% case: GossipSub arrives first (~15ms) β†’ This function skips pull (efficient!) -// - 10% case: DHT arrives first (~80ms) β†’ This function pulls (fallback) -// -// This ensures labels are always cached regardless of network race conditions. -func (r *routeRemote) handleCIDProviderNotification(ctx context.Context, notif *handlerSync) { - peerIDStr := notif.Peer.ID.String() - - if peerIDStr == r.server.Host().ID().String() { - remoteLogger.Debug("Ignoring self announcement", "cid", notif.Ref.GetCid()) - - return - } - - // Store peer addresses for later use - r.storePeerAddresses(ctx, peerIDStr, notif.Peer.ID, notif.Peer.Addrs, notif.Ref.GetCid()) - - // DHT autosync: if enabled and the announcing peer is trusted, schedule a - // pull+ingest of the record (and its referrers). Non-blocking and independent - // of the label-discovery logic below. - if r.autosyncMgr != nil { - r.autosyncMgr.MaybeEnqueue(notif.Ref, notif.Peer) - } - - // Check if we already have labels cached (from GossipSub announcement) - if r.hasRemoteRecordCached(ctx, notif.Ref.GetCid(), peerIDStr) { - // Labels already cached via GossipSub or previous pull - // Just update lastSeen timestamps for freshness - remoteLogger.Debug("Labels already cached (likely from GossipSub), updating lastSeen", - "cid", notif.Ref.GetCid(), - "peer", peerIDStr, - "source", "gossipsub_or_previous_pull") - - r.updateRemoteRecordLastSeen(ctx, notif.Ref.GetCid(), peerIDStr) - - return - } - - // FALLBACK: Labels not cached yet, need to pull record - // This happens when: - // - GossipSub message hasn't arrived yet (race condition) - // - GossipSub is disabled - // - GossipSub message was lost - // - Peer doesn't support GossipSub - remoteLogger.Debug("No cached labels, falling back to pull-based discovery", - "cid", notif.Ref.GetCid(), - "peer", peerIDStr, - "reason", "gossipsub_not_received") - - record, err := r.service.Pull(ctx, notif.Peer.ID, notif.Ref) - if err != nil { - remoteLogger.Error("Failed to pull remote content for label caching", - "cid", notif.Ref.GetCid(), - "peer", peerIDStr, - "error", err) - - return - } - - adapter, err := record.Decode() - if err != nil { - remoteLogger.Error("Failed to get record adapter for label extraction", - "cid", notif.Ref.GetCid(), - "peer", peerIDStr, - "error", err) - - return - } - - labelList := types.GetLabelsFromRecord(adapter) - if len(labelList) == 0 { - remoteLogger.Warn("No labels found in remote record", - "cid", notif.Ref.GetCid(), - "peer", peerIDStr) - - return - } - - now := time.Now() - cachedCount := 0 - - for _, label := range labelList { - enhancedKey := BuildEnhancedLabelKey(label, notif.Ref.GetCid(), peerIDStr) - - metadata := &types.LabelMetadata{ - Timestamp: now, - LastSeen: now, - } - - metadataBytes, err := json.Marshal(metadata) - if err != nil { - remoteLogger.Warn("Failed to marshal label metadata", - "enhanced_key", enhancedKey, - "error", err) - - continue - } - - err = r.dstore.Put(ctx, datastore.NewKey(enhancedKey), metadataBytes) - if err != nil { - remoteLogger.Warn("Failed to cache remote label", - "enhanced_key", enhancedKey, - "error", err) - } else { - cachedCount++ - } - } - - remoteLogger.Info("Successfully cached labels via DHT+Pull fallback", - "cid", notif.Ref.GetCid(), - "peer", peerIDStr, - "totalLabels", len(labelList), - "cached", cachedCount, - "source", "pull_fallback") -} - -// hasRemoteRecordCached checks if we already have cached labels for this remote record. -// This helps avoid duplicate work and identifies reannouncement events. -func (r *routeRemote) hasRemoteRecordCached(ctx context.Context, cid, peerID string) bool { - entries, err := QueryAllNamespaces(ctx, r.dstore) - if err != nil { - remoteLogger.Error("Failed to get namespace entries for cache check", "error", err) - - return false - } - - for _, entry := range entries { - // Parse enhanced key to check if it matches our CID/PeerID - _, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - continue - } - - if keyCID == cid && keyPeerID == peerID { - return true - } - } - - return false -} - -// handleRecordPublishEvent processes incoming record publication events from GossipSub. -// This is the primary label discovery mechanism when GossipSub is enabled. -// It converts the wire format to storage format using existing infrastructure. -// -// Parameters: -// - ctx: Operation context -// - authenticatedPeerID: Cryptographically verified author from the signed -// GossipSub message (msg.GetFrom) β€” the record publisher, not the forwarder -// - event: The announcement payload (CID, labels, timestamp) -// -// Flow: -// 1. Skip own announcements (already cached locally) -// 2. Convert []string labels to types.Label -// 3. Build enhanced keys: /skills/AI/CID/PeerID -// 4. Store types.LabelMetadata in datastore -// -// Security: -// - Uses authenticatedPeerID from libp2p transport (cannot be spoofed) -// - Prevents malicious peers from poisoning the label cache -// -// This completely avoids pulling the entire record from remote peers, -// providing ~95% bandwidth savings and ~5-20ms propagation time. -func (r *routeRemote) handleRecordPublishEvent(ctx context.Context, authenticatedPeerID string, event *pubsub.RecordPublishEvent) { - // Skip our own announcements (already cached during local Publish) - if authenticatedPeerID == r.server.Host().ID().String() { - return - } - - // DHT autosync trigger via GossipSub. GossipSub reaches NAT'd subscribers - // reliably (mesh-forwarded) where the DHT provider notification often does - // not, so it is the primary cross-NAT trigger. The authenticated peer ID is - // the signed message author (the publisher); the worker resolves its - // addresses via FindPeer before pulling. Non-blocking and independent of the - // label caching below. - if r.autosyncMgr != nil { - if authorID, err := peer.Decode(authenticatedPeerID); err == nil { - r.autosyncMgr.MaybeEnqueue(&corev1.RecordRef{Cid: event.CID}, peer.AddrInfo{ID: authorID}) - } else { - remoteLogger.Warn("Failed to decode authenticated peer ID for autosync", - "peer", authenticatedPeerID, "error", err) - } - } - - remoteLogger.Info("Caching labels from GossipSub announcement", - "cid", event.CID, - "peer", authenticatedPeerID, - "labels", len(event.Labels)) - - now := time.Now() - cachedCount := 0 - - // Convert wire format ([]string) to storage format using existing infrastructure - for _, labelStr := range event.Labels { - label := types.Label(labelStr) - - // Use authenticated peer ID (cryptographically verified by libp2p) - enhancedKey := BuildEnhancedLabelKey(label, event.CID, authenticatedPeerID) - - // Use existing types.LabelMetadata structure - metadata := &types.LabelMetadata{ - Timestamp: event.Timestamp, // When label was announced - LastSeen: now, // When we received it - } - - metadataBytes, err := json.Marshal(metadata) - if err != nil { - remoteLogger.Warn("Failed to marshal label metadata", - "key", enhancedKey, - "error", err) - - continue - } - - err = r.dstore.Put(ctx, datastore.NewKey(enhancedKey), metadataBytes) - if err != nil { - remoteLogger.Warn("Failed to cache label from GossipSub", - "key", enhancedKey, - "error", err) - } else { - cachedCount++ - } - } - - remoteLogger.Info("Successfully cached labels from GossipSub", - "cid", event.CID, - "peer", authenticatedPeerID, - "total", len(event.Labels), - "cached", cachedCount) -} - -// updateLabelMetadataTimestamp updates the lastSeen timestamp for a single cached label entry. -func (r *routeRemote) updateLabelMetadataTimestamp(ctx context.Context, key string, value []byte, timestamp time.Time) error { - var metadata types.LabelMetadata - if err := json.Unmarshal(value, &metadata); err != nil { - return fmt.Errorf("failed to unmarshal label metadata: %w", err) - } - - metadata.LastSeen = timestamp - - metadataBytes, err := json.Marshal(metadata) - if err != nil { - return fmt.Errorf("failed to marshal label metadata: %w", err) - } - - err = r.dstore.Put(ctx, datastore.NewKey(key), metadataBytes) - if err != nil { - return fmt.Errorf("failed to save label metadata: %w", err) - } - - return nil -} - -// updateRemoteRecordLastSeen updates the lastSeen timestamp for all cached labels -// from a specific remote peer/CID combination (for reannouncement handling). -func (r *routeRemote) updateRemoteRecordLastSeen(ctx context.Context, cid, peerID string) { - now := time.Now() - updatedCount := 0 - - entries, err := QueryAllNamespaces(ctx, r.dstore) - if err != nil { - remoteLogger.Error("Failed to get namespace entries for lastSeen update", "error", err) - - return - } - - for _, entry := range entries { - // Parse enhanced key to check if it matches our CID/PeerID - _, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - continue - } - - if keyCID == cid && keyPeerID == peerID { - if err := r.updateLabelMetadataTimestamp(ctx, entry.Key, entry.Value, now); err != nil { - remoteLogger.Warn("Failed to update lastSeen for cached label", "key", entry.Key, "error", err) - } else { - updatedCount++ - - remoteLogger.Debug("Updated lastSeen for cached label", "key", entry.Key) - } - } - } - - remoteLogger.Debug("Updated lastSeen timestamps for reannounced record", - "cid", cid, "peer", peerID, "updatedLabels", updatedCount) -} - // Stop stops the remote routing services and releases resources. // This should be called during server shutdown to clean up gracefully. func (r *routeRemote) Stop() error { remoteLogger.Info("Stopping routing subsystem") - // Cancel routing context to stop all background goroutines: - // - handleNotify (DHT provider notifications) - // - StartLabelRepublishTask (periodic republishing) - // - StartRemoteLabelCleanupTask (stale label cleanup) + // Cancel routing context to stop all background goroutines. r.cancel() // Wait for all goroutines to finish gracefully r.wg.Wait() remoteLogger.Debug("All routing background tasks stopped") - // Close GossipSub manager if enabled - if r.pubsubManager != nil { - if err := r.pubsubManager.Close(); err != nil { - remoteLogger.Error("Failed to close GossipSub manager", "error", err) - - return fmt.Errorf("failed to close pubsub manager: %w", err) - } - - remoteLogger.Debug("GossipSub manager closed") - } - // Close p2p server (host and DHT) r.server.Close() remoteLogger.Debug("P2P server closed") @@ -1045,7 +266,6 @@ func (r *routeRemote) Stop() error { // For regular nodes (connecting to existing network): // - DHT must have peers in routing table // - Must have connected peers -// - GossipSub mesh must be formed (if enabled) func (r *routeRemote) IsReady(ctx context.Context) bool { if r.server == nil { remoteLogger.Debug("Routing not ready: server is nil") @@ -1109,20 +329,7 @@ func (r *routeRemote) IsReady(ctx context.Context) bool { return false } - // If GossipSub is enabled, check if mesh is formed - // Bootstrap nodes may have 0 mesh peers initially, which is acceptable - if r.pubsubManager != nil { - meshPeers := r.pubsubManager.GetMeshPeerCount() - if meshPeers == 0 { - remoteLogger.Debug("Routing not ready: GossipSub mesh has no peers") - - return false - } - - remoteLogger.Debug("Routing ready", "routingTableSize", routingTableSize, "connectedPeers", connectedPeers, "meshPeers", meshPeers) - } else { - remoteLogger.Debug("Routing ready", "routingTableSize", routingTableSize, "connectedPeers", connectedPeers) - } + remoteLogger.Debug("Routing ready", "routingTableSize", routingTableSize, "connectedPeers", connectedPeers) return true } diff --git a/server/routing/routing_remote_or_logic_test.go b/server/routing/routing_remote_or_logic_test.go deleted file mode 100644 index 4c92637f3..000000000 --- a/server/routing/routing_remote_or_logic_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "encoding/json" - "testing" - "time" - - routingv1 "github.com/agntcy/dir/api/routing/v1" - "github.com/agntcy/dir/server/datastore" - "github.com/agntcy/dir/server/types" - ipfsdatastore "github.com/ipfs/go-datastore" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// This test bypasses DHT infrastructure issues and directly tests the calculateMatchScore method. -func TestRemoteSearch_ORLogicWithMinMatchScore(t *testing.T) { - ctx := t.Context() - - // Create test datastore - dstore, cleanup := setupTestDatastore(t) - defer cleanup() - - // Create routeRemote instance for testing - r := &routeRemote{ - dstore: dstore, - } - - // Setup test scenario: simulate cached remote announcements - testPeerID := "remote-peer-test" - testCID := "test-record-cid" - - // Simulate Peer 1 announced these skills for the test record - skillLabels := []string{ - "/skills/Natural Language Processing/Text Completion", - "/skills/Natural Language Processing/Problem Solving", - } - - // Store enhanced label announcements in datastore (simulating DHT cache) - for _, label := range skillLabels { - enhancedKey := BuildEnhancedLabelKey(types.Label(label), testCID, testPeerID) - metadata := &types.LabelMetadata{ - Timestamp: time.Now(), - LastSeen: time.Now(), - } - metadataBytes, err := json.Marshal(metadata) - require.NoError(t, err) - - err = dstore.Put(ctx, ipfsdatastore.NewKey(enhancedKey), metadataBytes) - require.NoError(t, err) - } - - t.Run("OR Logic Success - 2/3 queries match", func(t *testing.T) { - // Test queries: 2 real skills + 1 fake skill - queries := []*routingv1.RecordQuery{ - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "Natural Language Processing/Text Completion"}, - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "Natural Language Processing/Problem Solving"}, - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "NonexistentSkill"}, - } - - // Test calculateMatchScore directly (avoids server dependency) - matchQueries, score := r.calculateMatchScore(ctx, testCID, queries, testPeerID) - - // Should have 2 matching queries out of 3 - assert.Len(t, matchQueries, 2, "Should have 2 matching queries") - assert.Equal(t, uint32(2), score, "Score should be 2 (2 out of 3 queries matched)") - - // Test that minScore=2 would include this record - assert.GreaterOrEqual(t, score, uint32(2), "Score meets minScore=2 threshold") - - // Test that minScore=3 would exclude this record - assert.Less(t, score, uint32(3), "Score does not meet minScore=3 threshold") - }) - - t.Run("Single Query Match", func(t *testing.T) { - // Single query that should match - queries := []*routingv1.RecordQuery{ - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "Natural Language Processing/Text Completion"}, - } - - // Test calculateMatchScore - matchQueries, score := r.calculateMatchScore(ctx, testCID, queries, testPeerID) - - // Should have 1 matching query - assert.Len(t, matchQueries, 1, "Should have 1 matching query") - assert.Equal(t, uint32(1), score, "Score should be 1") - }) - - t.Run("Perfect Match - 2/2 queries match", func(t *testing.T) { - // Two queries that should both match - queries := []*routingv1.RecordQuery{ - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "Natural Language Processing/Text Completion"}, - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "Natural Language Processing/Problem Solving"}, - } - - // Test calculateMatchScore - matchQueries, score := r.calculateMatchScore(ctx, testCID, queries, testPeerID) - - // Should have 2 matching queries out of 2 - assert.Len(t, matchQueries, 2, "Should have 2 matching queries") - assert.Equal(t, uint32(2), score, "Score should be 2 (both queries matched)") - }) - - t.Run("No Queries Match", func(t *testing.T) { - // Query that doesn't match anything - queries := []*routingv1.RecordQuery{ - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "NonexistentSkill"}, - } - - // Test calculateMatchScore - matchQueries, score := r.calculateMatchScore(ctx, testCID, queries, testPeerID) - - // Should have 0 matching queries - assert.Empty(t, matchQueries, "Should have 0 matching queries") - assert.Equal(t, uint32(0), score, "Score should be 0") - }) - - t.Run("Empty Queries", func(t *testing.T) { - // No queries - var queries []*routingv1.RecordQuery - - // Test calculateMatchScore - matchQueries, score := r.calculateMatchScore(ctx, testCID, queries, testPeerID) - - // Should have 0 matching queries and 0 score - assert.Empty(t, matchQueries, "Should have 0 matching queries with empty query list") - assert.Equal(t, uint32(0), score, "Score should be 0 with empty queries") - }) - - t.Run("Hierarchical Skill Matching", func(t *testing.T) { - // Test hierarchical skill matching (prefix matching) - queries := []*routingv1.RecordQuery{ - {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: "Natural Language Processing"}, // Should match both skills via prefix - } - - // Test calculateMatchScore - matchQueries, score := r.calculateMatchScore(ctx, testCID, queries, testPeerID) - - // Should match at least 1 query (hierarchical matching) - assert.GreaterOrEqual(t, len(matchQueries), 1, "Should have at least 1 matching query with hierarchical matching") - assert.GreaterOrEqual(t, score, uint32(1), "Score should be at least 1 with hierarchical matching") - }) -} - -// setupTestDatastore creates a test datastore for routing tests. -func setupTestDatastore(t *testing.T) (types.Datastore, func()) { - t.Helper() - - dstore, err := datastore.New() - require.NoError(t, err) - - cleanup := func() { - if closer, ok := dstore.(interface{ Close() error }); ok { - closer.Close() - } - } - - return dstore, cleanup -} diff --git a/server/routing/rpc/query_records.go b/server/routing/rpc/query_records.go new file mode 100644 index 000000000..904311bf6 --- /dev/null +++ b/server/routing/rpc/query_records.go @@ -0,0 +1,231 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package rpc + +import ( + "context" + "fmt" + "strings" + + "github.com/agntcy/dir/server/types" + rpc "github.com/libp2p/go-libp2p-gorpc" + "github.com/libp2p/go-libp2p/core/peer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + DirServiceFuncQueryRecords = "QueryRecords" + + // MaxQueryRecords caps how many records one query call may return, + // whatever limit the caller asks for. + // + // This is a unary call rather than a stream: gorpc's streaming path hands a + // reflect.Value to the codec, and ugorji dropped its reflect.Value case in + // v1.3.1, so streamed values serialise as empty structs. + MaxQueryRecords = 1000 +) + +// RecordQuery is the wire form of a routing search query. +// +// routingv1.RecordQuery is a protobuf message and this transport is msgpack, so +// queries cross as plain structs rather than relying on reflection over +// generated protobuf internals. +type RecordQuery struct { + // Type is the label namespace being queried, matching types.LabelType: + // "skills", "domains", "modules" or "locators". + Type string + + // Value is the label value without its namespace, e.g. "AI/ML". + Value string +} + +type QueryRecordsRequest struct { + // Queries are OR'd: a record is returned if it matches any of them. The + // caller scores and thresholds the results itself, which is why every + // match ships its full label set. + Queries []RecordQuery + + // Limit caps the returned records. Zero means MaxQueryRecords. + Limit uint32 +} + +// RecordMatch is one record the peer holds that matched the query. +type RecordMatch struct { + Cid string + + // Labels is the record's complete label set, namespaced, so the caller can + // score it against the original queries without a second round trip. + Labels []string +} + +type QueryRecordsResponse struct { + Records []RecordMatch + + // Truncated reports that the peer had more matches than the limit allowed. + Truncated bool +} + +// QueryRecords answers a peer's search over the records this node holds. +// +// It returns candidates, not decisions: matching is deliberately permissive +// (any query, prefix-inclusive) and the caller applies its own match score. +func (r *RPCAPI) QueryRecords(ctx context.Context, in *QueryRecordsRequest, out *QueryRecordsResponse) error { + if in == nil || out == nil { + return status.Error(codes.InvalidArgument, "invalid request: nil request/response") //nolint:wrapcheck + } + + requestPeer, _ := rpc.GetRequestSender(ctx) + + logger.Debug("P2p RPC: Executing QueryRecords request on remote peer", + "peer", r.service.localPeerID(), + "request_peer", requestPeer, + "queries", len(in.Queries), + "limit", in.Limit, + ) + + if len(in.Queries) == 0 { + return status.Error(codes.InvalidArgument, "at least one query is required") //nolint:wrapcheck + } + + db, err := r.service.getDatabase() + if err != nil { + return err //nolint:wrapcheck + } + + limit := int(min(in.Limit, MaxQueryRecords)) + if limit == 0 { + limit = MaxQueryRecords + } + + cids, truncated, err := matchingCIDs(db, in.Queries, limit) + if err != nil { + return err //nolint:wrapcheck + } + + labels, err := db.GetRecordLabels(cids) + if err != nil { + return status.Errorf(codes.Internal, "failed to load record labels: %v", err) + } + + matches := make([]RecordMatch, 0, len(cids)) + + for _, cid := range cids { + recordLabels := labels[cid] + if len(recordLabels) == 0 { + // The record matched a label filter, so its labels vanished between + // the two queries. Nothing useful to score against. + continue + } + + asStrings := make([]string, len(recordLabels)) + for i, label := range recordLabels { + asStrings[i] = label.String() + } + + matches = append(matches, RecordMatch{Cid: cid, Labels: asStrings}) + } + + logger.Debug("P2p RPC: QueryRecords served matches", + "request_peer", requestPeer, + "candidates", len(cids), + "returned", len(matches), + "truncated", truncated, + ) + + *out = QueryRecordsResponse{Records: matches, Truncated: truncated} + + return nil +} + +// matchingCIDs runs each query separately and unions the results, reporting +// whether the limit cut the union short. +// +// One query per round trip because the filter API AND's different label kinds +// together, whereas routing search OR's its queries. Query counts are small. +func matchingCIDs(db types.DatabaseAPI, queries []RecordQuery, limit int) ([]string, bool, error) { + seen := make(map[string]struct{}, limit) + union := make([]string, 0, limit) + + for _, query := range queries { + filters, err := queryFilters(query, limit) + if err != nil { + logger.Debug("Skipping unusable query", "type", query.Type, "value", query.Value, "error", err) + + continue + } + + cids, err := db.GetRecordCIDs(filters...) + if err != nil { + return nil, false, status.Errorf(codes.Internal, "failed to query records: %v", err) + } + + for _, cid := range cids { + if _, ok := seen[cid]; ok { + continue + } + + seen[cid] = struct{}{} + + union = append(union, cid) + + if len(union) >= limit { + return union, true, nil + } + } + } + + return union, false, nil +} + +// queryFilters translates one query into database filters. +// +// Hierarchical namespaces match the value itself or any descendant, so a query +// for "AI" finds "AI/ML" β€” the same prefix semantics the local matcher applies. +// Locators are flat and match exactly. +func queryFilters(query RecordQuery, limit int) ([]types.FilterOption, error) { + value := strings.TrimSpace(query.Value) + if value == "" { + return nil, fmt.Errorf("query value is empty") + } + + descendants := value + "/*" + + switch types.LabelType(query.Type) { + case types.LabelTypeSkill: + return []types.FilterOption{types.WithSkillNames(value, descendants), types.WithLimit(limit)}, nil + case types.LabelTypeDomain: + return []types.FilterOption{types.WithDomainNames(value, descendants), types.WithLimit(limit)}, nil + case types.LabelTypeModule: + return []types.FilterOption{types.WithModuleNames(value, descendants), types.WithLimit(limit)}, nil + case types.LabelTypeLocator: + return []types.FilterOption{types.WithLocatorTypes(value), types.WithLimit(limit)}, nil + case types.LabelTypeUnknown: + return nil, fmt.Errorf("unknown query type %q", query.Type) + default: + return nil, fmt.Errorf("unknown query type %q", query.Type) + } +} + +// QueryRecords asks a peer which of the records it holds match the queries. +func (s *Service) QueryRecords(ctx context.Context, peerID peer.ID, req *QueryRecordsRequest) ([]RecordMatch, error) { + if req == nil { + return nil, status.Error(codes.InvalidArgument, "query request is required") //nolint:wrapcheck + } + + logger.Debug("P2p RPC: Executing QueryRecords request on remote peer", "peer", peerID, "queries", len(req.Queries)) + + var resp QueryRecordsResponse + + err := s.rpcClient.CallContext(ctx, peerID, DirService, DirServiceFuncQueryRecords, req, &resp) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to query records on peer %s: %v", peerID, err) + } + + if resp.Truncated { + logger.Warn("Peer truncated its query results", "peer", peerID, "returned", len(resp.Records)) + } + + return resp.Records, nil +} diff --git a/server/routing/rpc/query_records_test.go b/server/routing/rpc/query_records_test.go new file mode 100644 index 000000000..ec40f7eb9 --- /dev/null +++ b/server/routing/rpc/query_records_test.go @@ -0,0 +1,422 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package rpc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agntcy/dir/server/types" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeQueryDB implements only the two methods QueryRecords needs; the rest of +// DatabaseAPI is inherited as nil and would panic if ever touched, which is the +// point. +type fakeQueryDB struct { + types.DatabaseAPI + + // cidsByFilter is consulted with the SkillNames/DomainNames/ModuleNames/ + // LocatorTypes the handler derived, so tests can assert on translation. + respond func(filters *types.RecordFilters) ([]string, error) + + labels map[string][]types.Label + labelsErr error + + appliedFilters []*types.RecordFilters +} + +func (f *fakeQueryDB) GetRecordCIDs(opts ...types.FilterOption) ([]string, error) { + filters := &types.RecordFilters{} + for _, opt := range opts { + opt(filters) + } + + f.appliedFilters = append(f.appliedFilters, filters) + + return f.respond(filters) +} + +func (f *fakeQueryDB) GetRecordLabels(cids []string) (map[string][]types.Label, error) { + if f.labelsErr != nil { + return nil, f.labelsErr + } + + result := make(map[string][]types.Label, len(cids)) + + for _, cid := range cids { + if labels, ok := f.labels[cid]; ok { + result[cid] = labels + } + } + + return result, nil +} + +func newTestHost(t *testing.T) host.Host { + t.Helper() + + h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + require.NoError(t, err) + + t.Cleanup(func() { _ = h.Close() }) + + return h +} + +// newConnectedServices returns a client service and the peer ID of a server +// service backed by db, already dialled. +func newConnectedServices(t *testing.T, db types.DatabaseAPI) (*Service, peer.ID) { + t.Helper() + + serverHost := newTestHost(t) + clientHost := newTestHost(t) + + _, err := New(serverHost, nil, db) + require.NoError(t, err) + + clientService, err := New(clientHost, nil, nil) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + require.NoError(t, clientHost.Connect(ctx, peer.AddrInfo{ + ID: serverHost.ID(), + Addrs: serverHost.Addrs(), + })) + + return clientService, serverHost.ID() +} + +// collect runs a query against the peer and returns its matches. +func collect(t *testing.T, client *Service, peerID peer.ID, req QueryRecordsRequest) ([]RecordMatch, error) { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + + return client.QueryRecords(ctx, peerID, &req) +} + +// The method has to be accepted by gorpc's reflection-based registration and +// survive the msgpack round trip; only a real call across two hosts proves it. +func TestQueryRecordsReturnsMatchesAcrossPeers(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + return []string{"cid-a", "cid-b"}, nil + }, + labels: map[string][]types.Label{ + "cid-a": {"/skills/AI/ML", "/domains/finance"}, + "cid-b": {"/skills/AI/NLP"}, + }, + } + + client, serverID := newConnectedServices(t, db) + + matches, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, + }) + require.NoError(t, err) + + require.Len(t, matches, 2) + assert.Equal(t, "cid-a", matches[0].Cid) + assert.Equal(t, []string{"/skills/AI/ML", "/domains/finance"}, matches[0].Labels) + assert.Equal(t, "cid-b", matches[1].Cid) + assert.Equal(t, []string{"/skills/AI/NLP"}, matches[1].Labels) +} + +// A record matching several queries must be returned once. +func TestQueryRecordsUnionsQueriesWithoutDuplicates(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(filters *types.RecordFilters) ([]string, error) { + if len(filters.SkillNames) > 0 { + return []string{"shared", "skill-only"}, nil + } + + return []string{"shared", "domain-only"}, nil + }, + labels: map[string][]types.Label{ + "shared": {"/skills/AI", "/domains/finance"}, + "skill-only": {"/skills/AI"}, + "domain-only": {"/domains/finance"}, + }, + } + + client, serverID := newConnectedServices(t, db) + + matches, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{ + {Type: "skills", Value: "AI"}, + {Type: "domains", Value: "finance"}, + }, + }) + require.NoError(t, err) + + cids := make([]string, len(matches)) + for i, match := range matches { + cids[i] = match.Cid + } + + assert.Equal(t, []string{"shared", "skill-only", "domain-only"}, cids) +} + +func TestQueryRecordsCapsResultsAtLimit(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + return []string{"a", "b", "c", "d"}, nil + }, + labels: map[string][]types.Label{ + "a": {"/skills/AI"}, + "b": {"/skills/AI"}, + "c": {"/skills/AI"}, + "d": {"/skills/AI"}, + }, + } + + client, serverID := newConnectedServices(t, db) + + matches, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, + Limit: 2, + }) + require.NoError(t, err) + + assert.Len(t, matches, 2) + require.NotEmpty(t, db.appliedFilters) + assert.Equal(t, 2, db.appliedFilters[0].Limit, "the limit must reach the database, not just the result loop") +} + +// A caller needs to know its view of a peer is partial. +func TestQueryRecordsReportsTruncation(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + return []string{"a", "b", "c"}, nil + }, + labels: map[string][]types.Label{ + "a": {"/skills/AI"}, + "b": {"/skills/AI"}, + "c": {"/skills/AI"}, + }, + } + + client, serverID := newConnectedServices(t, db) + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + + var resp QueryRecordsResponse + + require.NoError(t, client.rpcClient.CallContext(ctx, serverID, DirService, DirServiceFuncQueryRecords, + &QueryRecordsRequest{Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, Limit: 2}, + &resp, + )) + + assert.True(t, resp.Truncated) + assert.Len(t, resp.Records, 2) +} + +// Records whose labels disappeared between the two queries carry nothing to +// score against, so they are dropped rather than streamed empty. +func TestQueryRecordsSkipsRecordsWithoutLabels(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + return []string{"present", "vanished"}, nil + }, + labels: map[string][]types.Label{ + "present": {"/skills/AI"}, + }, + } + + client, serverID := newConnectedServices(t, db) + + matches, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, + }) + require.NoError(t, err) + + require.Len(t, matches, 1) + assert.Equal(t, "present", matches[0].Cid) +} + +func TestQueryRecordsRejectsEmptyQueryList(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + t.Error("the database must not be consulted for an empty query list") + + return nil, nil + }, + } + + client, serverID := newConnectedServices(t, db) + + _, err := collect(t, client, serverID, QueryRecordsRequest{}) + require.Error(t, err) +} + +func TestQueryRecordsFailsWithoutDatabase(t *testing.T) { + t.Parallel() + + client, serverID := newConnectedServices(t, nil) + + _, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, + }) + require.Error(t, err) +} + +func TestQueryRecordsSurfacesDatabaseFailure(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + return nil, errors.New("database is down") + }, + } + + client, serverID := newConnectedServices(t, db) + + _, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, + }) + require.Error(t, err) +} + +// Unusable queries are skipped, not fatal, so one bad entry does not sink an +// otherwise valid multi-query search. +func TestQueryRecordsSkipsUnusableQueries(t *testing.T) { + t.Parallel() + + db := &fakeQueryDB{ + respond: func(*types.RecordFilters) ([]string, error) { + return []string{"cid-a"}, nil + }, + labels: map[string][]types.Label{"cid-a": {"/skills/AI"}}, + } + + client, serverID := newConnectedServices(t, db) + + matches, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{ + {Type: "nonsense", Value: "AI"}, + {Type: "skills", Value: " "}, + {Type: "skills", Value: "AI"}, + }, + }) + require.NoError(t, err) + + require.Len(t, matches, 1) + assert.Len(t, db.appliedFilters, 1, "only the usable query should reach the database") +} + +func TestQueryRecordsRejectsNilRequest(t *testing.T) { + t.Parallel() + + client, serverID := newConnectedServices(t, nil) + + _, err := client.QueryRecords(t.Context(), serverID, nil) + require.Error(t, err) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestQueryFiltersExpandsHierarchicalNamespaces(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query RecordQuery + expected func(*types.RecordFilters) []string + }{ + { + name: "skills match the value or any descendant", + query: RecordQuery{Type: "skills", Value: "AI"}, + expected: func(f *types.RecordFilters) []string { return f.SkillNames }, + }, + { + name: "domains match the value or any descendant", + query: RecordQuery{Type: "domains", Value: "AI"}, + expected: func(f *types.RecordFilters) []string { return f.DomainNames }, + }, + { + name: "modules match the value or any descendant", + query: RecordQuery{Type: "modules", Value: "AI"}, + expected: func(f *types.RecordFilters) []string { return f.ModuleNames }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + options, err := queryFilters(tt.query, 10) + require.NoError(t, err) + + filters := &types.RecordFilters{} + for _, option := range options { + option(filters) + } + + assert.Equal(t, []string{"AI", "AI/*"}, tt.expected(filters)) + assert.Equal(t, 10, filters.Limit) + }) + } +} + +// Locators are flat, so a descendant pattern would be meaningless. +func TestQueryFiltersMatchesLocatorsExactly(t *testing.T) { + t.Parallel() + + options, err := queryFilters(RecordQuery{Type: "locators", Value: "docker-image"}, 10) + require.NoError(t, err) + + filters := &types.RecordFilters{} + for _, option := range options { + option(filters) + } + + assert.Equal(t, []string{"docker-image"}, filters.LocatorTypes) + assert.Empty(t, filters.SkillNames) +} + +func TestQueryFiltersRejectsBadQueries(t *testing.T) { + t.Parallel() + + for _, query := range []RecordQuery{ + {Type: "skills", Value: ""}, + {Type: "skills", Value: " "}, + {Type: "", Value: "AI"}, + {Type: "unknown", Value: "AI"}, + } { + _, err := queryFilters(query, 10) + require.Error(t, err, "expected %+v to be rejected", query) + } +} + +func TestGetDatabaseReportsUnimplementedWhenAbsent(t *testing.T) { + t.Parallel() + + _, err := (&Service{}).getDatabase() + require.Error(t, err) + assert.Equal(t, codes.Unimplemented, status.Code(err)) +} diff --git a/server/routing/rpc/rpc.go b/server/routing/rpc/rpc.go index 86484e2b3..289630a88 100644 --- a/server/routing/rpc/rpc.go +++ b/server/routing/rpc/rpc.go @@ -29,9 +29,8 @@ var ( // TODO: proper cleanup and implementation needed! const ( - Protocol = protocol.ID("/dir/rpc/1.0.0") + Protocol = protocol.ID("/dir/rpc/2.0.0") DirService = "RPCAPI" - DirServiceFuncLookup = "Lookup" DirServiceFuncPull = "Pull" DirServiceFuncListReferrers = "ListReferrers" DirServiceFuncPullReferrer = "PullReferrer" @@ -296,9 +295,10 @@ type Service struct { host host.Host store types.StoreAPI refStore types.ReferrerStoreAPI + db types.DatabaseAPI } -func New(host host.Host, store types.StoreAPI) (*Service, error) { +func New(host host.Host, store types.StoreAPI, db types.DatabaseAPI) (*Service, error) { var refStore types.ReferrerStoreAPI if rs, ok := store.(types.ReferrerStoreAPI); ok { refStore = rs @@ -309,6 +309,7 @@ func New(host host.Host, store types.StoreAPI) (*Service, error) { host: host, store: store, refStore: refStore, + db: db, } // register api @@ -325,21 +326,6 @@ func New(host host.Host, store types.StoreAPI) (*Service, error) { return service, nil } -func (s *Service) Lookup(ctx context.Context, peer peer.ID, req *corev1.RecordRef) (*corev1.RecordRef, error) { - logger.Debug("P2p RPC: Executing Lookup request on remote peer", "peer", peer, "req", req) - - var resp LookupResponse - - err := s.rpcClient.CallContext(ctx, peer, DirService, DirServiceFuncLookup, req, &resp) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to call remote peer: %v", err) - } - - return &corev1.RecordRef{ - Cid: resp.Cid, - }, nil -} - func (s *Service) Pull(ctx context.Context, peer peer.ID, req *corev1.RecordRef) (*corev1.Record, error) { logger.Debug("P2p RPC: Executing Pull request on remote peer", "peer", peer, "req", req) @@ -402,6 +388,14 @@ func (s *Service) PullReferrer( return resp.Referrer, nil } +func (s *Service) getDatabase() (types.DatabaseAPI, error) { + if s.db == nil { + return nil, status.Error(codes.Unimplemented, "record queries are not supported by this node") //nolint:wrapcheck + } + + return s.db, nil +} + func (s *Service) getReferrerStore() (types.ReferrerStoreAPI, error) { if s.refStore == nil { return nil, status.Error(codes.Unimplemented, "referrer storage is not supported by the current store implementation") //nolint:wrapcheck diff --git a/server/routing/search_remote.go b/server/routing/search_remote.go new file mode 100644 index 000000000..b4439c8b9 --- /dev/null +++ b/server/routing/search_remote.go @@ -0,0 +1,366 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "context" + "strings" + "sync" + + corev1 "github.com/agntcy/dir/api/core/v1" + routingv1 "github.com/agntcy/dir/api/routing/v1" + "github.com/agntcy/dir/server/routing/internal/p2p" + "github.com/agntcy/dir/server/routing/rpc" + "github.com/agntcy/dir/server/types" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" +) + +// searchRemoteRecords finds records held by other peers and streams the matches. +// +// Three stages, overlapping rather than sequential: resolve one query label to +// a DHT key and ask who provides it, ask each of those peers which of their +// records match the full query set, then score the answers. Nothing consults a +// local cache of remote announcements β€” peers answer for themselves, so a peer +// that is down fails at discovery instead of at pull time. +// +// Results are best-effort. The lookup unions the views of whichever custodians +// it reaches inside the budget, and each budget expiring costs recall, not +// correctness. +func (r *routeRemote) searchRemoteRecords( + ctx context.Context, + queries []*routingv1.RecordQuery, + limit uint32, + minMatchScore uint32, + outCh chan<- *routingv1.SearchResponse, +) { + key, label, ok := discoveryKey(queries) + if !ok { + remoteLogger.Warn("Remote search needs a skill, domain, module or locator query to look up", + "queries", len(queries)) + + return + } + + searchCtx, cancel := context.WithTimeout(ctx, SearchTimeout) + defer cancel() + + request := &rpc.QueryRecordsRequest{ + Queries: peerQueries(queries), + Limit: peerLimit(limit, minMatchScore), + } + + remoteLogger.Debug("Starting remote search", "label", label, "key", key, "queries", len(queries), + "minMatchScore", minMatchScore, "limit", limit) + + emitted := make(map[string]struct{}) + + for answer := range r.queryProviders(searchCtx, key, request) { + for _, match := range answer.matches { + if _, done := emitted[match.Cid]; done { + continue + } + + matched, score := scoreMatch(queries, toLabels(match.Labels)) + if score < minMatchScore { + remoteLogger.Debug("Discarding record below the match threshold", + "cid", match.Cid, "peer", answer.provider.ID, "score", score) + + continue + } + + select { + case outCh <- &routingv1.SearchResponse{ + RecordRef: &corev1.RecordRef{Cid: match.Cid}, + Peer: r.peerInfo(answer.provider), + MatchQueries: matched, + MatchScore: score, + }: + case <-searchCtx.Done(): + return + } + + emitted[match.Cid] = struct{}{} + + // Returning cancels searchCtx, which unwinds the lookup and the + // workers still waiting to report. + if limit > 0 && safeIntToUint32(len(emitted)) >= limit { + remoteLogger.Debug("Remote search reached the requested limit", "limit", limit) + + return + } + } + } + + remoteLogger.Debug("Completed remote search", "label", label, "results", len(emitted)) +} + +// peerAnswer is one peer's reply to the record query. +type peerAnswer struct { + provider peer.AddrInfo + matches []rpc.RecordMatch +} + +// queryProviders asks every peer that provides key which of its records match. +// +// The returned channel closes once every discovered peer has answered or the +// context is done. +func (r *routeRemote) queryProviders(ctx context.Context, key cid.Cid, request *rpc.QueryRecordsRequest) <-chan peerAnswer { + answers := make(chan peerAnswer) + providers := make(chan peer.AddrInfo, searchProviderBuffer) + + go r.discoverProviders(ctx, key, providers) + + var wg sync.WaitGroup + + for range searchPeerWorkers { + wg.Go(func() { + for provider := range providers { + matches := r.queryPeer(ctx, provider, request) + if len(matches) == 0 { + continue + } + + select { + case answers <- peerAnswer{provider: provider, matches: matches}: + case <-ctx.Done(): + return + } + } + }) + } + + go func() { + wg.Wait() + close(answers) + }() + + return answers +} + +// discoverProviders drains the DHT provider stream straight into providers. +// +// Nothing slow may happen in this loop. The lookup writes to its channel from +// inside the Kademlia query and the package documents that not reading from it +// blocks the query from progressing, so dialling a peer here would throttle +// discovery itself. +func (r *routeRemote) discoverProviders(ctx context.Context, key cid.Cid, providers chan<- peer.AddrInfo) { + defer close(providers) + + discoveryCtx, cancel := context.WithTimeout(ctx, SearchDiscoveryTimeout) + defer cancel() + + self := r.server.Host().ID() + seen := make(map[peer.ID]struct{}) + + // count=0 asks for every provider. Any other value both caps the result and + // lets the local provider store satisfy the request without touching the + // network, which would make results depend on what this node has cached. + for provider := range r.server.DHT().FindProvidersAsync(discoveryCtx, key, 0) { + // We advertise the labels of the records we hold, so we are a provider + // of our own results. Search is defined as remote-only; List covers + // what this node holds. + if provider.ID == self { + continue + } + + // The lookup re-emits a peer whose first sighting carried no addresses, + // and we would otherwise query it twice. + if _, ok := seen[provider.ID]; ok { + continue + } + + seen[provider.ID] = struct{}{} + + select { + case providers <- provider: + case <-ctx.Done(): + return + } + } + + remoteLogger.Debug("Provider discovery finished", "key", key, "providers", len(seen)) +} + +// queryPeer asks one peer which of its records match, returning nothing if it +// cannot answer. A provider record outlives the peer that wrote it, so an +// unreachable peer is expected rather than exceptional. +func (r *routeRemote) queryPeer(ctx context.Context, provider peer.AddrInfo, request *rpc.QueryRecordsRequest) []rpc.RecordMatch { + peerCtx, cancel := context.WithTimeout(ctx, SearchPeerTimeout) + defer cancel() + + matches, err := r.service.QueryRecords(peerCtx, provider.ID, request) + if err != nil { + remoteLogger.Debug("Provider did not answer the record query", "peer", provider.ID, "error", err) + + return nil + } + + return matches +} + +// discoveryKey picks the label to look up and returns its DHT key. +// +// One label is enough. A record satisfying an AND query carries every label the +// query names, and a peer advertises every label of every record it holds, so +// the holder is a provider of each of them; looking up one finds it. Depth is a +// cheap proxy for selectivity β€” far fewer peers provide /skills/AI/ML than +// /skills/AI, so the deeper key gives a smaller and more accurate candidate set. +func discoveryKey(queries []*routingv1.RecordQuery) (cid.Cid, types.Label, bool) { + var ( + selected types.Label + depth int + ) + + for _, query := range queries { + label, ok := queryLabel(query) + if !ok { + continue + } + + if labelDepth := strings.Count(label.Value(), "/"); selected == "" || labelDepth > depth { + selected, depth = label, labelDepth + } + } + + if selected == "" { + return cid.Undef, "", false + } + + key, err := labelKey(selected) + if err != nil { + remoteLogger.Warn("Cannot derive a DHT key from the search label", "label", selected, "error", err) + + return cid.Undef, "", false + } + + return key, selected, true +} + +// peerQueries converts the request into its wire form, dropping queries that +// name no label. An unspecified query matches everything, so it contributes to +// the score without narrowing what a peer should return. +func peerQueries(queries []*routingv1.RecordQuery) []rpc.RecordQuery { + converted := make([]rpc.RecordQuery, 0, len(queries)) + + for _, query := range queries { + label, ok := queryLabel(query) + if !ok { + continue + } + + converted = append(converted, rpc.RecordQuery{ + Type: label.Type().String(), + Value: label.Value(), + }) + } + + return converted +} + +// peerLimit decides how many records to ask each peer for. +// +// Normally the caller's limit: every record a peer returns matched at least one +// query, which already clears the default threshold, so none of them are wasted. +// A higher threshold is scored here and not there, so the peer has to offer more +// candidates than the caller will keep; zero lets it apply its own cap. +func peerLimit(limit uint32, minMatchScore uint32) uint32 { + if minMatchScore > DefaultMinMatchScore { + return 0 + } + + return limit +} + +// queryLabel maps a query onto the label it searches for. +func queryLabel(query *routingv1.RecordQuery) (types.Label, bool) { + value := strings.TrimSpace(query.GetValue()) + if value == "" { + return "", false + } + + labelType, ok := queryLabelType(query.GetType()) + if !ok { + return "", false + } + + return labelType.LabelKey(value), true +} + +func queryLabelType(queryType routingv1.RecordQueryType) (types.LabelType, bool) { + switch queryType { + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL: + return types.LabelTypeSkill, true + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN: + return types.LabelTypeDomain, true + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE: + return types.LabelTypeModule, true + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR: + return types.LabelTypeLocator, true + case routingv1.RecordQueryType_RECORD_QUERY_TYPE_UNSPECIFIED: + return types.LabelTypeUnknown, false + default: + return types.LabelTypeUnknown, false + } +} + +// scoreMatch counts how many queries the record's labels satisfy. Queries are +// OR'd: the count is the score the caller thresholds on. +func scoreMatch(queries []*routingv1.RecordQuery, labels []types.Label) ([]*routingv1.RecordQuery, uint32) { + if len(queries) == 0 || len(labels) == 0 { + return nil, 0 + } + + matched := make([]*routingv1.RecordQuery, 0, len(queries)) + + for _, query := range queries { + if QueryMatchesLabels(query, labels) { + matched = append(matched, query) + } + } + + return matched, safeIntToUint32(len(matched)) +} + +func toLabels(values []string) []types.Label { + labels := make([]types.Label, len(values)) + for i, value := range values { + labels[i] = types.Label(value) + } + + return labels +} + +// peerInfo describes where a provider can be reached, advertising its Directory +// API (/dir/) and OCI registry (/oci/) endpoints in prefixed multiaddr form so +// the consumer can tell them apart. Either may be missing. +// +// Addresses come from the provider record, falling back to the peerstore for a +// record that arrived without any β€” a peer we are already connected to has told +// us its addresses over identify. +func (r *routeRemote) peerInfo(provider peer.AddrInfo) *routingv1.Peer { + known := provider.Addrs + if len(known) == 0 { + known = r.server.Host().Peerstore().Addrs(provider.ID) + } + + addrs := make([]string, 0, 2) //nolint:mnd // dir + oci + + for _, protocol := range []struct { + name string + code int + }{ + {p2p.DirProtocol, p2p.DirProtocolCode}, + {p2p.OciProtocol, p2p.OciProtocolCode}, + } { + if value := extractProtocolValue(known, protocol.code); value != "" { + addrs = append(addrs, "/"+protocol.name+"/"+value) + } + } + + return &routingv1.Peer{ + Id: provider.ID.String(), + Addrs: addrs, + } +} diff --git a/server/routing/search_remote_network_test.go b/server/routing/search_remote_network_test.go new file mode 100644 index 000000000..e44273c33 --- /dev/null +++ b/server/routing/search_remote_network_test.go @@ -0,0 +1,210 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "context" + "slices" + "testing" + "time" + + routingv1 "github.com/agntcy/dir/api/routing/v1" + "github.com/agntcy/dir/server/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// heldRecordsDB is what a node answers peer queries with. Which records match +// which filters is settled in the RPC package's own tests; here the point is +// that the answer reaches the searcher and gets scored. +type heldRecordsDB struct { + types.DatabaseAPI + + labels map[string][]types.Label +} + +func (h *heldRecordsDB) GetRecordCIDs(...types.FilterOption) ([]string, error) { + cids := make([]string, 0, len(h.labels)) + for cid := range h.labels { + cids = append(cids, cid) + } + + slices.Sort(cids) + + return cids, nil +} + +func (h *heldRecordsDB) GetRecordLabels(cids []string) (map[string][]types.Label, error) { + labels := make(map[string][]types.Label, len(cids)) + + for _, cid := range cids { + if recordLabels, ok := h.labels[cid]; ok { + labels[cid] = recordLabels + } + } + + return labels, nil +} + +func (h *heldRecordsDB) advertised() []types.Label { + var labels []types.Label + for _, recordLabels := range h.labels { + labels = append(labels, recordLabels...) + } + + return expandLabels(labels) +} + +// newSearchNetwork starts a node holding the given records and a second node +// bootstrapped off it, advertises the holder's labels, and returns both β€” the +// holder first. +func newSearchNetwork(t *testing.T, held *heldRecordsDB) (*route, *route) { + t.Helper() + + ctx := t.Context() + + holder := newTestServer(t, ctx, nil, held) + searcher := newTestServer(t, ctx, holder.remote.server.P2pAddrs(), nil) + + <-holder.remote.server.DHT().RefreshRoutingTable() + <-searcher.remote.server.DHT().RefreshRoutingTable() + time.Sleep(1 * time.Second) + + provideCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + require.Zero(t, holder.remote.provideLabels(provideCtx, held.advertised()), + "every label should reach the DHT in a two-node network") + + return holder, searcher +} + +func collectSearch(t *testing.T, node *route, req *routingv1.SearchRequest) []*routingv1.SearchResponse { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), SearchTimeout+10*time.Second) + defer cancel() + + responses, err := node.Search(ctx, req) + require.NoError(t, err) + + var collected []*routingv1.SearchResponse + for response := range responses { + collected = append(collected, response) + } + + return collected +} + +func responseCIDs(responses []*routingv1.SearchResponse) []string { + cids := make([]string, 0, len(responses)) + for _, response := range responses { + cids = append(cids, response.GetRecordRef().GetCid()) + } + + slices.Sort(cids) + + return cids +} + +func TestRemoteSearchOverTheNetwork(t *testing.T) { + held := &heldRecordsDB{labels: map[string][]types.Label{ + "record-ml": {"/skills/AI/ML", "/domains/healthcare"}, + "record-nlp": {"/skills/AI/NLP"}, + }} + + holder, searcher := newSearchNetwork(t, held) + + t.Run("a parent skill reaches records tagged with its descendants", func(t *testing.T) { + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{skillQuery("AI")}, + }) + + assert.Equal(t, []string{"record-ml", "record-nlp"}, responseCIDs(responses)) + + for _, response := range responses { + assert.Equal(t, holder.remote.server.Host().ID().String(), response.GetPeer().GetId()) + assert.Equal(t, uint32(1), response.GetMatchScore()) + } + }) + + t.Run("a record matching more queries scores higher", func(t *testing.T) { + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{ + skillQuery("AI"), + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, Value: "healthcare"}, + }, + }) + + scores := make(map[string]uint32, len(responses)) + for _, response := range responses { + scores[response.GetRecordRef().GetCid()] = response.GetMatchScore() + } + + assert.Equal(t, map[string]uint32{"record-ml": 2, "record-nlp": 1}, scores) + }) + + t.Run("the threshold drops records that match too few queries", func(t *testing.T) { + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{ + skillQuery("AI"), + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, Value: "healthcare"}, + }, + MinMatchScore: new(uint32(2)), + }) + + assert.Equal(t, []string{"record-ml"}, responseCIDs(responses)) + }) + + t.Run("the limit caps the results", func(t *testing.T) { + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{skillQuery("AI")}, + Limit: new(uint32(1)), + }) + + assert.Len(t, responses, 1) + }) + + t.Run("an unadvertised label finds nothing", func(t *testing.T) { + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{skillQuery("Robotics")}, + }) + + assert.Empty(t, responses) + }) + + t.Run("a node does not return its own records", func(t *testing.T) { + // The holder provides every one of these labels, so without the + // self-check it would query itself and return everything it holds. + responses := collectSearch(t, holder, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{skillQuery("AI")}, + }) + + assert.Empty(t, responses) + }) +} + +func TestRemoteSearchWhenTheProviderCannotAnswer(t *testing.T) { + // A node with no database rejects the query. The searcher has to finish + // cleanly rather than hang on a provider that discovery did find. + ctx := t.Context() + + holder := newTestServer(t, ctx, nil, nil) + searcher := newTestServer(t, ctx, holder.remote.server.P2pAddrs(), nil) + + <-holder.remote.server.DHT().RefreshRoutingTable() + <-searcher.remote.server.DHT().RefreshRoutingTable() + time.Sleep(1 * time.Second) + + provideCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + require.Zero(t, holder.remote.provideLabels(provideCtx, expandLabels([]types.Label{"/skills/AI/ML"}))) + + responses := collectSearch(t, searcher, &routingv1.SearchRequest{ + Queries: []*routingv1.RecordQuery{skillQuery("AI")}, + }) + + assert.Empty(t, responses) +} diff --git a/server/routing/search_remote_test.go b/server/routing/search_remote_test.go new file mode 100644 index 000000000..61373f33f --- /dev/null +++ b/server/routing/search_remote_test.go @@ -0,0 +1,174 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package routing + +import ( + "testing" + + routingv1 "github.com/agntcy/dir/api/routing/v1" + "github.com/agntcy/dir/server/routing/rpc" + "github.com/agntcy/dir/server/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func skillQuery(value string) *routingv1.RecordQuery { + return &routingv1.RecordQuery{Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, Value: value} +} + +func TestScoreMatch(t *testing.T) { + labels := []types.Label{ + types.Label("/skills/Natural Language Processing/Text Completion"), + types.Label("/skills/Natural Language Processing/Problem Solving"), + types.Label("/domains/healthcare"), + } + + tests := []struct { + name string + queries []*routingv1.RecordQuery + score uint32 + }{ + { + name: "counts only the queries that match", + queries: []*routingv1.RecordQuery{skillQuery("Natural Language Processing/Text Completion"), skillQuery("Natural Language Processing/Problem Solving"), skillQuery("Nonexistent")}, + score: 2, + }, + { + name: "single match", + queries: []*routingv1.RecordQuery{skillQuery("Natural Language Processing/Text Completion")}, + score: 1, + }, + { + name: "a parent skill matches its descendants, but counts once per query", + queries: []*routingv1.RecordQuery{skillQuery("Natural Language Processing")}, + score: 1, + }, + { + name: "queries of different kinds both count", + queries: []*routingv1.RecordQuery{ + skillQuery("Natural Language Processing"), + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, Value: "healthcare"}, + }, + score: 2, + }, + { + name: "no match scores zero", + queries: []*routingv1.RecordQuery{skillQuery("Nonexistent")}, + score: 0, + }, + { + name: "no queries scores zero", + queries: nil, + score: 0, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matched, score := scoreMatch(test.queries, labels) + + assert.Equal(t, test.score, score) + assert.Len(t, matched, int(test.score)) + }) + } +} + +func TestScoreMatchWithoutLabels(t *testing.T) { + matched, score := scoreMatch([]*routingv1.RecordQuery{skillQuery("AI")}, nil) + + assert.Empty(t, matched) + assert.Equal(t, uint32(0), score) +} + +func TestDiscoveryKeyPicksTheDeepestLabel(t *testing.T) { + queries := []*routingv1.RecordQuery{ + skillQuery("AI"), + skillQuery("AI/ML/Deep Learning"), + skillQuery("AI/ML"), + } + + key, label, ok := discoveryKey(queries) + require.True(t, ok) + assert.Equal(t, types.Label("/skills/AI/ML/Deep Learning"), label) + + expected, err := labelKey(types.Label("/skills/AI/ML/Deep Learning")) + require.NoError(t, err) + assert.Equal(t, expected, key) +} + +func TestDiscoveryKeyMatchesTheKeyThePublisherAdvertises(t *testing.T) { + // A record tagged /skills/AI/ML is only findable under /skills/AI because + // its holder advertises that ancestor too, so the searcher's key for "AI" + // has to be the same one expandLabel produces. + advertised := expandLabel(types.Label("/skills/AI/ML")) + require.Contains(t, advertised, types.Label("/skills/AI")) + + published, err := labelKey(types.Label("/skills/AI")) + require.NoError(t, err) + + searched, _, ok := discoveryKey([]*routingv1.RecordQuery{skillQuery("AI")}) + require.True(t, ok) + + assert.Equal(t, published, searched) +} + +func TestDiscoveryKeyRejectsQueriesWithNoLabel(t *testing.T) { + tests := []struct { + name string + queries []*routingv1.RecordQuery + }{ + {name: "no queries", queries: nil}, + {name: "empty value", queries: []*routingv1.RecordQuery{skillQuery(" ")}}, + { + name: "unspecified type matches everything and names nothing", + queries: []*routingv1.RecordQuery{{Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_UNSPECIFIED, Value: "AI"}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok := discoveryKey(test.queries) + assert.False(t, ok) + }) + } +} + +func TestDiscoveryKeySkipsUnusableQueries(t *testing.T) { + queries := []*routingv1.RecordQuery{ + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_UNSPECIFIED, Value: "anything"}, + skillQuery("AI"), + } + + _, label, ok := discoveryKey(queries) + require.True(t, ok) + assert.Equal(t, types.Label("/skills/AI"), label) +} + +func TestPeerQueries(t *testing.T) { + queries := []*routingv1.RecordQuery{ + skillQuery("AI/ML"), + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_DOMAIN, Value: "healthcare"}, + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_MODULE, Value: "runtime/model"}, + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_LOCATOR, Value: "docker-image"}, + {Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_UNSPECIFIED, Value: "dropped"}, + skillQuery(""), + } + + assert.Equal(t, []rpc.RecordQuery{ + {Type: "skills", Value: "AI/ML"}, + {Type: "domains", Value: "healthcare"}, + {Type: "modules", Value: "runtime/model"}, + {Type: "locators", Value: "docker-image"}, + }, peerQueries(queries)) +} + +func TestPeerLimit(t *testing.T) { + // Every record a peer returns matched at least one query, so at the default + // threshold the caller keeps all of them and can ask for exactly its limit. + assert.Equal(t, uint32(10), peerLimit(10, DefaultMinMatchScore)) + + // A higher threshold is applied by the caller, so the peer has to offer more + // candidates than the caller will keep. + assert.Equal(t, uint32(0), peerLimit(10, 2)) +} diff --git a/server/routing/search_simple_test.go b/server/routing/search_simple_test.go deleted file mode 100644 index 25a332b61..000000000 --- a/server/routing/search_simple_test.go +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package routing - -import ( - "context" - "encoding/json" - "os" - "testing" - "time" - - corev1 "github.com/agntcy/dir/api/core/v1" - routingv1 "github.com/agntcy/dir/api/routing/v1" - "github.com/agntcy/dir/server/datastore" - "github.com/agntcy/dir/server/types" - ipfsdatastore "github.com/ipfs/go-datastore" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Test the core Search functionality using a simplified approach. -func TestSearch_CoreLogic(t *testing.T) { - ctx := t.Context() - - // Create test datastore - dstore, cleanup := setupSearchTestDatastore(t) - defer cleanup() - - // Setup test data - simulate remote announcements from different peers - testData := []struct { - cid string - peerID string - labels []string - }{ - { - cid: "ai-record-1", - peerID: "remote-peer-1", - labels: []string{"/skills/AI", "/skills/AI/ML"}, - }, - { - cid: "ai-record-2", - peerID: "remote-peer-2", - labels: []string{"/skills/AI/NLP"}, - }, - { - cid: "web-record", - peerID: "remote-peer-3", - labels: []string{"/skills/web-development", "/skills/javascript"}, - }, - { - cid: "local-record", - peerID: testLocalPeerID, // This should be filtered out - labels: []string{"/skills/AI"}, - }, - } - - // Store test label metadata - for _, td := range testData { - for _, label := range td.labels { - enhancedKey := BuildEnhancedLabelKey(types.Label(label), td.cid, td.peerID) - metadata := &types.LabelMetadata{ - Timestamp: time.Now(), - LastSeen: time.Now(), - } - metadataBytes, err := json.Marshal(metadata) - require.NoError(t, err) - - err = dstore.Put(ctx, ipfsdatastore.NewKey(enhancedKey), metadataBytes) - require.NoError(t, err) - } - } - - t.Run("search_filters_remote_records_only", func(t *testing.T) { - // Test that we can find remote records and filter out local ones - localPeerID := testLocalPeerID - - // Simulate searching for AI skills - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - } - - // Use our simplified search logic - results := simulateSearch(ctx, dstore, localPeerID, queries, 10, 1) - - // Should return 2 remote records (ai-record-1, ai-record-2) but not local-record - assert.Len(t, results, 2) - - expectedCIDs := []string{"ai-record-1", "ai-record-2"} - - foundCIDs := make(map[string]bool) - for _, result := range results { - foundCIDs[result.GetRecordRef().GetCid()] = true - - // Verify it's not from local peer - assert.NotEqual(t, localPeerID, result.GetPeer().GetId()) - - // Verify structure - assert.NotNil(t, result.GetRecordRef()) - assert.NotNil(t, result.GetPeer()) - assert.Positive(t, result.GetMatchScore()) - } - - for _, expectedCID := range expectedCIDs { - assert.True(t, foundCIDs[expectedCID], "Expected CID %s not found", expectedCID) - } - }) - - t.Run("search_with_and_logic", func(t *testing.T) { - // Test AND logic with multiple queries - localPeerID := testLocalPeerID - - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI/ML", - }, - } - - results := simulateSearch(ctx, dstore, localPeerID, queries, 10, 2) - - // Only ai-record-1 should match both AI and AI/ML - assert.Len(t, results, 1) - assert.Equal(t, "ai-record-1", results[0].GetRecordRef().GetCid()) - assert.Equal(t, "remote-peer-1", results[0].GetPeer().GetId()) - assert.Equal(t, uint32(2), results[0].GetMatchScore()) - }) - - t.Run("search_with_limit", func(t *testing.T) { - // Test result limiting - localPeerID := testLocalPeerID - - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - } - - results := simulateSearch(ctx, dstore, localPeerID, queries, 1, 1) // Limit to 1 - - assert.Len(t, results, 1) - assert.NotEqual(t, localPeerID, results[0].GetPeer().GetId()) - }) - - t.Run("search_with_high_min_score", func(t *testing.T) { - // Test minimum match score filtering - localPeerID := testLocalPeerID - - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - } - - results := simulateSearch(ctx, dstore, localPeerID, queries, 10, 5) // Very high score - - assert.Empty(t, results) // No results should meet the high score requirement - }) - - t.Run("search_no_queries_returns_all_remote", func(t *testing.T) { - // Test that no queries returns all remote records - localPeerID := testLocalPeerID - - results := simulateSearch(ctx, dstore, localPeerID, []*routingv1.RecordQuery{}, 10, 0) - - // Should return 3 remote records (excluding local-peer) - assert.Len(t, results, 3) - - for _, result := range results { - assert.NotEqual(t, localPeerID, result.GetPeer().GetId()) - } - }) - - t.Run("search_with_different_local_peer_id", func(t *testing.T) { - // Test with a different localPeerID to validate the filtering logic - differentLocalPeer := "remote-peer-1" // This peer has records in our test data - - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - } - - results := simulateSearch(ctx, dstore, differentLocalPeer, queries, 10, 1) - - // Should return different results since "remote-peer-1" is now considered "local" - // and should be filtered out. Should find records from other peers: remote-peer-2, remote-peer-3, local-peer - assert.Len(t, results, 2) // ai-record-2 (remote-peer-2) + local-record (local-peer) - - foundPeers := make(map[string]bool) - - for _, result := range results { - assert.NotEqual(t, differentLocalPeer, result.GetPeer().GetId()) - - foundPeers[result.GetPeer().GetId()] = true - } - - // Should contain records from peers other than "remote-peer-1" - expectedPeers := []string{"remote-peer-2", testLocalPeerID} - for _, expectedPeer := range expectedPeers { - assert.True(t, foundPeers[expectedPeer], "Should find record from peer %s", expectedPeer) - } - }) - - t.Run("search_validates_peer_filtering_logic", func(t *testing.T) { - // Test that changing localPeerID actually changes which records are filtered - queries := []*routingv1.RecordQuery{ - { - Type: routingv1.RecordQueryType_RECORD_QUERY_TYPE_SKILL, - Value: "AI", - }, - } - - // Search with testLocalPeerID as local - resultsA := simulateSearch(ctx, dstore, testLocalPeerID, queries, 10, 1) - - // Search with "remote-peer-1" as local - resultsB := simulateSearch(ctx, dstore, "remote-peer-1", queries, 10, 1) - - // Results may have same count but should contain different peers - // resultsA filters out testLocalPeerID, resultsB filters out "remote-peer-1" - // Both should return 2 results, but from different peer combinations - - // Collect all peer IDs from each result set - peersA := make(map[string]bool) - for _, result := range resultsA { - peersA[result.GetPeer().GetId()] = true - } - - peersB := make(map[string]bool) - for _, result := range resultsB { - peersB[result.GetPeer().GetId()] = true - } - - // The peer sets should be different (different peers filtered out) - assert.NotEqual(t, peersA, peersB, "Different localPeerID should result in different peer sets") - }) -} - -// Simplified search simulation for testing. -// -//nolint:gocognit // Test helper function that replicates search logic - complexity is necessary -func simulateSearch(ctx context.Context, dstore types.Datastore, localPeerID string, queries []*routingv1.RecordQuery, limit uint32, minMatchScore uint32) []*routingv1.SearchResponse { - var results []*routingv1.SearchResponse - - processedCIDs := make(map[string]bool) - processedCount := 0 - limitInt := int(limit) - - // Query all namespaces using shared function - entries, err := QueryAllNamespaces(ctx, dstore) - if err != nil { - return results - } - - for _, entry := range entries { - if limitInt > 0 && processedCount >= limitInt { - break - } - - // Parse enhanced key - _, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - continue - } - - // Filter for REMOTE records only - if keyPeerID == localPeerID { - continue - } - - // Avoid duplicates - if processedCIDs[keyCID] { - continue - } - - // Check if matches all queries - if testMatchesAllQueriesSimple(ctx, dstore, keyCID, queries, keyPeerID) { - // Calculate score safely - score := safeIntToUint32(len(queries)) - if len(queries) == 0 { - score = 1 - } - - if score >= minMatchScore { - results = append(results, &routingv1.SearchResponse{ - RecordRef: &corev1.RecordRef{Cid: keyCID}, - Peer: &routingv1.Peer{Id: keyPeerID}, - MatchQueries: queries, - MatchScore: score, - }) - - processedCIDs[keyCID] = true - processedCount++ - - if limitInt > 0 && processedCount >= limitInt { - break - } - } - } - } - - return results -} - -// Simplified query matching for testing. -func testMatchesAllQueriesSimple(ctx context.Context, dstore types.Datastore, cid string, queries []*routingv1.RecordQuery, peerID string) bool { - if len(queries) == 0 { - return true - } - - // Get labels for this CID/PeerID using shared namespace iteration - entries, err := QueryAllNamespaces(ctx, dstore) - if err != nil { - return false - } - - var labelStrings []string - - for _, entry := range entries { - label, keyCID, keyPeerID, err := ParseEnhancedLabelKey(entry.Key) - if err != nil { - continue - } - - if keyCID == cid && keyPeerID == peerID { - labelStrings = append(labelStrings, label.String()) - } - } - - // Use shared query matching logic - convert strings to labels - labelRetriever := func(_ context.Context, _ string) []types.Label { - labelList := make([]types.Label, len(labelStrings)) - for i, labelStr := range labelStrings { - labelList[i] = types.Label(labelStr) - } - - return labelList - } - - return MatchesAllQueries(ctx, cid, queries, labelRetriever) -} - -// Helper functions for testing - -// setupSearchTestDatastore creates a temporary datastore for search testing. -func setupSearchTestDatastore(t *testing.T) (types.Datastore, func()) { - t.Helper() - - dsOpts := []datastore.Option{ - datastore.WithFsProvider("/tmp/test-search-" + t.Name()), - } - - dstore, err := datastore.New(dsOpts...) - require.NoError(t, err) - - cleanup := func() { - _ = dstore.Close() - _ = os.RemoveAll("/tmp/test-search-" + t.Name()) - } - - return dstore, cleanup -} diff --git a/server/routing/test_utils.go b/server/routing/test_utils.go index 543606f54..dd8c9a5f4 100644 --- a/server/routing/test_utils.go +++ b/server/routing/test_utils.go @@ -6,22 +6,42 @@ package routing import ( "context" + "path/filepath" "testing" "time" "github.com/agntcy/dir/server/config" + "github.com/agntcy/dir/server/database" + dbconfig "github.com/agntcy/dir/server/database/config" routingconfig "github.com/agntcy/dir/server/routing/config" "github.com/agntcy/dir/server/store" storeconfig "github.com/agntcy/dir/server/store/config" ociconfig "github.com/agntcy/dir/server/store/oci/config" "github.com/agntcy/dir/server/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -const testLocalPeerID = "local-peer" +// newTestDatabase returns a real SQLite-backed index. Local list is mostly +// filter translation, so it is worth testing against the query engine that +// runs in production rather than a hand-written matcher. +func newTestDatabase(t *testing.T) types.DatabaseAPI { + t.Helper() + + db, err := database.New(dbconfig.Config{ + Type: string(database.SQLite), + SQLite: dbconfig.SQLiteConfig{Path: filepath.Join(t.TempDir(), "test.db")}, + }) + require.NoError(t, err) + + return db +} +// newTestServer starts a routing node. Pass a database only when the test +// exercises the peer query RPC; a node without one answers Unimplemented. +// //nolint:revive -func newTestServer(t *testing.T, ctx context.Context, bootPeers []string) *route { +func newTestServer(t *testing.T, ctx context.Context, bootPeers []string, db types.DatabaseAPI) *route { t.Helper() refreshInterval := 1 * time.Second @@ -49,9 +69,7 @@ func newTestServer(t *testing.T, ctx context.Context, bootPeers []string) *route s, err := store.New(opts) assert.NoError(t, err) - // create example server - // Autosync is disabled in these tests, so no ingestion service or validator is required. - r, err := New(ctx, s, nil, nil, opts) + r, err := New(ctx, s, db, opts) assert.NoError(t, err) // check the type assertion diff --git a/server/routing/validators/validators.go b/server/routing/validators/validators.go deleted file mode 100644 index 775893359..000000000 --- a/server/routing/validators/validators.go +++ /dev/null @@ -1,440 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package validators - -import ( - "errors" - "strconv" - "strings" - - "github.com/agntcy/dir/server/types" - "github.com/agntcy/dir/utils/logging" - "github.com/ipfs/go-cid" - record "github.com/libp2p/go-libp2p-record" -) - -// Import routing utilities for label validation -// Note: Since validators is a sub-package of routing, it can import from the parent - -// IsValidLabelKey checks if a key starts with any valid label type prefix. -func IsValidLabelKey(key string) bool { - for _, labelType := range types.AllLabelTypes() { - if strings.HasPrefix(key, labelType.Prefix()) { - return true - } - } - - return false -} - -var validatorLogger = logging.Logger("routing/validators") - -// BaseValidator provides common validation logic for all label validators. -type BaseValidator struct{} - -// validateKeyFormat validates the enhanced DHT key format with PeerID. -func (v *BaseValidator) validateKeyFormat(key string, expectedNamespace string) ([]string, error) { - // Parse enhanced key format: //// - // Minimum parts: ["", "namespace", "path", "cid", "peer_id"] - parts := strings.Split(key, "/") - if len(parts) < types.MinLabelKeyParts { - return nil, errors.New("invalid key format: expected ////") - } - - // Validate namespace - if parts[1] != expectedNamespace { - return nil, errors.New("invalid namespace: expected " + expectedNamespace + ", got " + parts[1]) - } - - // Extract and validate PeerID (last part) first - peerID := parts[len(parts)-1] - if peerID == "" { - return nil, errors.New("missing PeerID in key") - } - - // Check if the last part looks like a CID (common mistake) - if _, err := cid.Decode(peerID); err == nil { - return nil, errors.New("invalid key format: expected ////") - } - - // Extract and validate CID (second to last part) - cidStr := parts[len(parts)-2] - if cidStr == "" { - return nil, errors.New("missing CID in key") - } - - // Validate CID format - _, err := cid.Decode(cidStr) - if err != nil { - return nil, errors.New("invalid CID format: " + err.Error()) - } - - return parts, nil -} - -// validateValue validates the DHT value (if present). -func (v *BaseValidator) validateValue(value []byte) error { - if len(value) > 0 { - // Value should be a valid CID if present - _, err := cid.Decode(string(value)) - if err != nil { - return errors.New("invalid CID in value: " + err.Error()) - } - } - - return nil -} - -// selectFirstValid provides default selection logic for all validators. -func (v *BaseValidator) selectFirstValid(key string, values [][]byte, validateFunc func(string, []byte) error) (int, error) { - validatorLogger.Debug("Selecting from multiple DHT record values", "key", key, "count", len(values)) - - if len(values) == 0 { - return -1, errors.New("no values to select from") - } - - for i, value := range values { - err := validateFunc(key, value) - if err == nil { - validatorLogger.Debug("Selected DHT record value", "key", key, "index", i) - - return i, nil - } - } - - validatorLogger.Warn("No valid values found for DHT record", "key", key) - - return -1, errors.New("no valid values found") -} - -// SkillValidator validates DHT records for skill-based content discovery. -type SkillValidator struct { - BaseValidator -} - -// Validate validates a skills DHT record. -// Key format: /skills/// -// Future: Can validate against skill taxonomy, required levels, etc. -func (v *SkillValidator) Validate(key string, value []byte) error { - validatorLogger.Debug("Validating skills DHT record", "key", key) - - // Basic format validation - parts, err := v.validateKeyFormat(key, types.LabelTypeSkill.String()) - if err != nil { - return err - } - - // Skills-specific validation - if err := v.validateSkillsSpecific(parts); err != nil { - return err - } - - // Value validation - if err := v.validateValue(value); err != nil { - return err - } - - validatorLogger.Debug("Skills DHT record validation successful", "key", key) - - return nil -} - -// validateSkillsSpecific performs skills-specific validation logic. -func (v *SkillValidator) validateSkillsSpecific(parts []string) error { - // parts[0] = "", parts[1] = "skills", parts[2:len-2] = skill path components, parts[len-2] = cid, parts[len-1] = peer_id - // Enhanced format: /skills/// - if len(parts) < types.MinLabelKeyParts { - return errors.New("skills key must have format: /skills///") - } - - // Extract skill path (everything between "skills" and CID) - skillParts := parts[2 : len(parts)-2] // Exclude CID and PeerID - if len(skillParts) == 0 { - return errors.New("skill path cannot be empty") - } - - // Validate that none of the skill path components are empty - for i, part := range skillParts { - if part == "" { - return errors.New("skill path component cannot be empty at position " + strconv.Itoa(i+1)) - } - } - - // Future: validate against skill taxonomy - // skillPath := strings.Join(skillParts, "/") - // if !v.isValidSkillPath(skillPath) { - // return errors.New("invalid skill path: " + skillPath) - // } - - return nil -} - -// Select chooses between multiple values for skills records. -func (v *SkillValidator) Select(key string, values [][]byte) (int, error) { - return v.selectFirstValid(key, values, v.Validate) -} - -// DomainValidator validates DHT records for domain-based content discovery. -type DomainValidator struct { - BaseValidator -} - -// Validate validates a domains DHT record. -// Key format: /domains// -// Future: Can validate against domain ontology, registry, etc. -func (v *DomainValidator) Validate(key string, value []byte) error { - validatorLogger.Debug("Validating domains DHT record", "key", key) - - // Basic format validation - parts, err := v.validateKeyFormat(key, types.LabelTypeDomain.String()) - if err != nil { - return err - } - - // Domains-specific validation - if err := v.validateDomainsSpecific(parts); err != nil { - return err - } - - // Value validation - if err := v.validateValue(value); err != nil { - return err - } - - validatorLogger.Debug("Domains DHT record validation successful", "key", key) - - return nil -} - -// validateDomainsSpecific performs domains-specific validation logic. -func (v *DomainValidator) validateDomainsSpecific(parts []string) error { - // parts[0] = "", parts[1] = "domains", parts[2:len-2] = domain path components, parts[len-2] = cid, parts[len-1] = peer_id - // Enhanced format: /domains/// - if len(parts) < types.MinLabelKeyParts { - return errors.New("domains key must have format: /domains///") - } - - // Extract domain path (everything between "domains" and CID) - domainParts := parts[2 : len(parts)-2] // Exclude CID and PeerID - if len(domainParts) == 0 { - return errors.New("domain path cannot be empty") - } - - // Future: validate against domain registry/ontology - // domain := strings.Join(domainParts, "/") - // if !v.isValidDomain(domain) { - // return errors.New("invalid domain: " + domain) - // } - - return nil -} - -// Select chooses between multiple values for domains records. -func (v *DomainValidator) Select(key string, values [][]byte) (int, error) { - return v.selectFirstValid(key, values, v.Validate) -} - -// ModuleValidator validates DHT records for module-based content discovery. -type ModuleValidator struct { - BaseValidator -} - -// Validate validates a modules DHT record. -// Key format: /modules// -// Future: Can validate against module specifications, versions, etc. -func (v *ModuleValidator) Validate(key string, value []byte) error { - validatorLogger.Debug("Validating modules DHT record", "key", key) - - // Basic format validation - parts, err := v.validateKeyFormat(key, types.LabelTypeModule.String()) - if err != nil { - return err - } - - // Modules-specific validation - if err := v.validateModulesSpecific(parts); err != nil { - return err - } - - // Value validation - if err := v.validateValue(value); err != nil { - return err - } - - validatorLogger.Debug("Modules DHT record validation successful", "key", key) - - return nil -} - -// validateModulesSpecific performs modules-specific validation logic. -func (v *ModuleValidator) validateModulesSpecific(parts []string) error { - // parts[0] = "", parts[1] = "modules", parts[2:len-2] = module path components, parts[len-2] = cid, parts[len-1] = peer_id - // Enhanced format: /modules/// - if len(parts) < types.MinLabelKeyParts { - return errors.New("modules key must have format: /modules///") - } - - // Extract module path (everything between "modules" and CID) - moduleParts := parts[2 : len(parts)-2] // Exclude CID and PeerID - if len(moduleParts) == 0 { - return errors.New("module path cannot be empty") - } - - // Future: validate against module specifications - // module := strings.Join(moduleParts, "/") - // if !v.isValidModule(module) { - // return errors.New("invalid module: " + module) - // } - - return nil -} - -// Select chooses between multiple values for modules records. -func (v *ModuleValidator) Select(key string, values [][]byte) (int, error) { - return v.selectFirstValid(key, values, v.Validate) -} - -// LocatorValidator validates DHT records for locator-based content discovery. -type LocatorValidator struct { - BaseValidator -} - -// Validate validates a locators DHT record. -// Key format: /locators/// -// Future: Can validate against supported locator types, registry, etc. -func (v *LocatorValidator) Validate(key string, value []byte) error { - validatorLogger.Debug("Validating locators DHT record", "key", key) - - // Basic format validation - parts, err := v.validateKeyFormat(key, types.LabelTypeLocator.String()) - if err != nil { - return err - } - - // Locators-specific validation - if err := v.validateLocatorsSpecific(parts); err != nil { - return err - } - - // Value validation - if err := v.validateValue(value); err != nil { - return err - } - - validatorLogger.Debug("Locators DHT record validation successful", "key", key) - - return nil -} - -// validateLocatorsSpecific performs locators-specific validation logic. -func (v *LocatorValidator) validateLocatorsSpecific(parts []string) error { - // parts[0] = "", parts[1] = "locators", parts[2:len-2] = locator path components, parts[len-2] = cid, parts[len-1] = peer_id - // Enhanced format: /locators/// - if len(parts) < types.MinLabelKeyParts { - return errors.New("locators key must have format: /locators///") - } - - // Extract locator type (everything between "locators" and CID) - locatorParts := parts[2 : len(parts)-2] // Exclude CID and PeerID - if len(locatorParts) == 0 { - return errors.New("locator type cannot be empty") - } - - // Validate that none of the locator path components are empty - for i, part := range locatorParts { - if part == "" { - return errors.New("locator path component cannot be empty at position " + strconv.Itoa(i+1)) - } - } - - // Future: validate against supported locator types - // locatorType := strings.Join(locatorParts, "/") - // if !v.isValidLocatorType(locatorType) { - // return errors.New("invalid locator type: " + locatorType) - // } - - return nil -} - -// Select chooses between multiple values for locators records. -func (v *LocatorValidator) Select(key string, values [][]byte) (int, error) { - return v.selectFirstValid(key, values, v.Validate) -} - -// CreateLabelValidators creates separate validators for each label namespace. -func CreateLabelValidators() map[string]record.Validator { - return map[string]record.Validator{ - types.LabelTypeSkill.String(): &SkillValidator{}, - types.LabelTypeDomain.String(): &DomainValidator{}, - types.LabelTypeModule.String(): &ModuleValidator{}, - types.LabelTypeLocator.String(): &LocatorValidator{}, - } -} - -// ValidateLabelKey validates a label key format before storing in DHT. -func ValidateLabelKey(key string) error { - parts := strings.Split(key, "/") - if len(parts) < types.MinLabelKeyParts { - return errors.New("invalid key format: expected ///") - } - - namespace := parts[1] - if _, valid := types.ParseLabelType(namespace); !valid { - return errors.New("unsupported namespace: " + namespace) - } - - // Extract and validate CID (last part) - cidStr := parts[len(parts)-1] - if cidStr == "" { - return errors.New("missing CID in key") - } - - _, err := cid.Decode(cidStr) - if err != nil { - return errors.New("invalid CID format: " + err.Error()) - } - - return nil -} - -// FormatLabelKey formats a label and CID into a proper DHT key. -func FormatLabelKey(label, cidStr string) string { - // Ensure label starts with / - if !strings.HasPrefix(label, "/") { - label = "/" + label - } - - // Ensure no double slashes and add CID - key := strings.TrimSuffix(label, "/") + "/" + cidStr - - return key -} - -// ExtractCIDFromLabelKey extracts CID from enhanced label key format. -// Example: "/skills/golang/CID123/Peer1" β†’ "CID123", nil. -func ExtractCIDFromLabelKey(labelKey string) (string, error) { - parts := strings.Split(labelKey, "/") - if len(parts) < types.MinLabelKeyParts { - return "", errors.New("invalid enhanced key format: expected ////") - } - - // Validate it's a proper label key - if !IsValidLabelKey(labelKey) { - return "", errors.New("invalid namespace in label key") - } - - // Extract and validate CID (second to last part) - cidStr := parts[len(parts)-2] - if cidStr == "" { - return "", errors.New("missing CID in label key") - } - - // Validate CID format - _, err := cid.Decode(cidStr) - if err != nil { - return "", errors.New("invalid CID format: " + err.Error()) - } - - return cidStr, nil -} diff --git a/server/routing/validators/validators_test.go b/server/routing/validators/validators_test.go deleted file mode 100644 index 2eb8b3e5f..000000000 --- a/server/routing/validators/validators_test.go +++ /dev/null @@ -1,893 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package validators - -import ( - "strings" - "testing" - - "github.com/agntcy/dir/server/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Add utility functions for testing. -func GetLabelTypeFromKey(key string) (types.LabelType, bool) { - for _, labelType := range types.AllLabelTypes() { - if strings.HasPrefix(key, labelType.Prefix()) { - return labelType, true - } - } - - return types.LabelTypeUnknown, false -} - -func TestSkillValidator_Validate(t *testing.T) { - validator := &SkillValidator{} - - tests := []struct { - name string - key string - value []byte - wantError bool - errorMsg string - }{ - { - name: "valid skills key with category and class", - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: false, - }, - { - name: "valid skills key with value", - key: "/skills/ai/machine-learning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2", - value: []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - wantError: false, - }, - { - name: "invalid namespace", - key: "/domains/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid namespace: expected skills, got domains", - }, - { - name: "missing skill path", - key: "/skills/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - { - name: "valid single skill path", - key: "/skills/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: false, - }, - { - name: "empty skill path component", - key: "/skills//golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "skill path component cannot be empty at position 1", - }, - { - name: "empty skill path component in middle", - key: "/skills/programming//advanced/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "skill path component cannot be empty at position 2", - }, - { - name: "invalid CID format", - key: "/skills/programming/golang/invalid-cid/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid CID format", - }, - { - name: "invalid value CID", - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte("invalid-cid-value"), - wantError: true, - errorMsg: "invalid CID in value", - }, - { - name: "missing CID", - key: "/skills/programming/golang//Peer1", - value: []byte{}, - wantError: true, - errorMsg: "missing CID in key", - }, - { - name: "missing PeerID", - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validator.Validate(tt.key, tt.value) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - } else { - require.NoError(t, err) - } - }) - } -} - -//nolint:dupl // Similar test structure is intentional for different validators -func TestDomainValidator_Validate(t *testing.T) { - validator := &DomainValidator{} - - tests := []struct { - name string - key string - value []byte - wantError bool - errorMsg string - }{ - { - name: "valid domains key with single domain", - key: "/domains/ai/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: false, - }, - { - name: "valid domains key with nested domain path", - key: "/domains/ai/machine-learning/nlp/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2", - value: []byte{}, - wantError: false, - }, - { - name: "valid domains key with value", - key: "/domains/software/web-development/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer3", - value: []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - wantError: false, - }, - { - name: "invalid namespace", - key: "/skills/ai/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid namespace: expected domains, got skills", - }, - { - name: "missing domain path", - key: "/domains/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - { - name: "invalid CID format", - key: "/domains/ai/invalid-cid/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid CID format", - }, - { - name: "invalid value CID", - key: "/domains/ai/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte("invalid-cid-value"), - wantError: true, - errorMsg: "invalid CID in value", - }, - { - name: "missing CID", - key: "/domains/ai//Peer1", - value: []byte{}, - wantError: true, - errorMsg: "missing CID in key", - }, - { - name: "missing PeerID", - key: "/domains/ai/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validator.Validate(tt.key, tt.value) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - } else { - require.NoError(t, err) - } - }) - } -} - -//nolint:dupl // Similar test structure is intentional for different validators -func TestModuleValidator_Validate(t *testing.T) { - validator := &ModuleValidator{} - - tests := []struct { - name string - key string - value []byte - wantError bool - errorMsg string - }{ - { - name: "valid modules key with single module", - key: "/modules/llm/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: false, - }, - { - name: "valid modules key with nested module path", - key: "/modules/ai/reasoning/logical/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2", - value: []byte{}, - wantError: false, - }, - { - name: "valid modules key with value", - key: "/modules/search/semantic/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer3", - value: []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - wantError: false, - }, - { - name: "invalid namespace", - key: "/domains/llm/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid namespace: expected modules, got domains", - }, - { - name: "missing module path", - key: "/modules/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - { - name: "invalid CID format", - key: "/modules/llm/invalid-cid/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid CID format", - }, - { - name: "invalid value CID", - key: "/modules/llm/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte("invalid-cid-value"), - wantError: true, - errorMsg: "invalid CID in value", - }, - { - name: "missing CID", - key: "/modules/llm//Peer1", - value: []byte{}, - wantError: true, - errorMsg: "missing CID in key", - }, - { - name: "missing PeerID", - key: "/modules/llm/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validator.Validate(tt.key, tt.value) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - } else { - require.NoError(t, err) - } - }) - } -} - -//nolint:dupl // Similar test structure is intentional for different validators -func TestLocatorValidator_Validate(t *testing.T) { - validator := &LocatorValidator{} - - tests := []struct { - name string - key string - value []byte - wantError bool - errorMsg string - }{ - { - name: "valid locators key with single locator type", - key: "/locators/docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: false, - }, - { - name: "valid locators key with nested locator path", - key: "/locators/container/docker/alpine/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2", - value: []byte{}, - wantError: false, - }, - { - name: "valid locators key with value", - key: "/locators/npm-package/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer3", - value: []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - wantError: false, - }, - { - name: "invalid namespace", - key: "/modules/docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid namespace: expected locators, got modules", - }, - { - name: "missing locator type", - key: "/locators/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - { - name: "invalid CID format", - key: "/locators/docker-image/invalid-cid/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "invalid CID format", - }, - { - name: "invalid value CID", - key: "/locators/docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte("invalid-cid-value"), - wantError: true, - errorMsg: "invalid CID in value", - }, - { - name: "empty locator path component", - key: "/locators//docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "locator path component cannot be empty at position 1", - }, - { - name: "empty locator path component in middle", - key: "/locators/container//alpine/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - value: []byte{}, - wantError: true, - errorMsg: "locator path component cannot be empty at position 2", - }, - { - name: "missing CID", - key: "/locators/docker-image//Peer1", - value: []byte{}, - wantError: true, - errorMsg: "missing CID in key", - }, - { - name: "missing PeerID", - key: "/locators/docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - value: []byte{}, - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validator.Validate(tt.key, tt.value) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - } else { - require.NoError(t, err) - } - }) - } -} - -func TestValidators_Select(t *testing.T) { - tests := []struct { - name string - validator interface { - Select(string, [][]byte) (int, error) - } - key string - values [][]byte - wantIndex int - wantError bool - errorMsg string - }{ - { - name: "skills validator - select first valid value", - validator: &SkillValidator{}, - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - values: [][]byte{ - []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - []byte("invalid-cid"), - }, - wantIndex: 0, - wantError: false, - }, - { - name: "domains validator - select first valid from multiple", - validator: &DomainValidator{}, - key: "/domains/ai/machine-learning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2", - values: [][]byte{ - []byte("invalid-cid"), - []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - []byte(""), - }, - wantIndex: 1, - wantError: false, - }, - { - name: "modules validator - no valid values", - validator: &ModuleValidator{}, - key: "/modules/llm/reasoning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer3", - values: [][]byte{ - []byte("invalid-cid-1"), - []byte("invalid-cid-2"), - }, - wantIndex: -1, - wantError: true, - errorMsg: "no valid values found", - }, - { - name: "locators validator - select first valid value", - validator: &LocatorValidator{}, - key: "/locators/docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - values: [][]byte{ - []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - []byte("invalid-cid"), - }, - wantIndex: 0, - wantError: false, - }, - { - name: "empty values slice", - validator: &SkillValidator{}, - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - values: [][]byte{}, - wantIndex: -1, - wantError: true, - errorMsg: "no values to select from", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - index, err := tt.validator.Select(tt.key, tt.values) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - assert.Equal(t, tt.wantIndex, index) - } else { - require.NoError(t, err) - assert.Equal(t, tt.wantIndex, index) - } - }) - } -} - -func TestLabelTypeIntegration(t *testing.T) { - // Test that LabelType works correctly with validators - // Test String() method - assert.Equal(t, "skills", types.LabelTypeSkill.String()) - assert.Equal(t, "domains", types.LabelTypeDomain.String()) - assert.Equal(t, "modules", types.LabelTypeModule.String()) - assert.Equal(t, "locators", types.LabelTypeLocator.String()) - - // Test Prefix() method - assert.Equal(t, "/skills/", types.LabelTypeSkill.Prefix()) - assert.Equal(t, "/domains/", types.LabelTypeDomain.Prefix()) - assert.Equal(t, "/modules/", types.LabelTypeModule.Prefix()) - assert.Equal(t, "/locators/", types.LabelTypeLocator.Prefix()) - - // Test IsValid() method - assert.True(t, types.LabelTypeSkill.IsValid()) - assert.True(t, types.LabelTypeDomain.IsValid()) - assert.True(t, types.LabelTypeModule.IsValid()) - assert.True(t, types.LabelTypeLocator.IsValid()) - assert.False(t, types.LabelType("invalid").IsValid()) - - // Test ParseLabelType() function - lt, valid := types.ParseLabelType("skills") - assert.True(t, valid) - assert.Equal(t, types.LabelTypeSkill, lt) - - lt, valid = types.ParseLabelType("invalid") - assert.False(t, valid) - assert.Equal(t, types.LabelTypeUnknown, lt) - - // Test AllLabelTypes() function - all := types.AllLabelTypes() - assert.Len(t, all, 4) - assert.Contains(t, all, types.LabelTypeSkill) - assert.Contains(t, all, types.LabelTypeDomain) - assert.Contains(t, all, types.LabelTypeModule) - assert.Contains(t, all, types.LabelTypeLocator) - - // Test IsValidLabelKey() function - assert.True(t, IsValidLabelKey("/skills/golang/CID123")) - assert.True(t, IsValidLabelKey("/domains/web/CID123")) - assert.True(t, IsValidLabelKey("/modules/chat/CID123")) - assert.True(t, IsValidLabelKey("/locators/docker-image/CID123")) - assert.False(t, IsValidLabelKey("/invalid/test/CID123")) - assert.False(t, IsValidLabelKey("/records/CID123")) - assert.False(t, IsValidLabelKey("skills/golang/CID123")) // missing leading slash - - // Test GetLabelTypeFromKey() function - lt, found := GetLabelTypeFromKey("/skills/golang/CID123") - assert.True(t, found) - assert.Equal(t, types.LabelTypeSkill, lt) - - lt, found = GetLabelTypeFromKey("/domains/web/CID123") - assert.True(t, found) - assert.Equal(t, types.LabelTypeDomain, lt) - - lt, found = GetLabelTypeFromKey("/modules/chat/CID123") - assert.True(t, found) - assert.Equal(t, types.LabelTypeModule, lt) - - lt, found = GetLabelTypeFromKey("/invalid/test/CID123") - assert.False(t, found) - assert.Equal(t, types.LabelTypeUnknown, lt) -} - -func TestCreateLabelValidators(t *testing.T) { - validators := CreateLabelValidators() - - // Test that all expected validators are created - assert.Len(t, validators, 4) - assert.Contains(t, validators, types.LabelTypeSkill.String()) - assert.Contains(t, validators, types.LabelTypeDomain.String()) - assert.Contains(t, validators, types.LabelTypeModule.String()) - assert.Contains(t, validators, types.LabelTypeLocator.String()) - - // Test that validators are of correct types - assert.IsType(t, &SkillValidator{}, validators[types.LabelTypeSkill.String()]) - assert.IsType(t, &DomainValidator{}, validators[types.LabelTypeDomain.String()]) - assert.IsType(t, &ModuleValidator{}, validators[types.LabelTypeModule.String()]) - assert.IsType(t, &LocatorValidator{}, validators[types.LabelTypeLocator.String()]) -} - -func TestValidateLabelKey(t *testing.T) { - tests := []struct { - name string - key string - wantError bool - errorMsg string - }{ - { - name: "valid skills key", - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: false, - }, - { - name: "valid domains key", - key: "/domains/ai/machine-learning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: false, - }, - { - name: "valid modules key", - key: "/modules/llm/reasoning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: false, - }, - { - name: "invalid format - too few parts", - key: "/skills/programming", - wantError: true, - errorMsg: "invalid key format: expected ///", - }, - { - name: "unsupported namespace", - key: "/unknown/path/value/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: true, - errorMsg: "unsupported namespace: unknown", - }, - { - name: "missing CID", - key: "/skills/programming/golang/", - wantError: true, - errorMsg: "missing CID in key", - }, - { - name: "invalid CID format", - key: "/skills/programming/golang/invalid-cid-format", - wantError: true, - errorMsg: "invalid CID format", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ValidateLabelKey(tt.key) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - } else { - require.NoError(t, err) - } - }) - } -} - -func TestFormatLabelKey(t *testing.T) { - tests := []struct { - name string - label string - cid string - expected string - }{ - { - name: "label with leading slash", - label: "/skills/programming/golang", - cid: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - expected: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - }, - { - name: "label without leading slash", - label: "skills/programming/golang", - cid: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - expected: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - }, - { - name: "label with trailing slash", - label: "/domains/ai/machine-learning/", - cid: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - expected: "/domains/ai/machine-learning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - }, - { - name: "single component label", - label: "/modules/llm", - cid: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - expected: "/modules/llm/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := FormatLabelKey(tt.label, tt.cid) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestBaseValidator_validateKeyFormat(t *testing.T) { - validator := &BaseValidator{} - - tests := []struct { - name string - key string - expectedNamespace string - wantError bool - errorMsg string - expectedParts []string - }{ - { - name: "valid key format", - key: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - expectedNamespace: types.LabelTypeSkill.String(), - wantError: false, - expectedParts: []string{"", "skills", "programming", "golang", "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", "Peer1"}, - }, - { - name: "invalid format - too few parts", - key: "/skills/programming", - expectedNamespace: types.LabelTypeSkill.String(), - wantError: true, - errorMsg: "invalid key format: expected ////", - }, - { - name: "wrong namespace", - key: "/domains/ai/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - expectedNamespace: types.LabelTypeSkill.String(), - wantError: true, - errorMsg: "invalid namespace: expected skills, got domains", - }, - { - name: "invalid CID", - key: "/skills/programming/golang/invalid-cid/Peer1", - expectedNamespace: types.LabelTypeSkill.String(), - wantError: true, - errorMsg: "invalid CID format", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - parts, err := validator.validateKeyFormat(tt.key, tt.expectedNamespace) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - assert.Nil(t, parts) - } else { - require.NoError(t, err) - assert.Equal(t, tt.expectedParts, parts) - } - }) - } -} - -func TestBaseValidator_validateValue(t *testing.T) { - validator := &BaseValidator{} - - tests := []struct { - name string - value []byte - wantError bool - errorMsg string - }{ - { - name: "empty value", - value: []byte{}, - wantError: false, - }, - { - name: "valid CID value", - value: []byte("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), - wantError: false, - }, - { - name: "invalid CID value", - value: []byte("invalid-cid-format"), - wantError: true, - errorMsg: "invalid CID in value", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validator.validateValue(tt.value) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - } else { - require.NoError(t, err) - } - }) - } -} - -// Benchmark tests to ensure validators perform well. -func BenchmarkSkillValidator_Validate(b *testing.B) { - validator := &SkillValidator{} - key := "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1" - value := []byte{} - - for b.Loop() { - _ = validator.Validate(key, value) - } -} - -func BenchmarkDomainValidator_Validate(b *testing.B) { - validator := &DomainValidator{} - key := "/domains/ai/machine-learning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2" - value := []byte{} - - for b.Loop() { - _ = validator.Validate(key, value) - } -} - -func BenchmarkModuleValidator_Validate(b *testing.B) { - validator := &ModuleValidator{} - key := "/modules/llm/reasoning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer3" - value := []byte{} - - for b.Loop() { - _ = validator.Validate(key, value) - } -} - -func BenchmarkLocatorValidator_Validate(b *testing.B) { - validator := &LocatorValidator{} - key := "/locators/docker-image/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1" - value := []byte{} - - for b.Loop() { - _ = validator.Validate(key, value) - } -} - -func TestExtractCIDFromLabelKey(t *testing.T) { - tests := []struct { - name string - labelKey string - wantCID string - wantError bool - errorMsg string - }{ - { - name: "valid skills key", - labelKey: "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - wantCID: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: false, - }, - { - name: "valid domains key", - labelKey: "/domains/ai/machine-learning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer2", - wantCID: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: false, - }, - { - name: "valid modules key", - labelKey: "/modules/llm/reasoning/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer3", - wantCID: "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - wantError: false, - }, - { - name: "invalid format - too few parts", - labelKey: "/skills/programming", - wantError: true, - errorMsg: "invalid enhanced key format", - }, - { - name: "invalid namespace", - labelKey: "/unknown/test/value/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1", - wantError: true, - errorMsg: "invalid namespace", - }, - { - name: "invalid CID format", - labelKey: "/skills/programming/golang/invalid-cid/Peer1", - wantError: true, - errorMsg: "invalid CID format", - }, - { - name: "missing CID", - labelKey: "/skills/programming/golang//Peer1", - wantError: true, - errorMsg: "missing CID", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cid, err := ExtractCIDFromLabelKey(tt.labelKey) - - if tt.wantError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errorMsg) - assert.Empty(t, cid) - } else { - require.NoError(t, err) - assert.Equal(t, tt.wantCID, cid) - } - }) - } -} - -func BenchmarkFormatLabelKey(b *testing.B) { - label := "/skills/programming/golang" - cid := "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" - - for b.Loop() { - _ = FormatLabelKey(label, cid) - } -} - -func BenchmarkExtractCIDFromLabelKey(b *testing.B) { - labelKey := "/skills/programming/golang/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/Peer1" - - for b.Loop() { - _, _ = ExtractCIDFromLabelKey(labelKey) - } -} diff --git a/server/server.go b/server/server.go index 732769a0d..353082fbc 100644 --- a/server/server.go +++ b/server/server.go @@ -248,8 +248,8 @@ func New(ctx context.Context, cfg *config.Config, opts ...ServerOption) (*Server return nil, fmt.Errorf("failed to create store: %w", err) } - // Database must be created before routing so the shared ingestion service - // (used by both the store controller and DHT autosync) can be wired in. + // Database must be created before routing, which queries it to decide what + // to advertise and to answer peer record queries. databaseAPI := o.database if databaseAPI == nil { databaseAPI, err = database.New(cfg.Database) @@ -262,7 +262,7 @@ func New(ctx context.Context, cfg *config.Config, opts ...ServerOption) (*Server // records/referrers (content store + search index + referrer DB state). ingestor := ingest.New(storeAPI, databaseAPI) - routingAPI, err := routing.New(ctx, storeAPI, ingestor, oasfValidator, options) + routingAPI, err := routing.New(ctx, storeAPI, databaseAPI, options) if err != nil { return nil, fmt.Errorf("failed to create routing: %w", err) } @@ -314,7 +314,7 @@ func New(ctx context.Context, cfg *config.Config, opts ...ServerOption) (*Server eventsv1.RegisterEventServiceServer(grpcServer, controller.NewEventsController(eventService)) storev1.RegisterStoreServiceServer(grpcServer, controller.NewStoreController(storeAPI, databaseAPI, ingestor, options.EventBus(), oasfValidator)) routingv1.RegisterRoutingServiceServer(grpcServer, controller.NewRoutingController(routingAPI, storeAPI, publicationService)) - routingv1.RegisterPublicationServiceServer(grpcServer, controller.NewPublicationController(databaseAPI, options)) + routingv1.RegisterPublicationServiceServer(grpcServer, controller.NewPublicationController(databaseAPI, publicationService, options)) searchv1.RegisterSearchServiceServer(grpcServer, controller.NewSearchController(databaseAPI, storeAPI)) storev1.RegisterSyncServiceServer(grpcServer, controller.NewSyncController(databaseAPI, options)) signv1.RegisterSignServiceServer(grpcServer, controller.NewSignController(databaseAPI)) @@ -430,7 +430,7 @@ func (s Server) Close(ctx context.Context) { } } - // Stop routing service (closes GossipSub, p2p server, DHT) + // Stop routing service (closes p2p server, DHT) if s.routing != nil { if err := s.routing.Stop(); err != nil { logger.Error("Failed to stop routing service", "error", err) diff --git a/server/skill/publisher.go b/server/skill/publisher.go index ef0024cd6..de2e6114a 100644 --- a/server/skill/publisher.go +++ b/server/skill/publisher.go @@ -48,8 +48,9 @@ func Publish(ctx context.Context, store types.StoreAPI, db types.DatabaseAPI, va return fmt.Errorf("push skill record: %w", err) } - // Update the search index in line with the gRPC store controller, so the - // record is discoverable without waiting for an external push. + // Index the record and mark it published. Records are unpublished by + // default, but this one describes the node itself, so being discoverable + // is its whole purpose. decoded, decodeErr := record.Decode() if decodeErr != nil { logger.Warn("DIR skill record pushed but could not be decoded for search index", @@ -61,6 +62,11 @@ func Publish(ctx context.Context, store types.StoreAPI, db types.DatabaseAPI, va "cid", ref.GetCid(), "error", addErr, ) + } else if pubErr := db.SetRecordPublished(ref.GetCid(), true); pubErr != nil { + logger.Warn("DIR skill record indexed but could not be marked published", + "cid", ref.GetCid(), + "error", pubErr, + ) } logger.Info("DIR skill record published", diff --git a/server/types/database.go b/server/types/database.go index 51dfa562d..6b4c3b3ba 100644 --- a/server/types/database.go +++ b/server/types/database.go @@ -53,11 +53,24 @@ type SearchDatabaseAPI interface { GetRecordCIDs(opts ...FilterOption) ([]string, error) // GetRecords retrieves full records based on the provided filters. + // + // Associations (skills, domains, modules, locators) are not loaded. Use + // GetRecordLabels when you need them. GetRecords(opts ...FilterOption) ([]coretypes.Record, error) + // GetRecordLabels returns the routing labels of each given record, keyed by + // CID. CIDs with no labels are absent from the result rather than mapped to + // an empty slice. + GetRecordLabels(cids []string) (map[string][]Label, error) + // RemoveRecord removes a record from the search database by CID. RemoveRecord(cid string) error + // SetRecordPublished sets whether this node announces the record to the + // network. Records are unpublished when first indexed; only Publish sets + // the flag, and Unpublish clears it. + SetRecordPublished(recordCID string, published bool) error + // SetRecordSigned marks a record as signed (called when a signature is attached). SetRecordSigned(recordCID string) error } diff --git a/server/types/label.go b/server/types/label.go index ac2c96cfc..771ec0256 100644 --- a/server/types/label.go +++ b/server/types/label.go @@ -4,13 +4,11 @@ package types // Label types and operations for the routing system. -// This file provides unified label types including Label, LabelType, and LabelMetadata, +// This file provides unified label types including Label and LabelType, // along with utilities for label extraction and manipulation. import ( - "errors" "strings" - "time" coretypes "github.com/agntcy/dir/api/core/types" ) @@ -129,52 +127,6 @@ func (l Label) Value() string { return strings.TrimPrefix(string(l), namespace) } -// LabelMetadata stores temporal information about a label announcement. -// The label itself is stored in the datastore key structure: /skills/AI/CID123/Peer1 -// where the metadata tracks when the label was first announced and last seen. -type LabelMetadata struct { - Timestamp time.Time `json:"timestamp"` // When label was first announced - LastSeen time.Time `json:"last_seen"` // When label was last seen/refreshed -} - -// Validate checks if the metadata is valid and all required fields are properly set. -func (m *LabelMetadata) Validate() error { - if m.Timestamp.IsZero() { - return errors.New("timestamp cannot be zero") - } - - if m.LastSeen.IsZero() { - return errors.New("last seen timestamp cannot be zero") - } - - if m.LastSeen.Before(m.Timestamp) { - return errors.New("last seen cannot be before creation timestamp") - } - - return nil -} - -// IsStale checks if the label is older than the given maximum age duration. -func (m *LabelMetadata) IsStale(maxAge time.Duration) bool { - return time.Since(m.LastSeen) > maxAge -} - -// Age returns how long ago the label was last seen. -func (m *LabelMetadata) Age() time.Duration { - return time.Since(m.LastSeen) -} - -// Update refreshes the LastSeen timestamp to the current time. -func (m *LabelMetadata) Update() { - m.LastSeen = time.Now() -} - -// Constants for label validation and processing. -const ( - // Enhanced format: /type/label/CID/PeerID splits into ["", "type", "label", "CID", "PeerID"] = 5 parts. - MinLabelKeyParts = 5 -) - // GetLabelsFromRecord extracts labels from a record. func GetLabelsFromRecord(record coretypes.Record) []Label { if record == nil { diff --git a/server/types/search.go b/server/types/search.go index ea1d77a9b..5d2c9e788 100644 --- a/server/types/search.go +++ b/server/types/search.go @@ -20,6 +20,7 @@ type RecordFilters struct { CreatedAts []string Authors []string SchemaVersions []string + Published *bool // Filter by whether this node announces the record to the network Verified *bool // Filter by verified status (name ownership verified via JWKS) Trusted *bool // Filter by trusted status (signature verification passed) ScanSafe *bool // Filter by is_safe: true = all scanners safe, false = at least one unsafe @@ -164,6 +165,14 @@ func WithModuleIDs(ids ...uint64) FilterOption { } } +// WithPublished filters records by whether this node announces them to the +// network. +func WithPublished(published bool) FilterOption { + return func(sc *RecordFilters) { + sc.Published = &published + } +} + // WithVerified filters records by verified status. func WithVerified(verified bool) FilterOption { return func(sc *RecordFilters) { diff --git a/tests/e2e/daemon/testenv/default/dir-daemon-config.yaml b/tests/e2e/daemon/testenv/default/dir-daemon-config.yaml index 6cf1ecd9f..b47ef71e8 100644 --- a/tests/e2e/daemon/testenv/default/dir-daemon-config.yaml +++ b/tests/e2e/daemon/testenv/default/dir-daemon-config.yaml @@ -16,8 +16,6 @@ server: enabled: true routing: listen_address: "/ip4/127.0.0.1/tcp/0" - gossipsub: - enabled: true database: type: "sqlite" sqlite: diff --git a/tests/e2e/daemon/testenv/external/dir-daemon-config.yaml b/tests/e2e/daemon/testenv/external/dir-daemon-config.yaml index fa869d268..6edc6a9da 100644 --- a/tests/e2e/daemon/testenv/external/dir-daemon-config.yaml +++ b/tests/e2e/daemon/testenv/external/dir-daemon-config.yaml @@ -16,8 +16,6 @@ server: routing: listen_address: "/ip4/127.0.0.1/tcp/0" datastore_dir: "routing" - gossipsub: - enabled: true database: type: "postgres" postgres: diff --git a/tests/e2e/local/testenv/local/dir-daemon-config.yaml b/tests/e2e/local/testenv/local/dir-daemon-config.yaml index 77e0f5435..91f8b8af0 100644 --- a/tests/e2e/local/testenv/local/dir-daemon-config.yaml +++ b/tests/e2e/local/testenv/local/dir-daemon-config.yaml @@ -22,8 +22,6 @@ server: routing: refresh_interval: "1s" listen_address: "/ip4/127.0.0.1/tcp/0" - gossipsub: - enabled: true database: type: "sqlite" sqlite: diff --git a/tests/e2e/network/04_gossipsub_test.go b/tests/e2e/network/04_gossipsub_test.go deleted file mode 100644 index 7c4e37351..000000000 --- a/tests/e2e/network/04_gossipsub_test.go +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -package network - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/agntcy/dir/tests/e2e/shared/testdata" - "github.com/agntcy/dir/tests/e2e/shared/utils" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -// Test file dedicated to testing GossipSub label announcement functionality. -// This verifies that labels are efficiently propagated via GossipSub mesh to ALL subscribed peers. - -// Package-level variables for cleanup (accessible by AfterSuite) -// CIDs are now tracked in network_suite_test.go - -var _ = ginkgo.Describe("Running GossipSub label announcement tests", ginkgo.Ordered, func() { - var cid string - - // Setup temp record file - tmpDir := os.TempDir() - tempPath := filepath.Join(tmpDir, "record_v070_gossipsub_test.json") - - // Create directory and write record data - _ = os.MkdirAll(filepath.Dir(tempPath), 0o755) - _ = os.WriteFile(tempPath, testdata.ExpectedRecordV070JSON, 0o600) - - ginkgo.BeforeEach(func() { - // Reset CLI state to ensure clean test environment - utils.ResetCLIState() - }) - - ginkgo.Context("GossipSub wide propagation to all peers", func() { - ginkgo.It("should push record_v070.json to peer 1", func() { - cid = testEnv.Peer1.Push(tempPath).WithArgs("--output", "raw").ShouldEventuallySucceed(15 * time.Second) - - // Track CID for cleanup - RegisterCIDForCleanup(cid, "gossipsub") - - // Validate that the returned CID correctly represents the pushed data - utils.LoadAndValidateCID(cid, tempPath) - }) - - ginkgo.It("should publish record to routing on peer 1", func() { - // Publish triggers both DHT.Provide() and GossipSub.PublishLabels() - testEnv.Peer1.Routing().Publish(cid).ShouldSucceed() - - ginkgo.GinkgoWriter.Printf("Published CID to routing with GossipSub: %s", cid) - }) - - ginkgo.It("should propagate labels via GossipSub to all subscribed peers", func() { - // GossipSub propagates much faster than DHT alone - // Expected: ~5 seconds vs 15 seconds for DHT-only propagation - ginkgo.GinkgoWriter.Printf("Waiting 5 seconds for GossipSub label propagation...") - time.Sleep(5 * time.Second) - - // Verify Peer2 received labels via GossipSub - ginkgo.GinkgoWriter.Printf("Testing label discovery on Peer2...") - utils.ResetCLIState() - - output2 := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing"). - WithLimit(10). - ShouldSucceed() - - gomega.Expect(output2).To(gomega.ContainSubstring(cid)) - ginkgo.GinkgoWriter.Printf("βœ… Peer2 discovered labels via GossipSub") - - // Verify Peer3 also received labels via GossipSub - ginkgo.GinkgoWriter.Printf("Testing label discovery on Peer3...") - utils.ResetCLIState() - - output3 := testEnv.Peer3.Routing().Search(). - WithSkill("natural_language_processing"). - WithLimit(10). - ShouldSucceed() - - gomega.Expect(output3).To(gomega.ContainSubstring(cid)) - ginkgo.GinkgoWriter.Printf("βœ… Peer3 discovered labels via GossipSub") - - ginkgo.GinkgoWriter.Printf("βœ… SUCCESS: GossipSub propagated labels to ALL 3 peers (not just k-closest)") - }) - - ginkgo.It("should verify labels are discoverable from both remote peers", func() { - // Additional verification with different skill query - utils.ResetCLIState() - - output2 := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing/natural_language_generation/text_completion"). - ShouldSucceed() - - gomega.Expect(output2).To(gomega.ContainSubstring(cid)) - gomega.Expect(output2).To(gomega.ContainSubstring("match_score")) - - utils.ResetCLIState() - - output3 := testEnv.Peer3.Routing().Search(). - WithSkill("natural_language_processing/analytical_reasoning/problem_solving"). - ShouldSucceed() - - gomega.Expect(output3).To(gomega.ContainSubstring(cid)) - gomega.Expect(output3).To(gomega.ContainSubstring("match_score")) - - ginkgo.GinkgoWriter.Printf("βœ… Both peers can search with specific skill queries") - }) - }) - - ginkgo.Context("GossipSub performance and timing", func() { - var ( - perfCID string - perfPath string - ) - - ginkgo.BeforeAll(func() { - // Setup separate record for performance testing - perfPath = filepath.Join(tmpDir, "record_v070_gossipsub_perf_test.json") - _ = os.WriteFile(perfPath, testdata.ExpectedRecordV070JSON, 0o600) - }) - - ginkgo.It("should push performance test record to peer 1", func() { - perfCID = testEnv.Peer1.Push(perfPath).WithArgs("--output", "raw").ShouldSucceed() - RegisterCIDForCleanup(perfCID, "gossipsub") - }) - - ginkgo.It("should discover labels in under 7 seconds via GossipSub", func() { - // Publish the record - testEnv.Peer1.Routing().Publish(perfCID).ShouldSucceed() - - startTime := time.Now() - ginkgo.GinkgoWriter.Printf("Starting timing test at %s", startTime.Format("15:04:05")) - - // Poll for label discovery with short intervals - // GossipSub should propagate in ~2-5 seconds - utils.ResetCLIState() - - output := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing"). - ShouldEventuallyContain(perfCID, 10*time.Second) // Max 10s timeout - - discoveryTime := time.Since(startTime) - ginkgo.GinkgoWriter.Printf("βœ… Labels discovered in %v", discoveryTime) - - // Verify it's faster than baseline DHT propagation (15s) - gomega.Expect(discoveryTime).To(gomega.BeNumerically("<", 7*time.Second), - "GossipSub should propagate faster than DHT-only baseline") - - gomega.Expect(output).To(gomega.ContainSubstring(perfCID)) - }) - }) - - ginkgo.Context("GossipSub bulk record propagation", func() { - var ( - bulkCIDs []string - bulkPaths []string - ) - - ginkgo.BeforeAll(func() { - // Prepare 5 test records for bulk testing - // Note: Reusing same record content but treating as separate for propagation test - bulkPaths = make([]string, 5) - for i := range 5 { - bulkPaths[i] = filepath.Join(tmpDir, fmt.Sprintf("record_v070_gossipsub_bulk_%d_test.json", i)) - _ = os.WriteFile(bulkPaths[i], testdata.ExpectedRecordV070JSON, 0o600) - } - }) - - ginkgo.It("should push 5 records to peer 1", func() { - bulkCIDs = make([]string, 5) - - for i, path := range bulkPaths { - cid := testEnv.Peer1.Push(path).WithArgs("--output", "raw").ShouldSucceed() - bulkCIDs[i] = cid - RegisterCIDForCleanup(cid, "gossipsub") - ginkgo.GinkgoWriter.Printf("Pushed bulk record %d/%d: %s", i+1, 5, cid) - } - }) - - ginkgo.It("should publish all 5 records sequentially", func() { - for i, bulkCID := range bulkCIDs { - testEnv.Peer1.Routing().Publish(bulkCID).ShouldSucceed() - ginkgo.GinkgoWriter.Printf("Published bulk record %d/%d via GossipSub", i+1, 5) - } - }) - - ginkgo.It("should propagate all 5 records' labels via GossipSub", func() { - // Verify all 5 records are discoverable from Peer2 - // Wait at least 10 seconds for GossipSub propagation of all announcements - utils.ResetCLIState() - - successCount := 0 - - for i, bulkCID := range bulkCIDs { - testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing"). - WithLimit(10). - ShouldEventuallyContain(bulkCID, 15*time.Second) - - successCount++ - - ginkgo.GinkgoWriter.Printf("βœ… Bulk record %d/%d discovered on Peer2", i+1, 5) - utils.ResetCLIState() - } - - // All 5 should be discoverable - gomega.Expect(successCount).To(gomega.Equal(5), - "All 5 records should be discoverable via GossipSub") - - ginkgo.GinkgoWriter.Printf("βœ… SUCCESS: GossipSub propagated all 5 records efficiently") - }) - - ginkgo.It("should verify bulk records are also discoverable from peer 3", func() { - // Verify propagation to Peer3 as well (proves mesh propagation) - utils.ResetCLIState() - - successCount := 0 - - for i, bulkCID := range bulkCIDs { - testEnv.Peer3.Routing().Search(). - WithSkill("natural_language_processing"). - WithLimit(10). - ShouldEventuallyContain(bulkCID, 15*time.Second) - - successCount++ - - ginkgo.GinkgoWriter.Printf("βœ… Bulk record %d/%d discovered on Peer3", i+1, 5) - utils.ResetCLIState() - } - - gomega.Expect(successCount).To(gomega.Equal(5), - "All 5 records should be discoverable on Peer3 via GossipSub") - - ginkgo.GinkgoWriter.Printf("βœ… SUCCESS: GossipSub mesh propagated to all peers") - }) - }) - - ginkgo.Context("GossipSub edge cases and validation", func() { - var edgeCID string - - ginkgo.It("should push edge case test record to peer 1", func() { - edgePath := filepath.Join(tmpDir, "record_v070_gossipsub_edge_test.json") - _ = os.WriteFile(edgePath, testdata.ExpectedRecordV070JSON, 0o600) - - edgeCID = testEnv.Peer1.Push(edgePath).WithArgs("--output", "raw").ShouldSucceed() - RegisterCIDForCleanup(edgeCID, "gossipsub") - }) - - ginkgo.It("should handle search with multiple label types via GossipSub", func() { - // Publish record - testEnv.Peer1.Routing().Publish(edgeCID).ShouldSucceed() - - // Wait for GossipSub propagation - time.Sleep(5 * time.Second) - - // Test search with OR logic across multiple label types - utils.ResetCLIState() - - output := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing"). // Should match - WithDomain("life_science"). // Should match (record has life_science/biotechnology) - WithMinScore(2). // Both should match - WithLimit(10). - WithArgs("--output", "json"). - ShouldSucceed() - - gomega.Expect(output).To(gomega.ContainSubstring(edgeCID)) - gomega.Expect(output).To(gomega.ContainSubstring("\"match_score\": 2")) - - ginkgo.GinkgoWriter.Printf("βœ… GossipSub propagates all label types correctly") - }) - - ginkgo.It("should verify labels persist across multiple searches", func() { - // Test that cached labels from GossipSub remain available - // This ensures the fallback to pull is NOT triggered on subsequent searches - - // First search - utils.ResetCLIState() - - output1 := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing"). - ShouldSucceed() - gomega.Expect(output1).To(gomega.ContainSubstring(edgeCID)) - - // Second search (should use cached labels, not pull again) - utils.ResetCLIState() - - output2 := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing/analytical_reasoning/problem_solving"). - ShouldSucceed() - gomega.Expect(output2).To(gomega.ContainSubstring(edgeCID)) - - // Third search with different peer - utils.ResetCLIState() - - output3 := testEnv.Peer3.Routing().Search(). - WithSkill("natural_language_processing/natural_language_generation"). - ShouldSucceed() - gomega.Expect(output3).To(gomega.ContainSubstring(edgeCID)) - - ginkgo.GinkgoWriter.Printf("βœ… Cached labels from GossipSub persist across multiple searches") - }) - }) - - ginkgo.Context("GossipSub comparison with baseline", func() { - ginkgo.It("should demonstrate faster propagation compared to DHT-only baseline", func() { - // This test compares against the known baseline from 01_deploy_test.go - // Baseline: 15 seconds wait for DHT propagation - // GossipSub: Should work in ~5 seconds - baselinePath := filepath.Join(tmpDir, "record_v070_gossipsub_baseline_test.json") - _ = os.WriteFile(baselinePath, testdata.ExpectedRecordV070JSON, 0o600) - - baselineCID := testEnv.Peer1.Push(baselinePath).WithArgs("--output", "raw").ShouldSucceed() - RegisterCIDForCleanup(baselineCID, "gossipsub") - - // Publish and start timing - testEnv.Peer1.Routing().Publish(baselineCID).ShouldSucceed() - - startTime := time.Now() - - // Poll for discovery with 1-second intervals - ginkgo.GinkgoWriter.Printf("Polling for label discovery (max 10 seconds)...") - utils.ResetCLIState() - - found := false - - maxAttempts := 10 - for attempt := 1; attempt <= maxAttempts; attempt++ { - output, err := testEnv.Peer2.Routing().Search(). - WithSkill("natural_language_processing"). - WithLimit(10). - Execute() - - if err == nil && strings.Contains(output, baselineCID) { - discoveryTime := time.Since(startTime) - ginkgo.GinkgoWriter.Printf("βœ… Labels discovered in %v (attempt %d/%d)", discoveryTime, attempt, maxAttempts) - - found = true - - // Verify it's faster than DHT baseline - gomega.Expect(discoveryTime).To(gomega.BeNumerically("<", 7*time.Second), - "GossipSub should be significantly faster than DHT-only baseline (15s)") - - break - } - - time.Sleep(1 * time.Second) - utils.ResetCLIState() - } - - gomega.Expect(found).To(gomega.BeTrue(), "Labels should be discovered within 10 seconds via GossipSub") - - // CLEANUP: This is the last test in this Describe block - ginkgo.DeferCleanup(func() { - CleanupNetworkRecords(gossipsubTestCIDs, "gossipsub tests", testEnv.PeerCLIs()) - }) - }) - }) -}) diff --git a/tests/e2e/network/cleanup.go b/tests/e2e/network/cleanup.go index 3e4f1f72f..c10ef727f 100644 --- a/tests/e2e/network/cleanup.go +++ b/tests/e2e/network/cleanup.go @@ -13,7 +13,6 @@ var ( deployTestCIDs []string syncTestCIDs []string remoteSearchTestCIDs []string - gossipsubTestCIDs []string nameResolutionTestCIDs []string ) @@ -56,8 +55,6 @@ func RegisterCIDForCleanup(cid, testFile string) { syncTestCIDs = append(syncTestCIDs, cid) case "search": remoteSearchTestCIDs = append(remoteSearchTestCIDs, cid) - case "gossipsub": - gossipsubTestCIDs = append(gossipsubTestCIDs, cid) case "name_resolution": nameResolutionTestCIDs = append(nameResolutionTestCIDs, cid) default: @@ -71,7 +68,6 @@ func CleanupAllNetworkTests(peers []*utils.CLI) { allCIDs = append(allCIDs, deployTestCIDs...) allCIDs = append(allCIDs, syncTestCIDs...) allCIDs = append(allCIDs, remoteSearchTestCIDs...) - allCIDs = append(allCIDs, gossipsubTestCIDs...) allCIDs = append(allCIDs, nameResolutionTestCIDs...) CleanupNetworkRecords(allCIDs, "all network tests", peers) diff --git a/tests/e2e/network/testenv/local/daemon-bootstrap-config.tpl.yaml b/tests/e2e/network/testenv/local/daemon-bootstrap-config.tpl.yaml index 62d28ac4c..3be1fa1fd 100644 --- a/tests/e2e/network/testenv/local/daemon-bootstrap-config.tpl.yaml +++ b/tests/e2e/network/testenv/local/daemon-bootstrap-config.tpl.yaml @@ -17,8 +17,6 @@ server: listen_address: "/ip4/127.0.0.1/tcp/18300" key_path: "${BOOTSTRAP_KEY_PATH}" datastore_dir: "routing" - gossipsub: - enabled: true database: type: "sqlite" sqlite: diff --git a/tests/go.mod b/tests/go.mod index a4b5d4fc1..e03ab935e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -390,7 +390,6 @@ require ( github.com/libp2p/go-libp2p-gorpc v0.6.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.41.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect - github.com/libp2p/go-libp2p-pubsub v0.16.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index c378ed046..95ea265d4 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -1171,8 +1171,6 @@ github.com/libp2p/go-libp2p-kad-dht v0.41.0 h1:sDigz5SgV20Crj8ItJmJpEAM+eJrzC/Sa github.com/libp2p/go-libp2p-kad-dht v0.41.0/go.mod h1:2qc4QGLvmIdznYbNg++FF76vp4q2SaBZyr76jHV8xgs= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= -github.com/libp2p/go-libp2p-pubsub v0.16.0 h1:j7G2C8kJwkcAQqYR7Wmq3d75d3Sgw/N0Hhiv0dVx7OY= -github.com/libp2p/go-libp2p-pubsub v0.16.0/go.mod h1:lr4oE8bFgQaifRcoc2uWhWWiK6tPdOEKpUuR408GFN4= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= diff --git a/tests/integration/server/testenv/test-config.yaml b/tests/integration/server/testenv/test-config.yaml index 3b46c1175..fcad11243 100644 --- a/tests/integration/server/testenv/test-config.yaml +++ b/tests/integration/server/testenv/test-config.yaml @@ -30,8 +30,6 @@ routing: bootstrap_peers: key_path: datastore_dir: - gossipsub: - enabled: true database: type: postgres postgres: From 6c6d7ffa6160de595f1af5ec803011aa4cabce2e Mon Sep 17 00:00:00 2001 From: Tibor Kircsi Date: Tue, 4 Aug 2026 17:32:46 +0200 Subject: [PATCH 2/3] fix(server): enforce SQLite foreign keys per connection Signed-off-by: Tibor Kircsi --- server/database/database.go | 26 ++++- .../004_purge_orphaned_record_children.go | 62 +++++++++++ server/database/gorm/record_orphan_test.go | 90 +++++++++++++++ server/database/sqlite_test.go | 105 ++++++++++++++++++ server/skill/publisher.go | 14 +-- 5 files changed, 282 insertions(+), 15 deletions(-) create mode 100644 server/database/gorm/migrations/004_purge_orphaned_record_children.go create mode 100644 server/database/gorm/record_orphan_test.go create mode 100644 server/database/sqlite_test.go diff --git a/server/database/database.go b/server/database/database.go index e36c90d71..f2a3680e6 100644 --- a/server/database/database.go +++ b/server/database/database.go @@ -75,6 +75,25 @@ func isMemoryDSN(path string) bool { return path == ":memory:" || strings.HasPrefix(path, "file::memory:") } +// withForeignKeys puts foreign key enforcement in the DSN so it applies to +// every connection the pool opens. +// +// SQLite disables foreign keys per connection, so running the pragma as a +// statement after Open only reaches whichever connection happened to serve it. +// The rest of the pool keeps enforcement off, and the ON DELETE CASCADE +// declared on Record's associations is silently skipped there, orphaning skill, +// domain, module and locator rows. +func withForeignKeys(path string) string { + const pragma = "_pragma=foreign_keys(1)" + + separator := "?" + if strings.Contains(path, "?") { + separator = "&" + } + + return path + separator + pragma +} + // newSQLite creates a new database connection using the pure-Go SQLite driver. func newSQLite(cfg config.SQLiteConfig) (*gormdb.DB, error) { path := cfg.Path @@ -86,7 +105,7 @@ func newSQLite(cfg config.SQLiteConfig) (*gormdb.DB, error) { path = config.EnsureFilePath(path) } - db, err := gorm.Open(sqlite.Open(path), &gorm.Config{ + db, err := gorm.Open(sqlite.Open(withForeignKeys(path)), &gorm.Config{ Logger: newCustomLogger(), }) if err != nil { @@ -97,11 +116,6 @@ func newSQLite(cfg config.SQLiteConfig) (*gormdb.DB, error) { return nil, fmt.Errorf("failed to configure SQLite connection pool: %w", err) } - // SQLite does not enforce foreign keys by default; enable for CASCADE support. - if err := db.Exec("PRAGMA foreign_keys = ON").Error; err != nil { - return nil, fmt.Errorf("failed to enable SQLite foreign keys: %w", err) - } - gdb, err := gormdb.New(db) if err != nil { return nil, fmt.Errorf("failed to initialize SQLite database: %w", err) diff --git a/server/database/gorm/migrations/004_purge_orphaned_record_children.go b/server/database/gorm/migrations/004_purge_orphaned_record_children.go new file mode 100644 index 000000000..a52c61582 --- /dev/null +++ b/server/database/gorm/migrations/004_purge_orphaned_record_children.go @@ -0,0 +1,62 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package migrations + +import ( + "fmt" + + "gorm.io/gorm" +) + +func init() { + register(Migration{ + ID: "004_purge_orphaned_record_children", + Details: "Delete rows in record child tables whose record no longer exists.", + Run: runPurgeOrphanedRecordChildren, + }) +} + +// orphanedChildTables are the tables that hang off records via ON DELETE +// CASCADE. SQLite enforces foreign keys per connection, and the pragma used to +// be issued as a statement after Open, so it only reached one connection in the +// pool. Deletes served by any other connection dropped the record and left its +// children behind. +// +// Orphaned label rows are the visible symptom: GetRecordLabels looks them up by +// record_cid, so re-pushing a deleted record (same content, same CID) returned +// each of its labels once per past life. +var orphanedChildTables = []string{ + "skills", + "domains", + "modules", + "locators", + "annotations", + "signature_verifications", + "scan_reports", + "name_verifications", + "record_usage_metrics", +} + +func runPurgeOrphanedRecordChildren(db *gorm.DB) error { + if !db.Migrator().HasTable("records") { + return nil + } + + for _, table := range orphanedChildTables { + if !db.Migrator().HasTable(table) { + continue + } + + stmt := fmt.Sprintf( + "DELETE FROM %s WHERE record_cid NOT IN (SELECT record_cid FROM records)", + table, + ) + + if err := db.Exec(stmt).Error; err != nil { + return fmt.Errorf("purge orphaned rows from %s: %w", table, err) + } + } + + return nil +} diff --git a/server/database/gorm/record_orphan_test.go b/server/database/gorm/record_orphan_test.go new file mode 100644 index 000000000..5bd159f9d --- /dev/null +++ b/server/database/gorm/record_orphan_test.go @@ -0,0 +1,90 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package gorm + +import ( + "testing" + "time" + + "github.com/agntcy/dir/server/database/gorm/migrations" + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +const purgeOrphansMigrationID = "004_purge_orphaned_record_children" + +// Records deleted while foreign keys were off left their children behind, and +// re-pushing the same content reuses the CID, so the stale rows attach +// themselves to the new record. +func TestOrphanedLabelsAreNotReturnedAfterRepush(t *testing.T) { + db, gdb := newOrphanTestDB(t) + + cid := "baeareitestorphan000000000000000000000000000000000000000000000000" + seedRecord(t, db, cid, "orphan-test", "signer-1", "key-1", time.Now().UTC()) + + // Reproduce a delete served by a connection without foreign keys on. + require.NoError(t, gdb.Exec("PRAGMA foreign_keys = OFF").Error) + require.NoError(t, db.RemoveRecord(cid)) + require.NoError(t, gdb.Exec("PRAGMA foreign_keys = ON").Error) + + // The record is gone but its labels are not. + var orphans int64 + require.NoError(t, gdb.Model(&Skill{}).Where("record_cid = ?", cid).Count(&orphans).Error) + require.Equal(t, int64(1), orphans, "expected the bug to leave an orphaned skill behind") + + runPurgeOrphansMigration(t, gdb) + + // Re-push the same content, which lands on the same CID. + seedRecord(t, db, cid, "orphan-test", "signer-1", "key-1", time.Now().UTC()) + + labels, err := db.GetRecordLabels([]string{cid}) + require.NoError(t, err) + + assert.Len(t, labels[cid], 4, "each label should appear once: %v", labels[cid]) +} + +func TestPurgeOrphansKeepsChildrenOfLiveRecords(t *testing.T) { + db, gdb := newOrphanTestDB(t) + + cid := "baeareitestlive00000000000000000000000000000000000000000000000000" + seedRecord(t, db, cid, "live-record", "signer-1", "key-1", time.Now().UTC()) + + runPurgeOrphansMigration(t, gdb) + + for _, model := range []any{ + &Skill{}, &Locator{}, &Module{}, &Domain{}, &Annotation{}, + &SignatureVerification{}, &NameVerification{}, &ScanReport{}, &RecordUsageMetrics{}, + } { + requireOneRowForCID(t, gdb, model, cid) + } +} + +func newOrphanTestDB(t *testing.T) (*DB, *gorm.DB) { + t.Helper() + + gdb, err := gorm.Open(sqlite.Open("file::memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, gdb.Exec("PRAGMA foreign_keys = ON").Error) + + db := &DB{gormDB: gdb} + require.NoError(t, db.migrate()) + + return db, gdb +} + +func runPurgeOrphansMigration(t *testing.T, gdb *gorm.DB) { + t.Helper() + + for _, m := range migrations.GetMigrations() { + if m.ID == purgeOrphansMigrationID { + require.NoError(t, m.Run(gdb)) + + return + } + } + + t.Fatalf("migration %s is not registered", purgeOrphansMigrationID) +} diff --git a/server/database/sqlite_test.go b/server/database/sqlite_test.go new file mode 100644 index 000000000..6b637fd2c --- /dev/null +++ b/server/database/sqlite_test.go @@ -0,0 +1,105 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +package database + +import ( + "path/filepath" + "sync" + "testing" + + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestWithForeignKeysBuildsADSNPerPathShape(t *testing.T) { + t.Parallel() + + tests := map[string]string{ + "/var/lib/dir/dir.db": "/var/lib/dir/dir.db?_pragma=foreign_keys(1)", + ":memory:": ":memory:?_pragma=foreign_keys(1)", + "file::memory:?cache=shared": "file::memory:?cache=shared&_pragma=foreign_keys(1)", + "file:dir.db?_txlock=immediate": "file:dir.db?_txlock=immediate&_pragma=foreign_keys(1)", + } + + for path, want := range tests { + assert.Equal(t, want, withForeignKeys(path), "path %q", path) + } +} + +// Foreign keys are a per-connection setting in SQLite. Running the pragma as a +// statement after Open only reaches one connection, leaving the rest of the +// pool to silently skip the ON DELETE CASCADE on Record's associations. +func TestForeignKeysAreOnForEveryPooledConnection(t *testing.T) { + t.Parallel() + + for name, path := range map[string]string{ + "file": filepath.Join(t.TempDir(), "pool.db"), + "memory": "file::memory:?cache=shared", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + db, err := gorm.Open(sqlite.Open(withForeignKeys(path)), &gorm.Config{}) + require.NoError(t, err) + + require.NoError(t, configureSQLPool(db)) + + sqlDB, err := db.DB() + require.NoError(t, err) + + // Hold every connection open at once so the pool has to open new + // ones rather than handing back the one Open configured. + const conns = 8 + + var ( + wg sync.WaitGroup + ready sync.WaitGroup + mu sync.Mutex + enabled []int + release = make(chan struct{}) + ) + + ready.Add(conns) + wg.Add(conns) + + for range conns { + go func() { + defer wg.Done() + + conn, err := sqlDB.Conn(t.Context()) + if !assert.NoError(t, err) { + ready.Done() + + return + } + defer conn.Close() + + var fk int + if assert.NoError(t, conn.QueryRowContext(t.Context(), "PRAGMA foreign_keys").Scan(&fk)) { + mu.Lock() + + enabled = append(enabled, fk) + + mu.Unlock() + } + + ready.Done() + <-release + }() + } + + ready.Wait() + close(release) + wg.Wait() + + require.Len(t, enabled, conns) + + for _, fk := range enabled { + assert.Equal(t, 1, fk, "a pooled connection had foreign keys off, so cascade deletes are skipped on it") + } + }) + } +} diff --git a/server/skill/publisher.go b/server/skill/publisher.go index de2e6114a..58719460b 100644 --- a/server/skill/publisher.go +++ b/server/skill/publisher.go @@ -48,9 +48,10 @@ func Publish(ctx context.Context, store types.StoreAPI, db types.DatabaseAPI, va return fmt.Errorf("push skill record: %w", err) } - // Index the record and mark it published. Records are unpublished by - // default, but this one describes the node itself, so being discoverable - // is its whole purpose. + // Index the record so it is locally searchable, in line with the gRPC store + // controller. It stays unpublished: announcing it would have every node in + // the network provide the same handful of label keys, which buries real + // records under one copy of this one per peer. decoded, decodeErr := record.Decode() if decodeErr != nil { logger.Warn("DIR skill record pushed but could not be decoded for search index", @@ -62,14 +63,9 @@ func Publish(ctx context.Context, store types.StoreAPI, db types.DatabaseAPI, va "cid", ref.GetCid(), "error", addErr, ) - } else if pubErr := db.SetRecordPublished(ref.GetCid(), true); pubErr != nil { - logger.Warn("DIR skill record indexed but could not be marked published", - "cid", ref.GetCid(), - "error", pubErr, - ) } - logger.Info("DIR skill record published", + logger.Info("DIR skill record stored", "cid", ref.GetCid(), "name", RecordName, "version", RecordVersion(), From 2ea99c34283bda8b274a44e12be35ba414d6752b Mon Sep 17 00:00:00 2001 From: Tibor Kircsi Date: Wed, 5 Aug 2026 00:08:48 +0200 Subject: [PATCH 3/3] fix(routing): unpublished records were remotely discoverable Signed-off-by: Tibor Kircsi --- server/routing/rpc/query_records.go | 14 +++++--- server/routing/rpc/query_records_test.go | 43 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/server/routing/rpc/query_records.go b/server/routing/rpc/query_records.go index 904311bf6..7fcec716d 100644 --- a/server/routing/rpc/query_records.go +++ b/server/routing/rpc/query_records.go @@ -184,6 +184,11 @@ func matchingCIDs(db types.DatabaseAPI, queries []RecordQuery, limit int) ([]str // Hierarchical namespaces match the value itself or any descendant, so a query // for "AI" finds "AI/ML" β€” the same prefix semantics the local matcher applies. // Locators are flat and match exactly. +// +// Only published records are served. A peer arrives here through any single +// label this node advertised, so without the same filter that governs +// advertising, one shared label would expose every other record the node holds +// β€” including private pushes and ingested replicas. func queryFilters(query RecordQuery, limit int) ([]types.FilterOption, error) { value := strings.TrimSpace(query.Value) if value == "" { @@ -191,16 +196,17 @@ func queryFilters(query RecordQuery, limit int) ([]types.FilterOption, error) { } descendants := value + "/*" + base := []types.FilterOption{types.WithPublished(true), types.WithLimit(limit)} switch types.LabelType(query.Type) { case types.LabelTypeSkill: - return []types.FilterOption{types.WithSkillNames(value, descendants), types.WithLimit(limit)}, nil + return append(base, types.WithSkillNames(value, descendants)), nil case types.LabelTypeDomain: - return []types.FilterOption{types.WithDomainNames(value, descendants), types.WithLimit(limit)}, nil + return append(base, types.WithDomainNames(value, descendants)), nil case types.LabelTypeModule: - return []types.FilterOption{types.WithModuleNames(value, descendants), types.WithLimit(limit)}, nil + return append(base, types.WithModuleNames(value, descendants)), nil case types.LabelTypeLocator: - return []types.FilterOption{types.WithLocatorTypes(value), types.WithLimit(limit)}, nil + return append(base, types.WithLocatorTypes(value)), nil case types.LabelTypeUnknown: return nil, fmt.Errorf("unknown query type %q", query.Type) default: diff --git a/server/routing/rpc/query_records_test.go b/server/routing/rpc/query_records_test.go index ec40f7eb9..7a75cd989 100644 --- a/server/routing/rpc/query_records_test.go +++ b/server/routing/rpc/query_records_test.go @@ -340,6 +340,45 @@ func TestQueryRecordsRejectsNilRequest(t *testing.T) { assert.Equal(t, codes.InvalidArgument, status.Code(err)) } +// A peer reaches this handler through one advertised label, so anything the +// node has not published must stay invisible to it β€” otherwise a single shared +// skill would expose private pushes and ingested replicas. +func TestQueryRecordsServesOnlyPublishedRecords(t *testing.T) { + t.Parallel() + + published := map[string]bool{"published": true, "private": false} + + db := &fakeQueryDB{ + respond: func(filters *types.RecordFilters) ([]string, error) { + cids := make([]string, 0, len(published)) + + for cid, isPublished := range published { + if filters.Published != nil && *filters.Published != isPublished { + continue + } + + cids = append(cids, cid) + } + + return cids, nil + }, + labels: map[string][]types.Label{ + "published": {"/skills/AI"}, + "private": {"/skills/AI"}, + }, + } + + client, serverID := newConnectedServices(t, db) + + matches, err := collect(t, client, serverID, QueryRecordsRequest{ + Queries: []RecordQuery{{Type: "skills", Value: "AI"}}, + }) + require.NoError(t, err) + + require.Len(t, matches, 1) + assert.Equal(t, "published", matches[0].Cid) +} + func TestQueryFiltersExpandsHierarchicalNamespaces(t *testing.T) { t.Parallel() @@ -379,6 +418,8 @@ func TestQueryFiltersExpandsHierarchicalNamespaces(t *testing.T) { assert.Equal(t, []string{"AI", "AI/*"}, tt.expected(filters)) assert.Equal(t, 10, filters.Limit) + require.NotNil(t, filters.Published) + assert.True(t, *filters.Published) }) } } @@ -397,6 +438,8 @@ func TestQueryFiltersMatchesLocatorsExactly(t *testing.T) { assert.Equal(t, []string{"docker-image"}, filters.LocatorTypes) assert.Empty(t, filters.SkillNames) + require.NotNil(t, filters.Published) + assert.True(t, *filters.Published) } func TestQueryFiltersRejectsBadQueries(t *testing.T) {