Skip to content

Commit ea16ee7

Browse files
authored
Refactor sync mgr (#59)
* filtering package Signed-off-by: Daniele Martinoli <[email protected]> * moved config package under cmd/thv-registry-api/pkg folder. added few missing configs (more are needed!) Signed-off-by: Daniele Martinoli <[email protected]> * updated User-Agent header Signed-off-by: Daniele Martinoli <[email protected]> * SyncStatus types Signed-off-by: Daniele Martinoli <[email protected]> * fixed ginkgo tests Signed-off-by: Daniele Martinoli <[email protected]> * replaced unnecessary imports of thv-operator packages Signed-off-by: Daniele Martinoli <[email protected]> * moved config package under cmd/thv-registry-api/pkg folder. added few missing configs (more are needed!) Signed-off-by: Daniele Martinoli <[email protected]> * fixed ginkgo tests Signed-off-by: Daniele Martinoli <[email protected]> * rebase issues Signed-off-by: Daniele Martinoli <[email protected]> * updated package docs Signed-off-by: Daniele Martinoli <[email protected]> --------- Signed-off-by: Daniele Martinoli <[email protected]>
1 parent 5533fa2 commit ea16ee7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+3573
-1290
lines changed

cmd/thv-registry-api/app/commands.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@ import (
77

88
"github.com/spf13/cobra"
99
"github.com/spf13/viper"
10-
11-
"github.com/stacklok/toolhive/pkg/logger"
1210
"github.com/stacklok/toolhive-registry-server/pkg/versions"
11+
"github.com/stacklok/toolhive/pkg/logger"
1312
)
1413

1514
var rootCmd = &cobra.Command{

go.mod

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ require (
2727
github.com/Masterminds/semver/v3 v3.4.0 // indirect
2828
github.com/Microsoft/go-winio v0.6.2 // indirect
2929
github.com/ProtonMail/go-crypto v1.1.6 // indirect
30+
github.com/beorn7/perks v1.0.1 // indirect
3031
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
32+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
3133
github.com/cloudflare/circl v1.6.1 // indirect
3234
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
3335
github.com/cyphar/filepath-securejoin v0.4.1 // indirect
@@ -62,6 +64,7 @@ require (
6264
github.com/godbus/dbus/v5 v5.1.0 // indirect
6365
github.com/gofrs/flock v0.13.0 // indirect
6466
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
67+
github.com/google/btree v1.1.3 // indirect
6568
github.com/google/gnostic-models v0.7.0 // indirect
6669
github.com/google/go-cmp v0.7.0 // indirect
6770
github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect
@@ -81,6 +84,10 @@ require (
8184
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
8285
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
8386
github.com/pjbgf/sha1cd v0.3.2 // indirect
87+
github.com/prometheus/client_golang v1.23.2 // indirect
88+
github.com/prometheus/client_model v0.6.2 // indirect
89+
github.com/prometheus/common v0.66.1 // indirect
90+
github.com/prometheus/procfs v0.17.0 // indirect
8491
github.com/russross/blackfriday/v2 v2.1.0 // indirect
8592
github.com/sagikazarmark/locafero v0.11.0 // indirect
8693
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
@@ -116,11 +123,13 @@ require (
116123
golang.org/x/time v0.14.0 // indirect
117124
golang.org/x/tools v0.37.0 // indirect
118125
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
126+
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
119127
google.golang.org/protobuf v1.36.9 // indirect
120128
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
121129
gopkg.in/inf.v0 v0.9.1 // indirect
122130
gopkg.in/warnings.v0 v0.1.2 // indirect
123131
gopkg.in/yaml.v2 v2.4.0 // indirect
132+
k8s.io/apiextensions-apiserver v0.34.1 // indirect
124133
k8s.io/klog/v2 v2.130.1 // indirect
125134
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
126135
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect

go.sum

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf
4848
github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
4949
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
5050
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
51+
github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
52+
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
5153
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
5254
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
5355
github.com/extism/go-sdk v1.7.0 h1:yHbSa2JbcF60kjGsYiGEOcClfbknqCJchyh9TRibFWo=
@@ -126,11 +128,15 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
126128
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
127129
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
128130
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
131+
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
132+
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
129133
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
130134
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
131135
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
132136
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
133137
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
138+
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
139+
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
134140
github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=
135141
github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
136142
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
@@ -159,13 +165,17 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4
159165
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
160166
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
161167
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
168+
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
169+
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
162170
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
163171
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
164172
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
165173
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
166174
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
167175
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
168176
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
177+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
178+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
169179
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
170180
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
171181
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
@@ -346,6 +356,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
346356
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
347357
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
348358
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
359+
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
360+
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
349361
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
350362
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
351363
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/api/v0/routes.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/go-chi/chi/v5"
1111
"gopkg.in/yaml.v3"
1212

13-
docs "github.com/stacklok/toolhive-registry-server/docs/thv-registry-api"
13+
"github.com/stacklok/toolhive-registry-server/cmd/thv-registry-api/docs"
1414
"github.com/stacklok/toolhive-registry-server/internal/service"
1515
"github.com/stacklok/toolhive/pkg/logger"
1616
"github.com/stacklok/toolhive/pkg/registry"

pkg/config/config.go

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,80 @@ import (
88
"gopkg.in/yaml.v3"
99
)
1010

11+
const (
12+
// RegistrySourceTypeConfigMap is the type for registry data stored in ConfigMaps
13+
SourceTypeConfigMap = "configmap"
14+
15+
// RegistrySourceTypeGit is the type for registry data stored in Git repositories
16+
SourceTypeGit = "git"
17+
18+
// SourceTypeAPI is the type for registry data fetched from API endpoints
19+
SourceTypeAPI = "api"
20+
)
21+
22+
const (
23+
// RegistryFormatToolHive is the native ToolHive registry format
24+
SourceFormatToolHive = "toolhive"
25+
26+
// RegistryFormatUpstream is the upstream MCP registry format
27+
SourceFormatUpstream = "upstream"
28+
)
29+
1130
// ConfigLoader defines the interface for loading configuration
1231
type ConfigLoader interface {
1332
LoadConfig(path string) (*Config, error)
1433
}
1534

1635
// Config represents the root configuration structure
1736
type Config struct {
18-
Source SourceConfig `yaml:"source"`
19-
SyncPolicy SyncPolicyConfig `yaml:"syncPolicy"`
20-
Filter FilterConfig `yaml:"filter"`
37+
Source SourceConfig `yaml:"source"`
38+
SyncPolicy *SyncPolicyConfig `yaml:"syncPolicy,omitempty"`
39+
Filter *FilterConfig `yaml:"filter,omitempty"`
2140
}
2241

2342
// SourceConfig defines the data source configuration
2443
type SourceConfig struct {
2544
Type string `yaml:"type"`
45+
Format string `yaml:"format"`
2646
ConfigMap *ConfigMapConfig `yaml:"configmap,omitempty"`
47+
Git *GitConfig `yaml:"git,omitempty"`
48+
API *APIConfig `yaml:"api,omitempty"`
2749
}
2850

2951
// ConfigMapConfig defines Kubernetes ConfigMap source settings
3052
type ConfigMapConfig struct {
31-
Name string `yaml:"name"`
53+
Namespace string `yaml:"namespace"`
54+
Name string `yaml:"name"`
55+
Key string `yaml:"key,omitempty"`
56+
}
57+
58+
// GitConfig defines Git source settings
59+
type GitConfig struct {
60+
// Repository is the Git repository URL (HTTP/HTTPS/SSH)
61+
Repository string `yaml:"repository"`
62+
63+
// Branch is the Git branch to use (mutually exclusive with Tag and Commit)
64+
Branch string `yaml:"branch,omitempty"`
65+
66+
// Tag is the Git tag to use (mutually exclusive with Branch and Commit)
67+
Tag string `yaml:"tag,omitempty"`
68+
69+
// Commit is the Git commit SHA to use (mutually exclusive with Branch and Tag)
70+
Commit string `yaml:"commit,omitempty"`
71+
72+
// Path is the path to the registry file within the repository
73+
Path string `yaml:"path,omitempty"`
74+
}
75+
76+
// APIConfig defines API source configuration for ToolHive Registry APIs
77+
type APIConfig struct {
78+
// Endpoint is the base API URL (without path)
79+
// The source handler will append the appropriate paths, for instance:
80+
// - /v0/servers - List all servers (single response, no pagination)
81+
// - /v0/servers/{name} - Get specific server (future)
82+
// - /v0/info - Get registry metadata (future)
83+
// Example: "http://my-registry-api.default.svc.cluster.local/api"
84+
Endpoint string `yaml:"endpoint"`
3285
}
3386

3487
// SyncPolicyConfig defines synchronization settings
@@ -38,7 +91,14 @@ type SyncPolicyConfig struct {
3891

3992
// FilterConfig defines filtering rules for registry entries
4093
type FilterConfig struct {
41-
Tags TagFilterConfig `yaml:"tags"`
94+
Names *NameFilterConfig `yaml:"names,omitempty"`
95+
Tags *TagFilterConfig `yaml:"tags,omitempty"`
96+
}
97+
98+
// NameFilterConfig defines name-based filtering
99+
type NameFilterConfig struct {
100+
Include []string `yaml:"include,omitempty"`
101+
Exclude []string `yaml:"exclude,omitempty"`
42102
}
43103

44104
// TagFilterConfig defines tag-based filtering

pkg/config/config_test.go

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ filter:
3636
Name: "minimal-registry-data",
3737
},
3838
},
39-
SyncPolicy: SyncPolicyConfig{
39+
SyncPolicy: &SyncPolicyConfig{
4040
Interval: "30m",
4141
},
42-
Filter: FilterConfig{
43-
Tags: TagFilterConfig{
42+
Filter: &FilterConfig{
43+
Tags: &TagFilterConfig{
4444
Include: []string{"database", "production"},
4545
Exclude: []string{"experimental", "deprecated", "beta"},
4646
},
@@ -67,11 +67,11 @@ filter:
6767
Name: "test-registry",
6868
},
6969
},
70-
SyncPolicy: SyncPolicyConfig{
70+
SyncPolicy: &SyncPolicyConfig{
7171
Interval: "1h",
7272
},
73-
Filter: FilterConfig{
74-
Tags: TagFilterConfig{
73+
Filter: &FilterConfig{
74+
Tags: &TagFilterConfig{
7575
Include: []string{},
7676
Exclude: []string{},
7777
},
@@ -97,11 +97,11 @@ filter:
9797
Name: "prod-registry",
9898
},
9999
},
100-
SyncPolicy: SyncPolicyConfig{
100+
SyncPolicy: &SyncPolicyConfig{
101101
Interval: "15m",
102102
},
103-
Filter: FilterConfig{
104-
Tags: TagFilterConfig{
103+
Filter: &FilterConfig{
104+
Tags: &TagFilterConfig{
105105
Include: []string{"api", "backend", "frontend"},
106106
Exclude: nil,
107107
},
@@ -127,11 +127,11 @@ filter:
127127
Name: "dev-registry",
128128
},
129129
},
130-
SyncPolicy: SyncPolicyConfig{
130+
SyncPolicy: &SyncPolicyConfig{
131131
Interval: "5m",
132132
},
133-
Filter: FilterConfig{
134-
Tags: TagFilterConfig{
133+
Filter: &FilterConfig{
134+
Tags: &TagFilterConfig{
135135
Include: nil,
136136
Exclude: []string{"test", "debug", "experimental"},
137137
},
@@ -202,11 +202,11 @@ func TestConfigStructure(t *testing.T) {
202202
Name: "test-configmap",
203203
},
204204
},
205-
SyncPolicy: SyncPolicyConfig{
205+
SyncPolicy: &SyncPolicyConfig{
206206
Interval: "45m",
207207
},
208-
Filter: FilterConfig{
209-
Tags: TagFilterConfig{
208+
Filter: &FilterConfig{
209+
Tags: &TagFilterConfig{
210210
Include: []string{"prod", "stable"},
211211
Exclude: []string{"beta", "alpha"},
212212
},
@@ -257,7 +257,7 @@ func TestConfigValidate(t *testing.T) {
257257
Name: "test-configmap",
258258
},
259259
},
260-
SyncPolicy: SyncPolicyConfig{
260+
SyncPolicy: &SyncPolicyConfig{
261261
Interval: "30m",
262262
},
263263
},
@@ -271,7 +271,7 @@ func TestConfigValidate(t *testing.T) {
271271
Name: "test",
272272
},
273273
},
274-
SyncPolicy: SyncPolicyConfig{
274+
SyncPolicy: &SyncPolicyConfig{
275275
Interval: "30m",
276276
},
277277
},
@@ -284,7 +284,7 @@ func TestConfigValidate(t *testing.T) {
284284
Source: SourceConfig{
285285
Type: "configmap",
286286
},
287-
SyncPolicy: SyncPolicyConfig{
287+
SyncPolicy: &SyncPolicyConfig{
288288
Interval: "30m",
289289
},
290290
},
@@ -298,7 +298,7 @@ func TestConfigValidate(t *testing.T) {
298298
Type: "configmap",
299299
ConfigMap: &ConfigMapConfig{},
300300
},
301-
SyncPolicy: SyncPolicyConfig{
301+
SyncPolicy: &SyncPolicyConfig{
302302
Interval: "30m",
303303
},
304304
},
@@ -314,7 +314,7 @@ func TestConfigValidate(t *testing.T) {
314314
Name: "test",
315315
},
316316
},
317-
SyncPolicy: SyncPolicyConfig{},
317+
SyncPolicy: &SyncPolicyConfig{},
318318
},
319319
wantErr: true,
320320
errMsg: "syncPolicy.interval is required",
@@ -328,7 +328,7 @@ func TestConfigValidate(t *testing.T) {
328328
Name: "test",
329329
},
330330
},
331-
SyncPolicy: SyncPolicyConfig{
331+
SyncPolicy: &SyncPolicyConfig{
332332
Interval: "invalid",
333333
},
334334
},

0 commit comments

Comments
 (0)