feat: code reference API

hack/delete-e2e-data-mysql/command.go

@@ -47,6 +47,7 @@ var (
 		{table: "tag", targetField: "name"},
 		{table: "ops_progressive_rollout", targetField: "feature_id"},
 		{table: "flag_trigger", targetField: "description"},
+		{table: "code_reference", targetField: "feature_id"},
 		{table: "feature", targetField: "id"},
 		{table: "api_key", targetField: "name"},
 	}

manifests/bucketeer/charts/api/templates/deployment.yaml

@@ -88,6 +88,8 @@ spec:
               value: "{{ .Values.env.accountService }}"
             - name: BUCKETEER_API_PUSH_SERVICE
               value: "{{ .Values.env.pushService }}"
+            - name: BUCKETEER_API_CODE_REF_SERVICE
+              value: "{{ .Values.env.codeRefService }}"
             - name: BUCKETEER_API_PORT
               value: "{{ .Values.env.port }}"
             - name: BUCKETEER_API_METRICS_PORT

manifests/bucketeer/charts/api/templates/envoy-configmap.yaml

@@ -285,6 +285,18 @@ data:
                               retry_policy:
                                 retry_on: 5xx
                                 num_retries: 3
+                          - match:
+                              headers:
+                                - name: content-type
+                                  string_match:
+                                    exact: application/grpc
+                              prefix: /bucketeer.coderef.CodeReferenceService
+                            route:
+                              cluster: web
+                              timeout: 15s
+                              retry_policy:
+                                retry_on: 5xx
+                                num_retries: 3
               transport_socket:
                 name: envoy.transport_sockets.tls
                 typed_config:

manifests/bucketeer/charts/web/templates/deployment.yaml

@@ -86,6 +86,9 @@ spec:
         - name: envoy-batch-descriptor-secret
           secret:
             secretName: {{ template "web.fullname" . }}-batch
+        - name: envoy-code-reference-descriptor-secret
+          secret:
+            secretName: {{ template "web.fullname" . }}-code-reference
       {{- if .Values.serviceAccount.annotations }}
       serviceAccountName: {{ template "web.fullname" . }}
       {{- end }}
@@ -171,6 +174,8 @@ spec:
               value: "{{ .Values.env.webConsoleServicePort }}"
             - name: BUCKETEER_WEB_DASHBOARD_SERVICE_PORT
               value: "{{ .Values.env.dashboardServicePort }}"
+            - name: BUCKETEER_WEB_CODE_REFERENCE_SERVICE_PORT
+              value: "{{ .Values.env.coderefServicePort }}"
             - name: BUCKETEER_WEB_ACCOUNT_SERVICE
               value: "{{ .Values.env.accountService }}"
             - name: BUCKETEER_WEB_AUTH_SERVICE
@@ -185,6 +190,8 @@ spec:
               value: "{{ .Values.env.featureService }}"
             - name: BUCKETEER_WEB_AUTO_OPS_SERVICE
               value: "{{ .Values.env.autoOpsService }}"
+            - name: BUCKETEER_WEB_CODE_REFERENCE_SERVICE
+              value: "{{ .Values.env.coderefService }}"
             - name: BUCKETEER_WEB_TIMEZONE
               value: "{{ .Values.env.timezone }}"
             - name: BUCKETEER_WEB_METRICS_PORT
@@ -315,6 +322,9 @@ spec:
             - name: envoy-batch-descriptor-secret
               mountPath: /usr/local/batch-secret
               readOnly: true
+            - name: envoy-code-reference-descriptor-secret
+              mountPath: /usr/local/code-reference-secret
+              readOnly: true
             - name: service-cert-secret
               mountPath: /usr/local/certs/service
               readOnly: true

manifests/bucketeer/charts/web/templates/envoy-configmap.yaml

@@ -403,6 +403,38 @@ data:
               explicit_http_config:
                 http2_protocol_options: {}
           ignore_health_on_host_removal: true
+        - name: coderef
+          type: strict_dns
+          connect_timeout: 5s
+          dns_lookup_family: V4_ONLY
+          lb_policy: {{ .Values.envoy.lbPolicy }}
+          load_assignment:
+            cluster_name: coderef
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: localhost
+                          port_value: 9105
+          transport_socket:
+            name: envoy.transport_sockets.tls
+            typed_config:
+              '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+              common_tls_context:
+                alpn_protocols:
+                  - h2
+                tls_certificates:
+                  - certificate_chain:
+                      filename: /usr/local/certs/service/tls.crt
+                    private_key:
+                      filename: /usr/local/certs/service/tls.key
+          typed_extension_protocol_options:
+            envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+              '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+              explicit_http_config:
+                http2_protocol_options: {}
+          ignore_health_on_host_removal: true
         - name: web_console
           type: strict_dns
           connect_timeout: 5s
@@ -647,6 +679,14 @@ data:
                                 retry_policy:
                                   retry_on: 5xx
                                   num_retries: 3
+                            - match:
+                                prefix: /bucketeer.coderef.CodeReferenceService
+                              route:
+                                cluster: coderef
+                                timeout: 15s
+                                retry_policy:
+                                  retry_on: 5xx
+                                  num_retries: 3
                             - match:
                                 prefix: /
                                 headers:
@@ -811,6 +851,15 @@ data:
                             - bucketeer.batch.BatchService
                           print_options:
                             always_print_primitive_fields: true
+                      - name: envoy.filters.http.grpc_json_transcoder
+                        typed_config:
+                          "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder
+                          auto_mapping: true
+                          proto_descriptor: /usr/local/code-reference-secret/code_reference_proto_descriptor.pb
+                          services:
+                            - bucketeer.coderef.CodeReferenceService
+                          print_options:
+                            always_print_primitive_fields: true
                       - name: envoy.filters.http.router
                         typed_config:
                           "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
@@ -940,6 +989,14 @@ data:
                                 retry_policy:
                                   retry_on: 5xx
                                   num_retries: 3
+                            - match:
+                                prefix: /bucketeer.coderef.CodeReferenceService
+                              route:
+                                cluster: coderef
+                                timeout: 15s
+                                retry_policy:
+                                  retry_on: 5xx
+                                  num_retries: 3
                             - match:
                                 prefix: /v3
                               route:
@@ -1091,6 +1148,14 @@ data:
                                   num_retries: 3
                                   retry_on: 5xx
                                 timeout: 15s
+                            - match:
+                                prefix: /bucketeer.coderef.CodeReferenceService
+                              route:
+                                cluster: coderef
+                                timeout: 15s
+                                retry_policy:
+                                  num_retries: 3
+                                  retry_on: 5xx
                     stat_prefix: egress_http
                     stream_idle_timeout: 300s
               transport_socket:

manifests/bucketeer/charts/web/templates/secret.yaml

@@ -157,6 +157,19 @@ data:
 ---
 apiVersion: v1
 kind: Secret
+metadata:
+  name: {{ template "web.fullname" . }}-code-reference
+  labels:
+    app: {{ template "web.name" . }}
+    chart: {{ template "web.chart" . }}
+    release: {{ template "web.fullname" . }}
+    heritage: {{ .Release.Service }}
+type: Opaque
+data:
+  code_reference_proto_descriptor.pb: {{ required "Envoy code reference descriptor is required" .Values.envoy.coderefDescriptor | quote }}
+---
+apiVersion: v1
+kind: Secret
 metadata:
   name: {{ template "web.fullname" . }}-batch
   labels:

migration/mysql/20250122261735_add_code_ref_table.sql

@@ -0,0 +1,21 @@
+CREATE TABLE code_reference (
+    id varchar(255) NOT NULL,
+    feature_id VARCHAR(255) NOT NULL,
+    file_path VARCHAR(512) NOT NULL,
+    line_number INT NOT NULL,
+    code_snippet TEXT NOT NULL,
+    content_hash VARCHAR(64) NOT NULL,
+    aliases JSON,
+    repository_name VARCHAR(255) NOT NULL,
+    repository_owner VARCHAR(255) NOT NULL,
+    repository_type TINYINT NOT NULL COMMENT '0:UNSPECIFIED, 1:GITHUB, 2:GITLAB, 3:BITBUCKET, 4:CUSTOM',
+    repository_branch VARCHAR(255) NOT NULL,
+    commit_hash VARCHAR(40) NOT NULL,
+    environment_id VARCHAR(255) NOT NULL,
+    created_at BIGINT NOT NULL,
+    updated_at BIGINT NOT NULL,
+    PRIMARY KEY (id),
+    INDEX idx_file_path (file_path),
+    CONSTRAINT foreign_code_references_feature FOREIGN KEY (feature_id, environment_id) 
+        REFERENCES feature (id, environment_id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

migration/mysql/atlas.sum

@@ -1,4 +1,4 @@
-h1:1gFvYJwtEn7dAVsV68Nzh0bXhPLPHh6OHmeV2U7vjQU=
+h1:VXTmzqYh41HnDCRZyYb9OvaSQx2m1+GLavQcpqgGCjQ=
 20240626022133_initialization.sql h1:u9qmPkwWX7PN92qEcDDfR92lrMpwadQSMxUJgv6LjQ0=
 20240708065726_update_audit_log_table.sql h1:k7gK8Njv1yHMsYXAQtSgMaSbXy4lxyZ9MPzbJyMyyoM=
 20240815043128_update_auto_ops_rule_table.sql h1:6ib+XfS1uu9AUO3qESvkpUfOu3qUsLwHm9KHcrGEz0E=
@@ -23,3 +23,4 @@ h1:1gFvYJwtEn7dAVsV68Nzh0bXhPLPHh6OHmeV2U7vjQU=
 20250115040347_update_account_v2_table.sql h1:nEzBj41mgpXbQGgbNaIZ9thMqvtfamLO9NHISpEwMm8=
 20250116054703_remove_environment_namespace_column.sql h1:UNiN/syYAs8nUQOkMw3RRvDQBQU/kKYv7B3XBhL6qPM=
 20250116060049_update_goal_table.sql h1:zEhXMf2hntdk2Im9klR76vx6PEAh+cJwk3fKgm2gGPs=
+20250122261735_add_code_ref_table.sql h1:ZqE6Ekn3PnhBFdpuczJYo5KjwTvcgeqSnkuLCNeHoCA=

pkg/api/api/api_grpc.go

@@ -32,6 +32,7 @@ import (
 	accountclient "github.com/bucketeer-io/bucketeer/pkg/account/client"
 	"github.com/bucketeer-io/bucketeer/pkg/cache"
 	cachev3 "github.com/bucketeer-io/bucketeer/pkg/cache/v3"
+	coderefclient "github.com/bucketeer-io/bucketeer/pkg/coderef/client"
 	featureclient "github.com/bucketeer-io/bucketeer/pkg/feature/client"
 	featuredomain "github.com/bucketeer-io/bucketeer/pkg/feature/domain"
 	"github.com/bucketeer-io/bucketeer/pkg/log"
@@ -142,6 +143,7 @@ type grpcGatewayService struct {
 	featureClient          featureclient.Client
 	accountClient          accountclient.Client
 	pushClient             pushclient.Client
+	codeRefClient          coderefclient.Client
 	goalPublisher          publisher.Publisher
 	evaluationPublisher    publisher.Publisher
 	userPublisher          publisher.Publisher
@@ -157,6 +159,7 @@ func NewGrpcGatewayService(
 	featureClient featureclient.Client,
 	accountClient accountclient.Client,
 	pushClient pushclient.Client,
+	codeRefClient coderefclient.Client,
 	gp publisher.Publisher,
 	ep publisher.Publisher,
 	up publisher.Publisher,
@@ -174,6 +177,7 @@ func NewGrpcGatewayService(
 		featureClient:          featureClient,
 		accountClient:          accountClient,
 		pushClient:             pushClient,
+		codeRefClient:          codeRefClient,
 		goalPublisher:          gp,
 		evaluationPublisher:    ep,
 		userPublisher:          up,

pkg/api/api/api_grpc_test.go

@@ -37,6 +37,7 @@ import (
 	accountclientmock "github.com/bucketeer-io/bucketeer/pkg/account/client/mock"
 	"github.com/bucketeer-io/bucketeer/pkg/cache"
 	cachev3mock "github.com/bucketeer-io/bucketeer/pkg/cache/v3/mock"
+	coderefclientmock "github.com/bucketeer-io/bucketeer/pkg/coderef/client/mock"
 	featureclientmock "github.com/bucketeer-io/bucketeer/pkg/feature/client/mock"
 	"github.com/bucketeer-io/bucketeer/pkg/log"
 	"github.com/bucketeer-io/bucketeer/pkg/metrics"
@@ -93,7 +94,7 @@ func TestWithLogger(t *testing.T) {
 
 func TestNewGrpcGatewayService(t *testing.T) {
 	t.Parallel()
-	g := NewGrpcGatewayService(nil, nil, nil, nil, nil, nil, nil)
+	g := NewGrpcGatewayService(nil, nil, nil, nil, nil, nil, nil, nil)
 	assert.IsType(t, &grpcGatewayService{}, g)
 }
 
@@ -3959,6 +3960,7 @@ func newGrpcGatewayServiceWithMock(t *testing.T, mockController *gomock.Controll
 		featureClient:          featureclientmock.NewMockClient(mockController),
 		accountClient:          accountclientmock.NewMockClient(mockController),
 		pushClient:             pushclientmock.NewMockClient(mockController),
+		codeRefClient:          coderefclientmock.NewMockClient(mockController),
 		goalPublisher:          publishermock.NewMockPublisher(mockController),
 		userPublisher:          publishermock.NewMockPublisher(mockController),
 		evaluationPublisher:    publishermock.NewMockPublisher(mockController),