Merge pull request #83 from kcp-dev/multi-refs

Allow references to select multiple values

Author: kcp-ci-bot

docs/content/publish-resources/index.md

@@ -303,20 +303,23 @@ existing" and not create an error.
 
 #### References
 
-A reference is a JSONPath-like expression that are evaluated on both sides of the synchronization.
-You configure a single path expression (like `spec.secretName`) and the sync agent will evaluate it
-in the original primary object (in kcp) and again in the copied primary object (on the service
-cluster). Since the primary object has already been mutated, the `spec.secretName` is already
-rewritten/adjusted to work on the service cluster (for example it was changed from `my-secret` to
-`jk23h4wz47329rz2r72r92-secret` on the service cluster side). By doing it this way, admins only have
-to think about mutations and rewrites once (when configuring the primary object in the
-PublishedResource) and the path will yield 2 ready to use values (`my-secret` and the computed value).
-
-The value selected by the path expression must be a string (or number, but it will be coalesced into
-a string) and can then be further adjusted by applying a regular expression to it.
-
-References can only ever select one related object. Their upside is that they are simple to understand
-and easy to use, but require a "link" in the primary object that would point to the related object.
+A reference is a JSONPath-like expression (more precisely, it follows the [gjson syntax](https://github.com/tidwall/gjson?tab=readme-ov-file#path-syntax))
+that are evaluated on both sides of the synchronization. You configure a single path expression
+(like `spec.secretName`) and the sync agent will evaluate it in the original primary object (in kcp)
+and again in the copied primary object (on the service cluster). Since the primary object has already
+been mutated, the `spec.secretName` is already rewritten/adjusted to work on the service cluster
+(for example it was changed from `my-secret` to `jk23h4wz47329rz2r72r92-secret` on the service
+cluster side). By doing it this way, admins only have to think about mutations and rewrites once
+(when configuring the primary object in the PublishedResource) and the path will yield 2 ready to
+use values (`my-secret` and the computed value).
+
+References can either return a single scalar (strings or integers that will be auto-converted to a
+string) (like in `spec.secretName`) or a list of strings/numbers (like `spec.users.#.name`). A
+reference must return the same number of items on both the local and remote object, otherwise the
+agent will not be able to map local related names to remote related names correctly.
+
+A regular expression can be configured to be applied to each found value (i.e. if the reference returns
+a list of values, the regular expression is applied to each individual value).
 
 Here's an example on how to use references to locate the related object.
 

docs/generators/crd-ref/crd.template.md

@@ -4,8 +4,8 @@ description: |
 {{- if .Description }}
 {{ .Description | indent 2 }}
 {{- else }}
-  Custom resource definition (CRD) schema reference page for the {{ .Title }} 
-  resource ({{ .NamePlural }}.{{ .Group }}), as part of the Giant Swarm 
+  Custom resource definition (CRD) schema reference page for the {{ .Title }}
+  resource ({{ .NamePlural }}.{{ .Group }}), as part of the Giant Swarm
   Management API documentation.
 {{- end }}
 weight: {{ .Weight }}
@@ -78,7 +78,7 @@ This CRD is being replaced by <a href="../{{ .FullName }}/">{{ .ShortName }}</a>
 </div>
 {{with .Description}}
 <div class="property-description">
-{{.|markdown}}
+{% raw %}{{.|markdown}}{% endraw %}
 </div>
 {{end}}
 </div>

internal/sync/syncer_related.go

@@ -251,27 +251,25 @@ func resolveRelatedResourceObjects(relatedOrigin, relatedDest syncSide, relRes s
 func resolveRelatedResourceOriginNamespaces(relatedOrigin, relatedDest syncSide, origin syncagentv1alpha1.RelatedResourceOrigin, spec syncagentv1alpha1.RelatedResourceObjectSpec) (map[string]string, error) {
 	switch {
 	case spec.Reference != nil:
-		originNamespace, err := resolveObjectReference(relatedOrigin.object, *spec.Reference)
+		originNamespaces, err := resolveObjectReference(relatedOrigin.object, *spec.Reference)
 		if err != nil {
 			return nil, err
 		}
 
-		if originNamespace == "" {
+		if len(originNamespaces) == 0 {
 			return nil, nil
 		}
 
-		destNamespace, err := resolveObjectReference(relatedDest.object, *spec.Reference)
+		destNamespaces, err := resolveObjectReference(relatedDest.object, *spec.Reference)
 		if err != nil {
 			return nil, err
 		}
 
-		if destNamespace == "" {
-			return nil, nil
+		if len(destNamespaces) != len(originNamespaces) {
+			return nil, fmt.Errorf("cannot sync related resources: found %d namespaces on the origin, but %d on the destination side", len(originNamespaces), len(destNamespaces))
 		}
 
-		return map[string]string{
-			originNamespace: destNamespace,
-		}, nil
+		return mapSlices(originNamespaces, destNamespaces), nil
 
 	case spec.Selector != nil:
 		namespaces := &corev1.NamespaceList{}
@@ -327,6 +325,22 @@ func resolveRelatedResourceOriginNamespaces(relatedOrigin, relatedDest syncSide,
 	}
 }
 
+func mapSlices(a, b []string) map[string]string {
+	mapping := map[string]string{}
+	for i, aItem := range a {
+		bItem := b[i]
+
+		// ignore any origin<->dest pair where either of the sides is empty
+		if bItem == "" || aItem == "" {
+			continue
+		}
+
+		mapping[aItem] = bItem
+	}
+
+	return mapping
+}
+
 func resolveRelatedResourceObjectsInNamespaces(relatedOrigin, relatedDest syncSide, relRes syncagentv1alpha1.RelatedResourceSpec, spec syncagentv1alpha1.RelatedResourceObjectSpec, namespaceMap map[string]string) ([]resolvedObject, error) {
 	result := []resolvedObject{}
 
@@ -368,27 +382,25 @@ func resolveRelatedResourceObjectsInNamespaces(relatedOrigin, relatedDest syncSi
 func resolveRelatedResourceObjectsInNamespace(relatedOrigin, relatedDest syncSide, relRes syncagentv1alpha1.RelatedResourceSpec, spec syncagentv1alpha1.RelatedResourceObjectSpec, namespace string) (map[string]string, error) {
 	switch {
 	case spec.Reference != nil:
-		originName, err := resolveObjectReference(relatedOrigin.object, *spec.Reference)
+		originNames, err := resolveObjectReference(relatedOrigin.object, *spec.Reference)
 		if err != nil {
 			return nil, err
 		}
 
-		if originName == "" {
+		if len(originNames) == 0 {
 			return nil, nil
 		}
 
-		destName, err := resolveObjectReference(relatedDest.object, *spec.Reference)
+		destNames, err := resolveObjectReference(relatedDest.object, *spec.Reference)
 		if err != nil {
 			return nil, err
 		}
 
-		if destName == "" {
-			return nil, nil
+		if len(destNames) != len(originNames) {
+			return nil, fmt.Errorf("cannot sync related resources: found %d names on the origin, but %d on the destination side", len(originNames), len(destNames))
 		}
 
-		return map[string]string{
-			originName: destName,
-		}, nil
+		return mapSlices(originNames, destNames), nil
 
 	case spec.Selector != nil:
 		originObjects := &unstructured.UnstructuredList{}
@@ -447,34 +459,44 @@ func resolveRelatedResourceObjectsInNamespace(relatedOrigin, relatedDest syncSid
 	}
 }
 
-func resolveObjectReference(object *unstructured.Unstructured, ref syncagentv1alpha1.RelatedResourceObjectReference) (string, error) {
+func resolveObjectReference(object *unstructured.Unstructured, ref syncagentv1alpha1.RelatedResourceObjectReference) ([]string, error) {
 	data, err := object.MarshalJSON()
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	return resolveReference(data, ref)
 }
 
-func resolveReference(jsonData []byte, ref syncagentv1alpha1.RelatedResourceObjectReference) (string, error) {
-	gval := gjson.Get(string(jsonData), ref.Path)
-	if !gval.Exists() {
-		return "", nil
+func resolveReference(jsonData []byte, ref syncagentv1alpha1.RelatedResourceObjectReference) ([]string, error) {
+	result := gjson.Get(string(jsonData), ref.Path)
+	if !result.Exists() {
+		return nil, nil
 	}
 
-	// this does apply some coalescing, like turning numbers into strings
-	strVal := gval.String()
+	var values []string
+	if result.IsArray() {
+		for _, elem := range result.Array() {
+			values = append(values, strings.TrimSpace(elem.String()))
+		}
+	} else {
+		values = append(values, strings.TrimSpace(result.String()))
+	}
 
 	if re := ref.Regex; re != nil {
 		var err error
 
-		strVal, err = applyRegularExpression(strVal, *re)
-		if err != nil {
-			return "", err
+		for i, value := range values {
+			value, err = applyRegularExpression(value, *re)
+			if err != nil {
+				return nil, err
+			}
+
+			values[i] = value
 		}
 	}
 
-	return strVal, nil
+	return values, nil
 }
 
 // applyTemplate is used after a label selector has been applied and a list of namespaces or objects

test/crds/backup.go

@@ -28,6 +28,11 @@ type Backup struct {
 }
 
 type BackupSpec struct {
-	Source      string `json:"source"`
-	Destination string `json:"destination"`
+	Source      string       `json:"source"`
+	Destination string       `json:"destination"`
+	Items       []BackupItem `json:"items,omitempty"`
+}
+
+type BackupItem struct {
+	Name string `json:"name"`
 }

test/crds/backup.yaml

@@ -25,3 +25,10 @@ spec:
                   type: string
                 destination:
                   type: string
+                items:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      name:
+                        type: string