perf(controlplane): make referrer discovery independent of the caller's organization count (#3445)

Signed-off-by: Javier Rodriguez <javier@chainloop.dev>

Author: javirln

app/controlplane/pkg/biz/referrer_integration_test.go

@@ -653,3 +653,37 @@ func (s *referrerIntegrationTestSuite) SetupTest() {
 func TestReferrerIntegration(t *testing.T) {
 	suite.Run(t, new(referrerIntegrationTestSuite))
 }
+
+// Project visibility is not an optional filter: a member restricted to a subset of an
+// organization's projects must not reach a referrer outside them, and a grant naming one
+// organization must not reach a project living in another.
+func (s *referrerIntegrationTestSuite) TestGetFromRootProjectVisibility() {
+	ctx := context.Background()
+	envelope, envBytes := testEnvelope(s.T(), "testdata/attestations/with-git-subject.json")
+	attDigest, _, err := v1.SHA256(bytes.NewReader(envBytes))
+	require.NoError(s.T(), err)
+	require.NoError(s.T(), s.Referrer.ExtractAndPersist(ctx, envelope, attDigest, s.workflow1.ID.String()))
+	digest := attDigest.String()
+
+	s.Run("a caller granted the project reaches it", func() {
+		granted := map[biz.OrgID][]biz.ProjectID{s.org1UUID: {s.workflow1.ProjectID}}
+		got, _, err := s.Referrer.GetFromRoot(ctx, digest, "", []uuid.UUID{s.org1UUID}, granted, nil)
+		s.NoError(err)
+		s.Equal(digest, got.Digest)
+	})
+
+	s.Run("a caller restricted to no project in the organization does not", func() {
+		noProjects := map[biz.OrgID][]biz.ProjectID{s.org1UUID: {}}
+		_, _, err := s.Referrer.GetFromRoot(ctx, digest, "", []uuid.UUID{s.org1UUID}, noProjects, nil)
+		s.Error(err)
+	})
+
+	s.Run("a grant recorded against another organization does not", func() {
+		crossOrg := map[biz.OrgID][]biz.ProjectID{
+			s.org1UUID: {},                      // restricted here, nothing granted
+			s.org2UUID: {s.workflow1.ProjectID}, // granted here, but the project lives in org1
+		}
+		_, _, err := s.Referrer.GetFromRoot(ctx, digest, "", []uuid.UUID{s.org1UUID, s.org2UUID}, crossOrg, nil)
+		s.Error(err)
+	})
+}

app/controlplane/pkg/data/export_test.go

@@ -0,0 +1,22 @@
+//
+// Copyright 2026 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+// Internals exposed to the package's external tests. This file is only compiled during tests, so
+// it does not widen the package's API. The visibility filter is reached from data_test because
+// asserting on it against a real database needs the shared suite in biz/testhelpers, which
+// imports this package and therefore cannot be imported from inside it.
+var ReferrerVisibleToOrgsForTest = referrerVisibleToOrgs

app/controlplane/pkg/data/referrer.go

@@ -23,7 +23,6 @@ import (
 	"entgo.io/ent/dialect/sql"
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/biz"
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/data/ent"
-	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/data/ent/organization"
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/data/ent/predicate"
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/data/ent/project"
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/data/ent/projectversion"
@@ -97,10 +96,10 @@ func (r *ReferrerRepo) Save(ctx context.Context, referrers []*biz.Referrer, work
 	for _, parentRef := range referrers {
 		// This is the current item stored in DB
 		storedReferrer := storedMap[parentRef.MapID()]
-		// Iterate on the items it refer to (references)
+		// Iterate on the items it refers to (references)
 		var references []uuid.UUID
 		for _, ref := range parentRef.References {
-			// amd find it in the DB
+			// and find it in the DB
 			storedReference, ok := storedMap[ref.MapID()]
 			if !ok {
 				return fmt.Errorf("referrer %v not found", ref)
@@ -162,13 +161,8 @@ func (r *ReferrerRepo) GetFromRoot(ctx context.Context, digest string, orgIDs []
 		predicateReferrer = append(predicateReferrer, referrer.Kind(*opts.RootKind))
 	}
 
-	// Prepare the workflow query predicate
-	predicateWF := []predicate.Workflow{
-		workflow.DeletedAtIsNil(), workflow.HasOrganizationWith(organization.IDIn(orgIDs...)),
-	}
-
-	// Attach the workflow predicate
-	predicateReferrer = append(predicateReferrer, referrer.HasWorkflowsWith(predicateWF...))
+	// Attach the visibility predicate
+	predicateReferrer = append(predicateReferrer, referrerVisibleToOrgs(orgIDs))
 
 	// If a project filter is requested, attach it as a subquery predicate. An attestation root
 	// matches only when its digest is one of the attestation_digests produced by a workflow run
@@ -216,10 +210,8 @@ func (r *ReferrerRepo) GetFromRoot(ctx context.Context, digest string, orgIDs []
 
 // projectScopePredicate returns a predicate matching referrers whose digest is the attestation
 // digest of a workflow run in the requested project (and, when non-empty, version), visible to
-// the caller. The predicate compiles to a SQL subquery — no digest list is materialized in Go,
-// so the cost is independent of how many runs the project has. Postgres plans this as a
-// semi-join via the index on workflow_run.attestation_digest, which is what makes the filter
-// scale at thousands of runs per project.
+// the caller. The predicate compiles to a SQL subquery rather than materializing a digest list in
+// Go, so the request carries no per-run data however many runs the project has.
 //
 // Visibility mirrors isReferrerVisible: a run is included when its workflow's project is in the
 // caller's RBAC-visible set. visibleProjectsMap follows the existing convention — an org entry
@@ -262,30 +254,114 @@ func (r *ReferrerRepo) projectScopePredicate(projectName, version string, orgIDs
 	}
 }
 
+// referrerVisibleToOrgs matches a referrer that is attached to at least one live workflow in the
+// given organizations.
+//
+// It is written by hand rather than with referrer.HasWorkflowsWith because the generated form
+// renders as `id IN (subquery)`, which describes a set to build rather than a condition to test.
+// Correlating the subquery to the referrer row says what is meant: for this referrer, does such a
+// workflow exist. The organization is matched on the workflow's own column rather than through a
+// nested relation predicate, which Ent renders back into the uncorrelated shape.
+func referrerVisibleToOrgs(orgIDs []uuid.UUID) predicate.Referrer {
+	orgs := make([]any, 0, len(orgIDs))
+	for _, id := range orgIDs {
+		orgs = append(orgs, id)
+	}
+
+	return func(s *sql.Selector) {
+		joinTable := sql.Table(referrer.WorkflowsTable).As("visible_rw")
+		workflows := sql.Table(referrer.WorkflowsInverseTable).As("visible_wf")
+		sub := sql.Dialect(s.Dialect()).
+			Select(joinTable.C(referrer.WorkflowsPrimaryKey[0])).
+			From(joinTable).
+			Join(workflows).
+			On(joinTable.C(referrer.WorkflowsPrimaryKey[1]), workflows.C(workflow.FieldID)).
+			Where(sql.And(
+				sql.ColumnsEQ(joinTable.C(referrer.WorkflowsPrimaryKey[0]), s.C(referrer.FieldID)),
+				sql.IsNull(workflows.C(workflow.FieldDeletedAt)),
+				sql.In(workflows.C(workflow.FieldOrganizationID), orgs...),
+			))
+		s.Where(sql.Exists(sub))
+	}
+}
+
 // projectVisibilityPredicate builds a project predicate that accepts a project iff it belongs to
 // one of the allowed orgs AND, when RBAC applies to that org, the project is in the caller's
 // visible set. Returns nil when no org grants any project visibility, so callers can treat that
 // as "nothing is visible".
+//
+// A caller reaches a project one of two ways, so the predicate has at most two branches:
+// every project of an org whose role carries no project restriction, or an individually
+// granted project in an org whose role does. Each branch is emitted once for the whole set
+// of orgs rather than once per org, which keeps the filter a constant size: a disjunction
+// that grows with the caller's org count stops the planner from using the root's index and
+// turns the surrounding referrer query into a full scan.
+//
+// The second branch matches (org, project) pairs rather than the two sets independently.
+// Intersecting the sets would also admit a project that belongs to one of the caller's
+// restricted orgs and happens to be granted in another, which is not a grant the caller
+// holds. Pairs cannot express that, and they keep the branch a single condition.
 func projectVisibilityPredicate(orgIDs []uuid.UUID, visibleProjectsMap map[uuid.UUID][]uuid.UUID) predicate.Project {
-	perOrg := make([]predicate.Project, 0, len(orgIDs))
+	unrestrictedOrgs := make([]uuid.UUID, 0, len(orgIDs))
+	grants := make([]projectGrant, 0, len(orgIDs))
 	for _, orgID := range orgIDs {
-		visible, hasRBAC := visibleProjectsMap[orgID]
-		if !hasRBAC {
-			perOrg = append(perOrg, project.HasOrganizationWith(organization.ID(orgID)))
+		visible, restricted := visibleProjectsMap[orgID]
+		if !restricted {
+			unrestrictedOrgs = append(unrestrictedOrgs, orgID)
 			continue
 		}
-		if len(visible) == 0 {
-			continue // RBAC applies but no project is visible in this org
+		// Restricted: nothing in this org is visible beyond the projects granted in it,
+		// so an org with no grants contributes nothing.
+		for _, projectID := range visible {
+			grants = append(grants, projectGrant{orgID: orgID, projectID: projectID})
 		}
-		perOrg = append(perOrg, project.And(
-			project.HasOrganizationWith(organization.ID(orgID)),
-			project.IDIn(visible...),
-		))
 	}
-	if len(perOrg) == 0 {
+
+	predicates := make([]predicate.Project, 0, 2)
+	if len(unrestrictedOrgs) > 0 {
+		// The org is a column on the project row, so this needs no subquery.
+		predicates = append(predicates, project.OrganizationIDIn(unrestrictedOrgs...))
+	}
+	if len(grants) > 0 {
+		predicates = append(predicates, projectGrantsPredicate(grants))
+	}
+
+	switch len(predicates) {
+	case 0:
 		return nil
+	case 1:
+		return predicates[0]
+	default:
+		return project.Or(predicates...)
+	}
+}
+
+// projectGrant is a project the caller may see and the org the grant was recorded under.
+type projectGrant struct {
+	orgID, projectID uuid.UUID
+}
+
+// projectGrantsPredicate matches a project iff it is one of the granted projects AND sits in the
+// org that grant was recorded under, as a single (organization_id, id) IN ((..), (..)) condition.
+func projectGrantsPredicate(grants []projectGrant) predicate.Project {
+	return func(s *sql.Selector) {
+		s.Where(sql.P().Append(func(b *sql.Builder) {
+			b.Wrap(func(nb *sql.Builder) {
+				nb.IdentComma(s.C(project.FieldOrganizationID), s.C(project.FieldID))
+			})
+			b.WriteString(" IN ")
+			b.Wrap(func(nb *sql.Builder) {
+				for i, g := range grants {
+					if i > 0 {
+						nb.Comma()
+					}
+					nb.Wrap(func(vb *sql.Builder) {
+						vb.Args(g.orgID, g.projectID)
+					})
+				}
+			})
+		}))
 	}
-	return project.Or(perOrg...)
 }
 
 // max number of recursive levels to traverse
@@ -331,12 +407,8 @@ func (r *ReferrerRepo) doGet(ctx context.Context, root *ent.Referrer, allowedOrg
 	// and by the visibility if needed
 	predicateReferrer := []predicate.Referrer{}
 
-	predicateWF := []predicate.Workflow{
-		workflow.DeletedAtIsNil(), workflow.HasOrganizationWith(organization.IDIn(allowedOrgs...)),
-	}
-
-	// Attach the workflow predicate
-	predicateReferrer = append(predicateReferrer, referrer.HasWorkflowsWith(predicateWF...))
+	// Attach the visibility predicate
+	predicateReferrer = append(predicateReferrer, referrerVisibleToOrgs(allowedOrgs))
 
 	// When scoping to a project, attestation references must belong to that project (optionally
 	// narrowed to a version). Non-attestation references (materials/subjects) are kept as-is:
@@ -358,6 +430,18 @@ func (r *ReferrerRepo) doGet(ctx context.Context, root *ent.Referrer, allowedOrg
 
 	// Sort references by creation date and ID in descending order for deterministic pagination
 	q := root.QueryReferences().Where(predicateReferrer...).WithWorkflows().
+		// Ent adds a DISTINCT to any traversal by default, because an edge query can multiply
+		// rows. This one cannot: the join table is keyed on (referrer_id, referred_by_id), so a
+		// fixed root matches each referrer at most once, and the selected columns include the
+		// primary key, so there is nothing to collapse. Every predicate above is a semi-join
+		// (EXISTS, IN) or a scalar comparison.
+		//
+		// Any predicate added here must stay join-free. A join would return a reference twice,
+		// which consumes a slot in the page and leaves the cursor on the wrong row, so references
+		// are skipped rather than repeated. sql.OrPredicates applies each predicate to this same
+		// selector, so a joining predicate inside an Or leaks its join here while its WHERE is
+		// OR-ed away.
+		Unique(false).
 		Order(referrer.ByCreatedAt(sql.OrderDesc())).
 		Order(referrer.ByID(sql.OrderDesc())).
 		Limit(p.Limit + 1) // fetch limit+1 to detect next page
@@ -407,7 +491,7 @@ func (r *ReferrerRepo) doGet(ctx context.Context, root *ent.Referrer, allowedOrg
 			Where(
 				referrer.KindEQ(biz.ReferrerAttestationType),
 				projectPred,
-				referrer.HasWorkflowsWith(predicateWF...),
+				referrerVisibleToOrgs(allowedOrgs),
 			).
 			Exist(ctx)
 		if err != nil {