azure: fix lost customDomain binding when a service has multiple hostnames

pkgs/defang/cli.nix

@@ -7,7 +7,7 @@ buildGo125Module {
   pname = "defang-cli";
   version = "git";
   src = lib.cleanSource ../../src;
-  vendorHash = "sha256-3NYtGGBZ13Ln9sGrvJGs4l3dx7ZsNIysku9yfcVik5Q=";
+  vendorHash = "sha256-o7Rx0Qo1QUb9nRVk3hS5pqZPqbtUYthQfQ8FiU3UELQ=";
 
   subPackages = [ "cmd/cli" ];
 

src/go.mod

@@ -80,6 +80,7 @@ require (
 	go.yaml.in/yaml/v4 v4.0.0-rc.4
 	golang.org/x/crypto v0.52.0
 	golang.org/x/mod v0.35.0
+	golang.org/x/net v0.55.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.45.0
 	golang.org/x/term v0.43.0
@@ -158,7 +159,6 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
-	golang.org/x/net v0.55.0 // indirect
 	google.golang.org/genai v1.30.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect

src/pkg/cli/cert.go

@@ -100,8 +100,21 @@ func newCertHTTPClient(r dns.Resolver) HTTPClient {
 // CNAME→fabric→ACME redirect dance used on AWS BYOD / Playground. Azure
 // implements this so `defang cert generate` can drive the Container Apps
 // hostname-add + managed-cert + SniEnabled-bind sequence end-to-end.
+//
+// The two methods mirror the two phases of runACMEJobs: PreflightCert only
+// probes and reports, so every domain's outstanding records can be printed in
+// one block up front, and IssueCert then does the work, routing all of its
+// user-facing output through the per-domain logger it is handed.
 type CertIssuer interface {
-	IssueCert(ctx context.Context, projectName, serviceName, hostname string, resolverAt func(string) dns.Resolver) error
+	// PreflightCert reports the DNS records hostname still needs before a cert
+	// can be issued, without provisioning anything. An empty result means
+	// there is nothing for the user to configure. Because the caller prints
+	// these, IssueCert must not print them again.
+	PreflightCert(ctx context.Context, projectName, serviceName, hostname string, resolverAt func(string) dns.Resolver) ([]dns.RequiredRecord, error)
+	// IssueCert provisions and binds the cert, emitting every user-facing
+	// progress line through log rather than printing directly, so concurrent
+	// per-domain workers don't interleave unattributed output.
+	IssueCert(ctx context.Context, projectName, serviceName, hostname string, resolverAt func(string) dns.Resolver, log func(string, ...any)) error
 }
 
 // domainJob is one (service, domain, targets) tuple processed by a worker.
@@ -191,11 +204,26 @@ func getDomainTargets(serviceInfo *defangv1.ServiceInfo, service compose.Service
 	}
 }
 
-// runIssuerJobs runs the provider-driven cert issuance in parallel. Per-domain
-// state transitions are emitted via a per-domain prefixed logger so the log
-// stream remains readable across concurrent workers.
+// runIssuerJobs runs the provider-driven cert issuance in the same two phases
+// as runACMEJobs:
+// Phase 1 — pre-flight every domain in parallel and print the union of the
+// records still missing as one block, so the user configures them all in a
+// single DNS-console sitting instead of learning about them one worker at a
+// time.
+// Phase 2 — parallel per-domain workers, each emitting state transitions
+// through a prefixed logger so the log stream stays readable.
 func runIssuerJobs(ctx context.Context, projectName string, jobs []domainJob, fab client.FabricClient, issuer CertIssuer) error {
 	pad := maxDomainLen(jobs)
+	resolverAt := dns.NewFabricResolverAt(fab)
+
+	// Phase 1: pre-flight.
+	pending := preflightIssuerDNS(ctx, projectName, jobs, issuer, resolverAt)
+	if len(pending) > 0 {
+		printGroupedRecords(jobs, pending, pad)
+		term.Infof("Awaiting DNS record setup and propagation for %d domain(s)…", len(pending))
+	}
+
+	// Phase 2: parallel workers.
 	eg, gctx := errgroup.WithContext(ctx)
 	eg.SetLimit(maxCertWorkers)
 	var (
@@ -208,7 +236,7 @@ func runIssuerJobs(ctx context.Context, projectName string, jobs []domainJob, fa
 		eg.Go(func() error {
 			start := time.Now()
 			log("issuing cert…")
-			if err := issuer.IssueCert(gctx, projectName, job.serviceName, job.domain, dns.NewFabricResolverAt(fab)); err != nil {
+			if err := issuer.IssueCert(gctx, projectName, job.serviceName, job.domain, resolverAt, log); err != nil {
 				log("failed: %v", err)
 				errMu.Lock()
 				errs = append(errs, fmt.Errorf("%v: %w", job.domain, err))
@@ -321,6 +349,69 @@ func printGroupedCNAMEs(jobs []domainJob, pad int) {
 	}
 }
 
+// preflightIssuerDNS asks the provider, once per domain and in parallel, which
+// DNS records are still missing. A pre-flight error is not fatal: this phase is
+// purely presentational, and the domain's worker re-does the same discovery in
+// phase 2 where the failure can be attributed and reported properly.
+func preflightIssuerDNS(ctx context.Context, projectName string, jobs []domainJob, issuer CertIssuer, resolverAt func(string) dns.Resolver) map[string][]dns.RequiredRecord {
+	pending := make(map[string][]dns.RequiredRecord, len(jobs))
+	var mu sync.Mutex
+	eg, gctx := errgroup.WithContext(ctx)
+	eg.SetLimit(maxCertWorkers)
+	for _, j := range jobs {
+		job := j
+		eg.Go(func() error {
+			records, err := issuer.PreflightCert(gctx, projectName, job.serviceName, job.domain, resolverAt)
+			if err != nil {
+				term.Debugf("Pre-flight cert check for %v failed: %v", job.domain, err)
+				return nil
+			}
+			if len(records) == 0 {
+				return nil
+			}
+			mu.Lock()
+			pending[job.domain] = records
+			mu.Unlock()
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return nil // ignore errors, this should never happen
+	}
+	return pending
+}
+
+// printGroupedRecords prints every pending record across every domain in a
+// single block, iterating jobs (not the map) so sibling hostnames stay in a
+// stable, source order. Rows carry the same [domain] prefix the phase-2 workers
+// use, and the name column is padded across the whole block so the arrows line
+// up as one table rather than one table per domain.
+func printGroupedRecords(jobs []domainJob, pending map[string][]dns.RequiredRecord, pad int) {
+	namePad := 0
+	for _, records := range pending {
+		for _, r := range records {
+			if len(r.Name) > namePad {
+				namePad = len(r.Name)
+			}
+		}
+	}
+	term.Infof("Configure the following DNS record(s):")
+	for _, j := range jobs {
+		records, ok := pending[j.domain]
+		if !ok {
+			continue
+		}
+		log := newDomainLogger(j.domain, pad)
+		for _, r := range records {
+			note := ""
+			if r.Note != "" {
+				note = "  (" + r.Note + ")"
+			}
+			log("%-5s  %-*s  ->  %s%s", r.Type, namePad, r.Name, r.Value, note)
+		}
+	}
+}
+
 // runACMEForDomain runs the per-domain ACME flow: wait for DNS, probe for an
 // existing cert, trigger generation, wait for the cert to come online. All
 // inline progress goes through log so concurrent workers don't fight for the

src/pkg/cli/cert_test.go

@@ -348,18 +348,40 @@ func (m *mockCertFabricClient) calls() int {
 // interface so GenerateLetsEncryptCert's `provider.(CertIssuer)` succeeds.
 // Captures every (project, service, hostname) tuple the SUT calls IssueCert
 // with, and lets the test inject an error per call. The mutex makes the call
-// log safe under the new parallel-worker dispatch.
+// log safe under the parallel-worker dispatch.
+//
+// events records both phases in call order so tests can assert the pre-flight
+// pass completes before any issuance starts without racing on terminal output.
 type mockCertIssuerProvider struct {
 	mockCertProvider
-	mu        sync.Mutex
-	issueErr  error
-	issueCall []string
+	mu           sync.Mutex
+	issueErr     error
+	issueCall    []string
+	events       []string
+	preflight    map[string][]dns.RequiredRecord // hostname -> records still missing
+	preflightErr error
+	onIssue      func(log func(string, ...any))
 }
 
-func (m *mockCertIssuerProvider) IssueCert(_ context.Context, projectName, serviceName, hostname string, _ func(string) dns.Resolver) error {
+func (m *mockCertIssuerProvider) PreflightCert(_ context.Context, _, _, hostname string, _ func(string) dns.Resolver) ([]dns.RequiredRecord, error) {
+	m.mu.Lock()
+	m.events = append(m.events, "preflight:"+hostname)
+	m.mu.Unlock()
+	if m.preflightErr != nil {
+		return nil, m.preflightErr
+	}
+	return m.preflight[hostname], nil
+}
+
+func (m *mockCertIssuerProvider) IssueCert(_ context.Context, projectName, serviceName, hostname string, _ func(string) dns.Resolver, log func(string, ...any)) error {
 	m.mu.Lock()
 	m.issueCall = append(m.issueCall, fmt.Sprintf("%s/%s/%s", projectName, serviceName, hostname))
+	m.events = append(m.events, "issue:"+hostname)
+	onIssue := m.onIssue
 	m.mu.Unlock()
+	if onIssue != nil {
+		onIssue(log)
+	}
 	return m.issueErr
 }
 
@@ -371,6 +393,12 @@ func (m *mockCertIssuerProvider) sortedCalls() []string {
 	return out
 }
 
+func (m *mockCertIssuerProvider) orderedEvents() []string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return slices.Clone(m.events)
+}
+
 func TestGenerateLetsEncryptCert(t *testing.T) {
 	t.Run("error when no services", func(t *testing.T) {
 		provider := &mockCertProvider{
@@ -746,6 +774,141 @@ func TestRunIssuerJobs_ParallelMultipleDomains(t *testing.T) {
 	}
 }
 
+// TestRunIssuerJobs_PreflightsBeforeIssuing pins the two-phase shape: every
+// domain is pre-flighted before any domain starts issuing, so the batched DNS
+// instructions can't be printed after a worker has already begun waiting. A
+// pre-flight failure must not abort or skip issuance — phase 2 re-does the
+// discovery and reports the error against its own domain.
+func TestRunIssuerJobs_PreflightsBeforeIssuing(t *testing.T) {
+	tests := []struct {
+		name         string
+		preflightErr error
+	}{
+		{name: "preflight succeeds"},
+		{name: "preflight fails", preflightErr: errors.New("container app not found")},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			provider := &mockCertIssuerProvider{
+				mockCertProvider: mockCertProvider{
+					services: &defangv1.GetServicesResponse{
+						Services: []*defangv1.ServiceInfo{
+							{Service: &defangv1.Service{Name: "web"}, UseAcmeCert: true, Domainname: "a.example.com"},
+							{Service: &defangv1.Service{Name: "api"}, UseAcmeCert: true, Domainname: "b.example.com"},
+						},
+					},
+				},
+				preflightErr: tt.preflightErr,
+			}
+			project := &compose.Project{
+				Name: "two",
+				Services: compose.Services{
+					"web": {Name: "web", DomainName: "a.example.com"},
+					"api": {Name: "api", DomainName: "b.example.com"},
+				},
+			}
+			if err := GenerateLetsEncryptCert(t.Context(), project, &mockCertFabricClient{}, provider); err != nil {
+				t.Fatalf("unexpected err: %v", err)
+			}
+			events := provider.orderedEvents()
+			if len(events) != 4 {
+				t.Fatalf("expected 2 preflights + 2 issues, got %v", events)
+			}
+			for i, e := range events {
+				wantPrefix := "issue:"
+				if i < 2 {
+					wantPrefix = "preflight:"
+				}
+				if !strings.HasPrefix(e, wantPrefix) {
+					t.Errorf("event %d = %q, want a %q event (all preflights must precede all issues): %v", i, e, wantPrefix, events)
+				}
+			}
+		})
+	}
+}
+
+// TestRunIssuerJobs_PrefixesIssuerOutput asserts the logger handed to IssueCert
+// is the same per-domain prefixed logger the workers use, so provider-internal
+// progress lines are attributed like every other line. Single domain keeps the
+// terminal buffer single-writer.
+func TestRunIssuerJobs_PrefixesIssuerOutput(t *testing.T) {
+	stdout, _ := term.SetupTestTerm(t)
+	provider := &mockCertIssuerProvider{
+		mockCertProvider: mockCertProvider{
+			services: &defangv1.GetServicesResponse{
+				Services: []*defangv1.ServiceInfo{
+					{Service: &defangv1.Service{Name: "web"}, UseAcmeCert: true, Domainname: "a.example.com"},
+				},
+			},
+		},
+		onIssue: func(log func(string, ...any)) { log("registering custom hostname") },
+	}
+	project := &compose.Project{
+		Name:     "one",
+		Services: compose.Services{"web": {Name: "web", DomainName: "a.example.com"}},
+	}
+	if err := GenerateLetsEncryptCert(t.Context(), project, &mockCertFabricClient{}, provider); err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	got := stripANSI(stdout.String())
+	if !strings.Contains(got, "[a.example.com] registering custom hostname") {
+		t.Errorf("issuer output not domain-prefixed:\n%s", got)
+	}
+}
+
+func TestPrintGroupedRecords(t *testing.T) {
+	jobs := []domainJob{
+		{serviceName: "web", domain: "example.com"},
+		{serviceName: "api", domain: "ready.example.com"},
+		{serviceName: "web", domain: "www.example.com"},
+	}
+	pending := map[string][]dns.RequiredRecord{
+		"example.com": {
+			{Type: "TXT", Name: "asuid.example.com", Value: "VID123", Note: "add this first"},
+			{Type: "A", Name: "example.com", Value: "20.1.2.3"},
+		},
+		"www.example.com": {
+			{Type: "CNAME", Name: "www.example.com", Value: "app.westus.azurecontainerapps.io"},
+		},
+	}
+
+	stdout, _ := term.SetupTestTerm(t)
+	printGroupedRecords(jobs, pending, maxDomainLen(jobs))
+	lines := strings.Split(strings.TrimRight(stripANSI(stdout.String()), "\n"), "\n")
+
+	if len(lines) != 4 {
+		t.Fatalf("expected 1 header + 3 record lines, got %d:\n%s", len(lines), strings.Join(lines, "\n"))
+	}
+	if !strings.Contains(lines[0], "Configure the following DNS record(s):") {
+		t.Errorf("missing single grouped header, got %q", lines[0])
+	}
+	// Source order of jobs, not map order; the already-ready domain is omitted.
+	wantPrefixes := []string{" * [example.com]", " * [example.com]", " * [www.example.com]"}
+	for i, want := range wantPrefixes {
+		if !strings.HasPrefix(lines[i+1], want) {
+			t.Errorf("line %d = %q, want prefix %q", i+1, lines[i+1], want)
+		}
+	}
+	if strings.Contains(stripANSI(stdout.String()), "ready.example.com") {
+		t.Error("domain with no pending records should not appear in the block")
+	}
+	if !strings.Contains(lines[1], "(add this first)") {
+		t.Errorf("note not rendered: %q", lines[1])
+	}
+	if strings.Contains(lines[2], "(") {
+		t.Errorf("empty note should render nothing: %q", lines[2])
+	}
+	// The arrow column is padded across the whole block, not per domain, so
+	// records for different domains still line up as one table.
+	cols := make([]int, len(lines)-1)
+	for i, l := range lines[1:] {
+		cols[i] = strings.Index(l, "->")
+	}
+	if cols[0] != cols[1] || cols[1] != cols[2] {
+		t.Errorf("arrow column not aligned across domains: %v\n%s", cols, strings.Join(lines[1:], "\n"))
+	}
+}
+
 // stripANSI removes the colour escape sequences that term.Infof prepends so
 // tests can assert on the post-formatting plain text. The escapes are simple
 // CSI sequences (ESC `[` … `m`) which is all we generate.

src/pkg/cli/client/byoc/azure/cert.go

@@ -7,17 +7,29 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/dns"
 )
 
+// PreflightCert implements the pre-flight half of the cli.CertIssuer interface:
+// it reports the DNS records the hostname still needs so the CLI can print
+// every domain's records in one block before any of them starts waiting.
+func (b *ByocAzure) PreflightCert(ctx context.Context, projectName, serviceName, hostname string, resolverAt func(string) dns.Resolver) ([]dns.RequiredRecord, error) {
+	cred, err := b.driver.NewCreds()
+	if err != nil {
+		return nil, err
+	}
+	rg := b.projectResourceGroupName(projectName)
+	return aca.PreflightCert(ctx, cred, b.driver.SubscriptionID, rg, serviceName, hostname, resolverAt)
+}
+
 // IssueCert implements the cli.CertIssuer interface for the BYOD `defang cert
 // generate` flow: it resolves the provider's subscription, credentials, and
 // project resource group, then hands off to aca.IssueCert (cloud-SDK layer).
 // The CD task also calls aca.IssueCert directly after a deploy to provision
 // delegate-domain certs without depending on the CLI staying up. See
 // pkg/clouds/azure/aca/cert.go for the shared cert flow.
-func (b *ByocAzure) IssueCert(ctx context.Context, projectName, serviceName, hostname string, resolverAt func(string) dns.Resolver) error {
+func (b *ByocAzure) IssueCert(ctx context.Context, projectName, serviceName, hostname string, resolverAt func(string) dns.Resolver, log func(string, ...any)) error {
 	cred, err := b.driver.NewCreds()
 	if err != nil {
 		return err
 	}
 	rg := b.projectResourceGroupName(projectName)
-	return aca.IssueCert(ctx, cred, b.driver.SubscriptionID, rg, serviceName, hostname, resolverAt)
+	return aca.IssueCert(ctx, cred, b.driver.SubscriptionID, rg, serviceName, hostname, resolverAt, log)
 }