Skip to content

Commit fc1f185

Browse files
shreyas-goenkashreyas-goenka
authored
Move deploy telemetry function to a separate file (#2753)
Based on feedback in #2593 (comment)
1 parent a45caaf commit fc1f185

File tree

2 files changed

+168
-158
lines changed

2 files changed

+168
-158
lines changed

‎bundle/phases/deploy.go

Lines changed: 1 addition & 158 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package phases
33
import (
44
"context"
55
"errors"
6-
"slices"
76

87
"github.com/databricks/cli/bundle"
98
"github.com/databricks/cli/bundle/apps"
@@ -21,11 +20,8 @@ import (
2120
"github.com/databricks/cli/bundle/trampoline"
2221
"github.com/databricks/cli/libs/cmdio"
2322
"github.com/databricks/cli/libs/diag"
24-
"github.com/databricks/cli/libs/dyn"
2523
"github.com/databricks/cli/libs/log"
2624
"github.com/databricks/cli/libs/sync"
27-
"github.com/databricks/cli/libs/telemetry"
28-
"github.com/databricks/cli/libs/telemetry/protos"
2925
terraformlib "github.com/databricks/cli/libs/terraform"
3026
tfjson "github.com/hashicorp/terraform-json"
3127
)
@@ -233,7 +229,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
233229
return diags
234230
}
235231

236-
logTelemetry(ctx, b)
232+
logDeployTelemetry(ctx, b)
237233
return diags.Extend(bundle.Apply(ctx, b, scripts.Execute(config.ScriptPostDeploy)))
238234
}
239235

@@ -242,156 +238,3 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
242238
// Since we have a timeout of 3 seconds, we cap the maximum number of IDs
243239
// we send in a single request to have reliable telemetry.
244240
const ResourceIdLimit = 1000
245-
246-
func logTelemetry(ctx context.Context, b *bundle.Bundle) {
247-
resourcesCount := int64(0)
248-
_, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
249-
resourcesCount++
250-
return v, nil
251-
})
252-
if err != nil {
253-
log.Debugf(ctx, "failed to count resources: %s", err)
254-
}
255-
256-
var jobsIds []string
257-
for _, job := range b.Config.Resources.Jobs {
258-
if len(jobsIds) >= ResourceIdLimit {
259-
break
260-
}
261-
262-
// Do not include missing IDs in telemetry. We can still detect them
263-
// by comparing against the resource count.
264-
if job == nil || job.ID == "" {
265-
continue
266-
}
267-
jobsIds = append(jobsIds, job.ID)
268-
}
269-
var pipelineIds []string
270-
for _, pipeline := range b.Config.Resources.Pipelines {
271-
if len(pipelineIds) >= ResourceIdLimit {
272-
break
273-
}
274-
275-
// Do not include missing IDs in telemetry. We can still detect them
276-
// by comparing against the resource count.
277-
if pipeline == nil || pipeline.ID == "" {
278-
continue
279-
}
280-
pipelineIds = append(pipelineIds, pipeline.ID)
281-
}
282-
var clusterIds []string
283-
for _, cluster := range b.Config.Resources.Clusters {
284-
if len(clusterIds) >= ResourceIdLimit {
285-
break
286-
}
287-
288-
// Do not include missing IDs in telemetry. We can still detect them
289-
// by comparing against the resource count.
290-
if cluster == nil || cluster.ID == "" {
291-
continue
292-
}
293-
clusterIds = append(clusterIds, cluster.ID)
294-
}
295-
var dashboardIds []string
296-
for _, dashboard := range b.Config.Resources.Dashboards {
297-
if len(dashboardIds) >= ResourceIdLimit {
298-
break
299-
}
300-
301-
// Do not include missing IDs in telemetry. We can still detect them
302-
// by comparing against the resource count.
303-
if dashboard == nil || dashboard.ID == "" {
304-
continue
305-
}
306-
dashboardIds = append(dashboardIds, dashboard.ID)
307-
}
308-
309-
// sort the IDs to make the record generated deterministic
310-
// this is important for testing purposes
311-
slices.Sort(jobsIds)
312-
slices.Sort(pipelineIds)
313-
slices.Sort(clusterIds)
314-
slices.Sort(dashboardIds)
315-
316-
// If the bundle UUID is not set, we use a default 0 value.
317-
bundleUuid := "00000000-0000-0000-0000-000000000000"
318-
if b.Config.Bundle.Uuid != "" {
319-
bundleUuid = b.Config.Bundle.Uuid
320-
}
321-
322-
variableCount := len(b.Config.Variables)
323-
complexVariableCount := int64(0)
324-
lookupVariableCount := int64(0)
325-
for _, v := range b.Config.Variables {
326-
// If the resolved value of the variable is a complex type, we count it as a complex variable.
327-
// We can't rely on the "type: complex" annotation because the annotation is optional in some contexts
328-
// like bundle YAML files.
329-
if v.IsComplexValued() {
330-
complexVariableCount++
331-
}
332-
333-
if v.Lookup != nil {
334-
lookupVariableCount++
335-
}
336-
}
337-
338-
artifactPathType := protos.BundleDeployArtifactPathTypeUnspecified
339-
if libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) {
340-
artifactPathType = protos.BundleDeployArtifactPathTypeVolume
341-
} else if libraries.IsWorkspacePath(b.Config.Workspace.ArtifactPath) {
342-
artifactPathType = protos.BundleDeployArtifactPathTypeWorkspace
343-
}
344-
345-
mode := protos.BundleModeUnspecified
346-
if b.Config.Bundle.Mode == config.Development {
347-
mode = protos.BundleModeDevelopment
348-
} else if b.Config.Bundle.Mode == config.Production {
349-
mode = protos.BundleModeProduction
350-
}
351-
352-
experimentalConfig := b.Config.Experimental
353-
if experimentalConfig == nil {
354-
experimentalConfig = &config.Experimental{}
355-
}
356-
357-
telemetry.Log(ctx, protos.DatabricksCliLog{
358-
BundleDeployEvent: &protos.BundleDeployEvent{
359-
BundleUuid: bundleUuid,
360-
DeploymentId: b.Metrics.DeploymentId.String(),
361-
362-
ResourceCount: resourcesCount,
363-
ResourceJobCount: int64(len(b.Config.Resources.Jobs)),
364-
ResourcePipelineCount: int64(len(b.Config.Resources.Pipelines)),
365-
ResourceModelCount: int64(len(b.Config.Resources.Models)),
366-
ResourceExperimentCount: int64(len(b.Config.Resources.Experiments)),
367-
ResourceModelServingEndpointCount: int64(len(b.Config.Resources.ModelServingEndpoints)),
368-
ResourceRegisteredModelCount: int64(len(b.Config.Resources.RegisteredModels)),
369-
ResourceQualityMonitorCount: int64(len(b.Config.Resources.QualityMonitors)),
370-
ResourceSchemaCount: int64(len(b.Config.Resources.Schemas)),
371-
ResourceVolumeCount: int64(len(b.Config.Resources.Volumes)),
372-
ResourceClusterCount: int64(len(b.Config.Resources.Clusters)),
373-
ResourceDashboardCount: int64(len(b.Config.Resources.Dashboards)),
374-
ResourceAppCount: int64(len(b.Config.Resources.Apps)),
375-
376-
ResourceJobIDs: jobsIds,
377-
ResourcePipelineIDs: pipelineIds,
378-
ResourceClusterIDs: clusterIds,
379-
ResourceDashboardIDs: dashboardIds,
380-
381-
Experimental: &protos.BundleDeployExperimental{
382-
BundleMode: mode,
383-
ConfigurationFileCount: b.Metrics.ConfigurationFileCount,
384-
TargetCount: b.Metrics.TargetCount,
385-
WorkspaceArtifactPathType: artifactPathType,
386-
BoolValues: b.Metrics.BoolValues,
387-
PythonAddedResourcesCount: b.Metrics.PythonAddedResourcesCount,
388-
PythonUpdatedResourcesCount: b.Metrics.PythonUpdatedResourcesCount,
389-
PythonResourceLoadersCount: int64(len(experimentalConfig.Python.Resources)),
390-
PythonResourceMutatorsCount: int64(len(experimentalConfig.Python.Mutators)),
391-
VariableCount: int64(variableCount),
392-
ComplexVariableCount: complexVariableCount,
393-
LookupVariableCount: lookupVariableCount,
394-
},
395-
},
396-
})
397-
}

‎bundle/phases/telemetry.go

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
package phases
2+
3+
import (
4+
"context"
5+
"slices"
6+
7+
"github.com/databricks/cli/bundle"
8+
"github.com/databricks/cli/bundle/config"
9+
"github.com/databricks/cli/bundle/libraries"
10+
"github.com/databricks/cli/libs/dyn"
11+
"github.com/databricks/cli/libs/log"
12+
"github.com/databricks/cli/libs/telemetry"
13+
"github.com/databricks/cli/libs/telemetry/protos"
14+
)
15+
16+
func logDeployTelemetry(ctx context.Context, b *bundle.Bundle) {
17+
resourcesCount := int64(0)
18+
_, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
19+
resourcesCount++
20+
return v, nil
21+
})
22+
if err != nil {
23+
log.Debugf(ctx, "failed to count resources: %s", err)
24+
}
25+
26+
var jobsIds []string
27+
for _, job := range b.Config.Resources.Jobs {
28+
if len(jobsIds) >= ResourceIdLimit {
29+
break
30+
}
31+
32+
// Do not include missing IDs in telemetry. We can still detect them
33+
// by comparing against the resource count.
34+
if job == nil || job.ID == "" {
35+
continue
36+
}
37+
jobsIds = append(jobsIds, job.ID)
38+
}
39+
var pipelineIds []string
40+
for _, pipeline := range b.Config.Resources.Pipelines {
41+
if len(pipelineIds) >= ResourceIdLimit {
42+
break
43+
}
44+
45+
// Do not include missing IDs in telemetry. We can still detect them
46+
// by comparing against the resource count.
47+
if pipeline == nil || pipeline.ID == "" {
48+
continue
49+
}
50+
pipelineIds = append(pipelineIds, pipeline.ID)
51+
}
52+
var clusterIds []string
53+
for _, cluster := range b.Config.Resources.Clusters {
54+
if len(clusterIds) >= ResourceIdLimit {
55+
break
56+
}
57+
58+
// Do not include missing IDs in telemetry. We can still detect them
59+
// by comparing against the resource count.
60+
if cluster == nil || cluster.ID == "" {
61+
continue
62+
}
63+
clusterIds = append(clusterIds, cluster.ID)
64+
}
65+
var dashboardIds []string
66+
for _, dashboard := range b.Config.Resources.Dashboards {
67+
if len(dashboardIds) >= ResourceIdLimit {
68+
break
69+
}
70+
71+
// Do not include missing IDs in telemetry. We can still detect them
72+
// by comparing against the resource count.
73+
if dashboard == nil || dashboard.ID == "" {
74+
continue
75+
}
76+
dashboardIds = append(dashboardIds, dashboard.ID)
77+
}
78+
79+
// sort the IDs to make the record generated deterministic
80+
// this is important for testing purposes
81+
slices.Sort(jobsIds)
82+
slices.Sort(pipelineIds)
83+
slices.Sort(clusterIds)
84+
slices.Sort(dashboardIds)
85+
86+
// If the bundle UUID is not set, we use a default 0 value.
87+
bundleUuid := "00000000-0000-0000-0000-000000000000"
88+
if b.Config.Bundle.Uuid != "" {
89+
bundleUuid = b.Config.Bundle.Uuid
90+
}
91+
92+
variableCount := len(b.Config.Variables)
93+
complexVariableCount := int64(0)
94+
lookupVariableCount := int64(0)
95+
for _, v := range b.Config.Variables {
96+
// If the resolved value of the variable is a complex type, we count it as a complex variable.
97+
// We can't rely on the "type: complex" annotation because the annotation is optional in some contexts
98+
// like bundle YAML files.
99+
if v.IsComplexValued() {
100+
complexVariableCount++
101+
}
102+
103+
if v.Lookup != nil {
104+
lookupVariableCount++
105+
}
106+
}
107+
108+
artifactPathType := protos.BundleDeployArtifactPathTypeUnspecified
109+
if libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) {
110+
artifactPathType = protos.BundleDeployArtifactPathTypeVolume
111+
} else if libraries.IsWorkspacePath(b.Config.Workspace.ArtifactPath) {
112+
artifactPathType = protos.BundleDeployArtifactPathTypeWorkspace
113+
}
114+
115+
mode := protos.BundleModeUnspecified
116+
if b.Config.Bundle.Mode == config.Development {
117+
mode = protos.BundleModeDevelopment
118+
} else if b.Config.Bundle.Mode == config.Production {
119+
mode = protos.BundleModeProduction
120+
}
121+
122+
experimentalConfig := b.Config.Experimental
123+
if experimentalConfig == nil {
124+
experimentalConfig = &config.Experimental{}
125+
}
126+
127+
telemetry.Log(ctx, protos.DatabricksCliLog{
128+
BundleDeployEvent: &protos.BundleDeployEvent{
129+
BundleUuid: bundleUuid,
130+
DeploymentId: b.Metrics.DeploymentId.String(),
131+
132+
ResourceCount: resourcesCount,
133+
ResourceJobCount: int64(len(b.Config.Resources.Jobs)),
134+
ResourcePipelineCount: int64(len(b.Config.Resources.Pipelines)),
135+
ResourceModelCount: int64(len(b.Config.Resources.Models)),
136+
ResourceExperimentCount: int64(len(b.Config.Resources.Experiments)),
137+
ResourceModelServingEndpointCount: int64(len(b.Config.Resources.ModelServingEndpoints)),
138+
ResourceRegisteredModelCount: int64(len(b.Config.Resources.RegisteredModels)),
139+
ResourceQualityMonitorCount: int64(len(b.Config.Resources.QualityMonitors)),
140+
ResourceSchemaCount: int64(len(b.Config.Resources.Schemas)),
141+
ResourceVolumeCount: int64(len(b.Config.Resources.Volumes)),
142+
ResourceClusterCount: int64(len(b.Config.Resources.Clusters)),
143+
ResourceDashboardCount: int64(len(b.Config.Resources.Dashboards)),
144+
ResourceAppCount: int64(len(b.Config.Resources.Apps)),
145+
146+
ResourceJobIDs: jobsIds,
147+
ResourcePipelineIDs: pipelineIds,
148+
ResourceClusterIDs: clusterIds,
149+
ResourceDashboardIDs: dashboardIds,
150+
151+
Experimental: &protos.BundleDeployExperimental{
152+
BundleMode: mode,
153+
ConfigurationFileCount: b.Metrics.ConfigurationFileCount,
154+
TargetCount: b.Metrics.TargetCount,
155+
WorkspaceArtifactPathType: artifactPathType,
156+
BoolValues: b.Metrics.BoolValues,
157+
PythonAddedResourcesCount: b.Metrics.PythonAddedResourcesCount,
158+
PythonUpdatedResourcesCount: b.Metrics.PythonUpdatedResourcesCount,
159+
PythonResourceLoadersCount: int64(len(experimentalConfig.Python.Resources)),
160+
PythonResourceMutatorsCount: int64(len(experimentalConfig.Python.Mutators)),
161+
VariableCount: int64(variableCount),
162+
ComplexVariableCount: complexVariableCount,
163+
LookupVariableCount: lookupVariableCount,
164+
},
165+
},
166+
})
167+
}

0 commit comments

Comments
 (0)