From 9ef996c483087301936b979d27e009564db2336f Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Fri, 17 Jan 2025 14:21:27 +0100 Subject: [PATCH 1/8] [create-vm-agent-env-adxt-84] Updated python --- tasks/aws/deploy.py | 2 ++ tasks/aws/docker.py | 3 +++ tasks/deploy.py | 2 ++ tasks/doc.py | 1 + 4 files changed, 8 insertions(+) diff --git a/tasks/aws/deploy.py b/tasks/aws/deploy.py index 2b09f4aa4..07faa7616 100644 --- a/tasks/aws/deploy.py +++ b/tasks/aws/deploy.py @@ -34,6 +34,7 @@ def deploy( full_image_path: Optional[str] = None, cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, + agent_env: Optional[str] = None, ) -> str: flags = extra_flags if extra_flags else {} @@ -89,6 +90,7 @@ def deploy( full_image_path, cluster_agent_full_image_path, agent_flavor, + agent_env, ) diff --git a/tasks/aws/docker.py b/tasks/aws/docker.py index fe245353a..4c1e6bb40 100644 --- a/tasks/aws/docker.py +++ b/tasks/aws/docker.py @@ -26,6 +26,7 @@ "interactive": doc.interactive, "full_image_path": doc.full_image_path, "agent_flavor": doc.agent_flavor, + "agent_env": doc.agent_env, }, ) def create_docker( @@ -40,6 +41,7 @@ def create_docker( interactive: Optional[bool] = True, full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, + agent_env: Optional[str] = None, ): """ Create a docker environment. @@ -62,6 +64,7 @@ def create_docker( extra_flags=extra_flags, full_image_path=full_image_path, agent_flavor=agent_flavor, + agent_env=agent_env, ) if interactive: diff --git a/tasks/deploy.py b/tasks/deploy.py index c4a0696fd..07a4a851f 100644 --- a/tasks/deploy.py +++ b/tasks/deploy.py @@ -28,6 +28,7 @@ def deploy( full_image_path: Optional[str] = None, cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, + agent_env: Optional[str] = None, ) -> str: flags = extra_flags if extra_flags else {} @@ -52,6 +53,7 @@ def deploy( flags["ddagent:fakeintake"] = use_fakeintake flags["ddagent:fullImagePath"] = full_image_path flags["ddagent:clusterAgentFullImagePath"] = cluster_agent_full_image_path + flags["ddagent:extraEnvVars"] = agent_env if install_agent: flags["ddagent:apiKey"] = _get_api_key(cfg) diff --git a/tasks/doc.py b/tasks/doc.py index 558034505..0a2a367dd 100644 --- a/tasks/doc.py +++ b/tasks/doc.py @@ -33,3 +33,4 @@ cluster_agent_full_image_path: str = "The full image path (registry:tag) of the Cluster Agent image to deploy" add_known_host: str = "Add the host to the known_hosts file (default True)" agent_flavor: str = "Use a specific Agent flavor (such as datadog-fips-agent, see PackageFlavor https://github.com/DataDog/agent-release-management/blob/main/generator/const.py)" +agent_env: str = "Extra envionment variables to run the agent with, the format is `VAR1=val1,VAR2=val2`" From a7ed04ee9ee6d41d6214130cc317899e16f0b8dc Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Fri, 17 Jan 2025 14:21:38 +0100 Subject: [PATCH 2/8] [create-vm-agent-env-adxt-84] Go implem --- common/config/environment.go | 2 +- components/datadog/dockeragentparams/params.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/common/config/environment.go b/common/config/environment.go index ce8bb1c5a..120748da2 100644 --- a/common/config/environment.go +++ b/common/config/environment.go @@ -55,7 +55,7 @@ const ( DDAgentDualShipping = "dualshipping" DDAgentSite = "site" DDAgentMajorVersion = "majorVersion" - DDAgentExtraEnvVars = "extraEnvVars" // extraEnvVars is expected in the format: =,=,... + DDAgentExtraEnvVars = "extraEnvVars" // expected in the format: =,=,... DDAgentJMX = "jmx" DDAgentFIPS = "fips" diff --git a/components/datadog/dockeragentparams/params.go b/components/datadog/dockeragentparams/params.go index 67fd25d54..eacbbecc8 100644 --- a/components/datadog/dockeragentparams/params.go +++ b/components/datadog/dockeragentparams/params.go @@ -61,6 +61,10 @@ func NewParams(e config.Env, options ...Option) (*Params, error) { EnvironmentVariables: pulumi.StringMap{}, } + for k, v := range e.AgentExtraEnvVars() { + version.AgentServiceEnvironment[k] = pulumi.String(v) + } + if e.PipelineID() != "" && e.CommitSHA() != "" { options = append(options, WithFullImagePath(utils.BuildDockerImagePath( From 2f48e9d9deda89f1fb2f53a8d2b7bfd167666b0d Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 20 Jan 2025 10:27:21 +0100 Subject: [PATCH 3/8] [create-vm-agent-env-2-adxt-84] helm config: Added to python / helm in go --- common/config/environment.go | 6 ++++++ components/datadog/agent/kubernetes_helm.go | 15 ++++++++++++++- tasks/aws/deploy.py | 2 ++ tasks/aws/kind.py | 5 +++++ tasks/deploy.py | 2 ++ tasks/doc.py | 1 + 6 files changed, 30 insertions(+), 1 deletion(-) diff --git a/common/config/environment.go b/common/config/environment.go index 120748da2..2c0a61f84 100644 --- a/common/config/environment.go +++ b/common/config/environment.go @@ -33,6 +33,7 @@ const ( DDInfraExtraResourcesTags = "extraResourcesTags" DDInfraSSHUser = "sshUser" DDInfraInitOnly = "initOnly" + DDInfraHelmConfig = "helmConfig" // Agent Namespace DDAgentDeployParamName = "deploy" @@ -98,6 +99,7 @@ type Env interface { InfraEnvironmentNames() []string InfraOSDescriptor() string InfraOSImageID() string + InfraHelmConfig() string KubernetesVersion() string DefaultResourceTags() map[string]string ExtraResourcesTags() map[string]string @@ -214,6 +216,10 @@ func (e *CommonEnvironment) InitOnly() bool { return e.GetBoolWithDefault(e.InfraConfig, DDInfraInitOnly, false) } +func (e *CommonEnvironment) InfraHelmConfig() string { + return e.GetStringWithDefault(e.InfraConfig, DDInfraHelmConfig, "") +} + func (e *CommonEnvironment) ExtraResourcesTags() map[string]string { tags, err := tagListToKeyValueMap(e.GetStringListWithDefault(e.InfraConfig, DDInfraExtraResourcesTags, []string{})) if err != nil { diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index ca16a78ce..77b421e3f 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -2,6 +2,7 @@ package agent import ( "fmt" + "os" "golang.org/x/exp/maps" "gopkg.in/yaml.v3" @@ -155,6 +156,16 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi valuesYAML = append(valuesYAML, buildOTelConfigWithFakeintake(args.OTelConfig, args.Fakeintake)) } + // Read and merge custom helm config if provided + if helmConfig := e.InfraHelmConfig(); helmConfig != "" { + customHelm, err := os.ReadFile(helmConfig) + if err != nil { + return nil, err + } + config := pulumi.NewStringAsset(string(customHelm)) + valuesYAML = append(valuesYAML, config) + } + linux, err := helm.NewInstallation(e, helm.InstallArgs{ RepoURL: DatadogHelmRepo, ChartName: "datadog", @@ -215,7 +226,7 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi type HelmValues pulumi.Map func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentImagePath, clusterAgentImageTag string, clusterAgentToken pulumi.StringInput, logsContainerCollectAll bool) HelmValues { - return HelmValues{ + values := HelmValues{ "datadog": pulumi.Map{ "apiKeyExistingSecret": pulumi.String(baseName + "-datadog-credentials"), "appKeyExistingSecret": pulumi.String(baseName + "-datadog-credentials"), @@ -521,6 +532,8 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI }, }, } + + return values } func buildLinuxHelmValuesAutopilot(baseName, agentImagePath, agentImageTag, clusterAgentImagePath, clusterAgentImageTag string, clusterAgentToken pulumi.StringInput) HelmValues { diff --git a/tasks/aws/deploy.py b/tasks/aws/deploy.py index 07faa7616..db0e333e1 100644 --- a/tasks/aws/deploy.py +++ b/tasks/aws/deploy.py @@ -35,6 +35,7 @@ def deploy( cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, agent_env: Optional[str] = None, + helm_config: Optional[str] = None, ) -> str: flags = extra_flags if extra_flags else {} @@ -91,6 +92,7 @@ def deploy( cluster_agent_full_image_path, agent_flavor, agent_env, + helm_config, ) diff --git a/tasks/aws/kind.py b/tasks/aws/kind.py index ee826dfcc..ceff0c251 100644 --- a/tasks/aws/kind.py +++ b/tasks/aws/kind.py @@ -29,6 +29,7 @@ "full_image_path": doc.full_image_path, "cluster_agent_full_image_path": doc.cluster_agent_full_image_path, "agent_flavor": doc.agent_flavor, + "helm_config": doc.helm_config, } ) def create_kind( @@ -45,6 +46,8 @@ def create_kind( full_image_path: Optional[str] = None, cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, + agent_env: Optional[str] = None, + helm_config: Optional[str] = None, ): """ Create a kind environment. @@ -71,6 +74,8 @@ def create_kind( full_image_path=full_image_path, cluster_agent_full_image_path=cluster_agent_full_image_path, agent_flavor=agent_flavor, + agent_env=agent_env, + helm_config=helm_config, ) if interactive: diff --git a/tasks/deploy.py b/tasks/deploy.py index 07a4a851f..691f84b6f 100644 --- a/tasks/deploy.py +++ b/tasks/deploy.py @@ -29,6 +29,7 @@ def deploy( cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, agent_env: Optional[str] = None, + helm_config: Optional[str] = None, ) -> str: flags = extra_flags if extra_flags else {} @@ -54,6 +55,7 @@ def deploy( flags["ddagent:fullImagePath"] = full_image_path flags["ddagent:clusterAgentFullImagePath"] = cluster_agent_full_image_path flags["ddagent:extraEnvVars"] = agent_env + flags["ddinfra:helmConfig"] = helm_config if install_agent: flags["ddagent:apiKey"] = _get_api_key(cfg) diff --git a/tasks/doc.py b/tasks/doc.py index 0a2a367dd..9db3a6f78 100644 --- a/tasks/doc.py +++ b/tasks/doc.py @@ -34,3 +34,4 @@ add_known_host: str = "Add the host to the known_hosts file (default True)" agent_flavor: str = "Use a specific Agent flavor (such as datadog-fips-agent, see PackageFlavor https://github.com/DataDog/agent-release-management/blob/main/generator/const.py)" agent_env: str = "Extra envionment variables to run the agent with, the format is `VAR1=val1,VAR2=val2`" +helm_config: str = "Path to a custom helm config file that will be merged with the default one" From 355d2bc82b125c3de71ece4ecb75d4b5287b667c Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 20 Jan 2025 10:29:20 +0100 Subject: [PATCH 4/8] [create-vm-agent-env-2-adxt-84] Implemented with other k8s vms --- tasks/aws/eks.py | 3 +++ tasks/azure/aks.py | 3 +++ tasks/gcp/gke.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/tasks/aws/eks.py b/tasks/aws/eks.py index ce2345d4a..c3665b0d0 100644 --- a/tasks/aws/eks.py +++ b/tasks/aws/eks.py @@ -33,6 +33,7 @@ "full_image_path": doc.full_image_path, "cluster_agent_full_image_path": doc.cluster_agent_full_image_path, "agent_flavor": doc.agent_flavor, + "helm_config": doc.helm_config, } ) def create_eks( @@ -51,6 +52,7 @@ def create_eks( full_image_path: Optional[str] = None, cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, + helm_config: Optional[str] = None, ): """ Create a new EKS environment. It lasts around 20 minutes. @@ -84,6 +86,7 @@ def create_eks( full_image_path=full_image_path, cluster_agent_full_image_path=cluster_agent_full_image_path, agent_flavor=agent_flavor, + helm_config=helm_config, ) tool.notify(ctx, "Your EKS cluster is now created") diff --git a/tasks/azure/aks.py b/tasks/azure/aks.py index 7b826ea24..85aa7fdc9 100644 --- a/tasks/azure/aks.py +++ b/tasks/azure/aks.py @@ -23,6 +23,7 @@ "agent_version": doc.container_agent_version, "stack_name": doc.stack_name, "agent_flavor": doc.agent_flavor, + "helm_config": doc.helm_config, } ) def create_aks( @@ -39,6 +40,7 @@ def create_aks( cluster_agent_full_image_path: Optional[str] = None, use_fakeintake: Optional[bool] = False, agent_flavor: Optional[str] = None, + helm_config: Optional[str] = None, ): """ Create a new AKS environment. It lasts around 5 minutes. @@ -69,6 +71,7 @@ def create_aks( cluster_agent_full_image_path=cluster_agent_full_image_path, use_fakeintake=use_fakeintake, agent_flavor=agent_flavor, + helm_config=helm_config, ) if interactive: diff --git a/tasks/gcp/gke.py b/tasks/gcp/gke.py index ca92d5dd3..d16797d60 100644 --- a/tasks/gcp/gke.py +++ b/tasks/gcp/gke.py @@ -22,6 +22,7 @@ "agent_version": doc.container_agent_version, "stack_name": doc.stack_name, "agent_flavor": doc.agent_flavor, + "helm_config": doc.helm_config, } ) def create_gke( @@ -39,6 +40,7 @@ def create_gke( use_fakeintake: Optional[bool] = False, use_autopilot: Optional[bool] = False, agent_flavor: Optional[str] = None, + helm_config: Optional[str] = None, ) -> None: """ Create a new GKE environment. @@ -70,6 +72,7 @@ def create_gke( cluster_agent_full_image_path=cluster_agent_full_image_path, use_fakeintake=use_fakeintake, agent_flavor=agent_flavor, + helm_config=helm_config, ) if interactive: From d8a7f395275d6251d27ab90f771ee67077827b15 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 20 Jan 2025 11:09:17 +0100 Subject: [PATCH 5/8] [create-vm-agent-env-2-adxt-84] Removed extra agent env flag --- tasks/aws/kind.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tasks/aws/kind.py b/tasks/aws/kind.py index ceff0c251..8c3775bae 100644 --- a/tasks/aws/kind.py +++ b/tasks/aws/kind.py @@ -46,7 +46,6 @@ def create_kind( full_image_path: Optional[str] = None, cluster_agent_full_image_path: Optional[str] = None, agent_flavor: Optional[str] = None, - agent_env: Optional[str] = None, helm_config: Optional[str] = None, ): """ @@ -74,7 +73,6 @@ def create_kind( full_image_path=full_image_path, cluster_agent_full_image_path=cluster_agent_full_image_path, agent_flavor=agent_flavor, - agent_env=agent_env, helm_config=helm_config, ) From 5a8fa683abf72ce2ee3a975bc7e1578ae6956735 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Tue, 21 Jan 2025 17:22:22 +0100 Subject: [PATCH 6/8] [create-vm-agent-env-2-adxt-84] Undone changes --- common/config/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/config/environment.go b/common/config/environment.go index 2c0a61f84..18529876c 100644 --- a/common/config/environment.go +++ b/common/config/environment.go @@ -56,7 +56,7 @@ const ( DDAgentDualShipping = "dualshipping" DDAgentSite = "site" DDAgentMajorVersion = "majorVersion" - DDAgentExtraEnvVars = "extraEnvVars" // expected in the format: =,=,... + DDAgentExtraEnvVars = "extraEnvVars" // extraEnvVars is expected in the format: =,=,... DDAgentJMX = "jmx" DDAgentFIPS = "fips" From 8875883979662c7fddd6be09c34269ffc512ba4f Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Tue, 21 Jan 2025 17:26:55 +0100 Subject: [PATCH 7/8] [create-vm-agent-env-2-adxt-84] NITs --- components/datadog/agent/kubernetes_helm.go | 4 +--- tasks/doc.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index 77b421e3f..525eaf4b1 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -226,7 +226,7 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi type HelmValues pulumi.Map func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentImagePath, clusterAgentImageTag string, clusterAgentToken pulumi.StringInput, logsContainerCollectAll bool) HelmValues { - values := HelmValues{ + return HelmValues{ "datadog": pulumi.Map{ "apiKeyExistingSecret": pulumi.String(baseName + "-datadog-credentials"), "appKeyExistingSecret": pulumi.String(baseName + "-datadog-credentials"), @@ -532,8 +532,6 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI }, }, } - - return values } func buildLinuxHelmValuesAutopilot(baseName, agentImagePath, agentImageTag, clusterAgentImagePath, clusterAgentImageTag string, clusterAgentToken pulumi.StringInput) HelmValues { diff --git a/tasks/doc.py b/tasks/doc.py index 9db3a6f78..0c0405035 100644 --- a/tasks/doc.py +++ b/tasks/doc.py @@ -33,5 +33,5 @@ cluster_agent_full_image_path: str = "The full image path (registry:tag) of the Cluster Agent image to deploy" add_known_host: str = "Add the host to the known_hosts file (default True)" agent_flavor: str = "Use a specific Agent flavor (such as datadog-fips-agent, see PackageFlavor https://github.com/DataDog/agent-release-management/blob/main/generator/const.py)" -agent_env: str = "Extra envionment variables to run the agent with, the format is `VAR1=val1,VAR2=val2`" +agent_env: str = "Extra environment variables to run the agent with, the format is `VAR1=val1,VAR2=val2`" helm_config: str = "Path to a custom helm config file that will be merged with the default one" From bd1d2f12a2ee5804f2c1dd0ddd21b750b57e02fb Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Thu, 23 Jan 2025 09:40:45 +0100 Subject: [PATCH 8/8] [create-vm-agent-env-2-adxt-84] Applied suggestions --- common/config/environment.go | 12 ++++++------ components/datadog/agent/kubernetes_helm.go | 2 +- tasks/deploy.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/config/environment.go b/common/config/environment.go index 18529876c..c2e73af9e 100644 --- a/common/config/environment.go +++ b/common/config/environment.go @@ -33,7 +33,6 @@ const ( DDInfraExtraResourcesTags = "extraResourcesTags" DDInfraSSHUser = "sshUser" DDInfraInitOnly = "initOnly" - DDInfraHelmConfig = "helmConfig" // Agent Namespace DDAgentDeployParamName = "deploy" @@ -59,6 +58,7 @@ const ( DDAgentExtraEnvVars = "extraEnvVars" // extraEnvVars is expected in the format: =,=,... DDAgentJMX = "jmx" DDAgentFIPS = "fips" + DDAgentHelmConfig = "helmConfig" // Updater Namespace DDUpdaterParamName = "deploy" @@ -99,7 +99,6 @@ type Env interface { InfraEnvironmentNames() []string InfraOSDescriptor() string InfraOSImageID() string - InfraHelmConfig() string KubernetesVersion() string DefaultResourceTags() map[string]string ExtraResourcesTags() map[string]string @@ -127,6 +126,7 @@ type Env interface { DogstatsdFullImagePath() string UpdaterDeploy() bool MajorVersion() string + AgentHelmConfig() string GetBoolWithDefault(config *sdkconfig.Config, paramName string, defaultValue bool) bool GetStringListWithDefault(config *sdkconfig.Config, paramName string, defaultValue []string) []string @@ -216,10 +216,6 @@ func (e *CommonEnvironment) InitOnly() bool { return e.GetBoolWithDefault(e.InfraConfig, DDInfraInitOnly, false) } -func (e *CommonEnvironment) InfraHelmConfig() string { - return e.GetStringWithDefault(e.InfraConfig, DDInfraHelmConfig, "") -} - func (e *CommonEnvironment) ExtraResourcesTags() map[string]string { tags, err := tagListToKeyValueMap(e.GetStringListWithDefault(e.InfraConfig, DDInfraExtraResourcesTags, []string{})) if err != nil { @@ -451,3 +447,7 @@ func (e *CommonEnvironment) AgentFIPS() bool { func (e *CommonEnvironment) AgentJMX() bool { return e.GetBoolWithDefault(e.AgentConfig, DDAgentJMX, false) } + +func (e *CommonEnvironment) AgentHelmConfig() string { + return e.GetStringWithDefault(e.AgentConfig, DDAgentHelmConfig, "") +} diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index 525eaf4b1..05093d48c 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -157,7 +157,7 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi } // Read and merge custom helm config if provided - if helmConfig := e.InfraHelmConfig(); helmConfig != "" { + if helmConfig := e.AgentHelmConfig(); helmConfig != "" { customHelm, err := os.ReadFile(helmConfig) if err != nil { return nil, err diff --git a/tasks/deploy.py b/tasks/deploy.py index 691f84b6f..31d30d939 100644 --- a/tasks/deploy.py +++ b/tasks/deploy.py @@ -55,7 +55,7 @@ def deploy( flags["ddagent:fullImagePath"] = full_image_path flags["ddagent:clusterAgentFullImagePath"] = cluster_agent_full_image_path flags["ddagent:extraEnvVars"] = agent_env - flags["ddinfra:helmConfig"] = helm_config + flags["ddagent:helmConfig"] = helm_config if install_agent: flags["ddagent:apiKey"] = _get_api_key(cfg)