Gangams/enable release pipe line (#393)

* comment acs env and uncomment aks * fix line ending issues * readme update * scripts for ado release pipeline * script to update placeholders * fix issue with tag * readme updates * release pipeline scripts * script updates * trim line endings * improve regex * fix file endings * add comment to yaml * readme updates * update release process instructions * update readme * readme updates * release process updates * updates to release process * pr feedback
microsoft · Jul 14, 2020 · 0260eba · 0260eba
1 parent c16e795
commit 0260eba
Show file tree

Hide file tree

Showing 12 changed files with 1,132 additions and 832 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1 +1,3 @@
-+*.sh    text eol=lf
++*.yaml  text eol=lf
++*.sh    text eol=lf
++*.yml   text eol=lf
diff --git a/.gitignore b/.gitignore
@@ -18,8 +18,16 @@ build/windows/installer/certificategenerator/obj
 kubernetes/windows/omsagentwindows
 # ignore everything under intermediate directory
 intermediate
-# ignore all these files
-*[.pyc, .so, .zip, .exe, .jar, .dll, .obj]
+# ignore files which are having these extensions
+*.pyc
+*.so
+*.zip
+*.tar
+*.exe
+*.jar
+*.dll
+*.obj
+
 # ignore docker provider shell bundle
 kubernetes/linux/Linux_ULINUX_1.0_x64_64_Release
 # ignore intermediate .h files under go src

diff --git a/.pipelines/get-aad-app-creds-from-kv.sh b/.pipelines/get-aad-app-creds-from-kv.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+echo "end: get app id and secret from specified key vault"
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           KV) KV=$VALUE ;;
+           KVSECRETNAMEAPPID) AppId=$VALUE ;;
+           KVSECRETNAMEAPPSECRET) AppSecret=$VALUE ;;
+           *)
+    esac
+done
+
+echo "key vault name:${KV}"
+echo "key vault secret name for appid:${KVSECRETNAMEAPPID}"
+echo "key vault secret name for appsecret:${KVSECRETNAMEAPPSECRET}"
+
+az keyvault secret download --file ./acrappid --vault-name ${KV}  --name ${AppId}
+
+echo "downloaded the appid from KV:${KV} and KV secret:${AppId}"
+
+az keyvault secret download --file ./acrappsecret --vault-name ${KV}  --name ${AppSecret}
+
+echo "downloaded the appsecret from KV:${KV} and KV secret:${AppSecret}"
+
+echo "end: get app id and secret from specified key vault"
diff --git a/.pipelines/get-workspace-id-and-key.sh b/.pipelines/get-workspace-id-and-key.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+echo "start: get workspace id and key from WorkspaceResourceId etc.."
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           WorkspaceResourceId) WorkspaceResourceId=$VALUE ;;
+
+           *)
+    esac
+done
+
+echo "Log Analytics Workspace ResourceId: ${WorkspaceResourceId}"
+
+echo "getting workspace Guid"
+workspaceGuid=$(az resource show --ids $WorkspaceResourceId --resource-type Microsoft.OperationalInsights/workspaces --query properties.customerId -o tsv)
+echo "writing workspace guid to WSID file"
+echo $workspaceGuid > ~/WSID
+
+echo "getting workspace primaryshared key"
+workspaceKey=$(az rest --method post --uri $WorkspaceResourceId/sharedKeys?api-version=2015-11-01-preview --query primarySharedKey -o tsv)
+echo "writing workspace key to WSKEY file"
+echo $workspaceKey > ~/WSKEY
+
+echo "end: get workspace id and key from WorkspaceResourceId etc.."
diff --git a/.pipelines/pipeline.user.linux.yml b/.pipelines/pipeline.user.linux.yml
@@ -1,44 +1,44 @@
-environment:
-  host:
-    os: 'linux'
-    flavor: 'ubuntu'
-    version: '16.04'
-  runtime:
-    provider: 'appcontainer'
-    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:1.0'
-
-version:
-  name: 'DockerProvider'
-  major: 10
-  minor: 0
-  tag: 'beta'
-  system: 'custom'
-  exclude_commit: true
-
-restore:
-  commands:
-    - !!defaultcommand
-      name: 'get go modules'
-      command: '.pipelines/restore-linux.sh'
-      fail_on_stderr: false
-
-build:
-  commands:
-    - !!defaultcommand
-      name: 'Build Docker Provider Shell Bundle'
-      command: '.pipelines/build-linux.sh'
-      fail_on_stderr: false
-
-package:
-  commands:
-    - !!dockerbuildcommand                               # REQUIRED: This maps the command data to a concrete type in the CDPX orchestrator.
-      name: 'Build Docker Image'                          # REQUIRED: All commands have a name field. All console output captured when
-                                                          #           this command runs is tagged with the value of this field.
-      context_folder: 'kubernetes/linux'                  # REQUIRED: The repository root relative path of the folder containing the Dockerfile to build.
-                                                          # In effect, the context folder will be repository_checkout_folder/src/DockerFinal.
-      dockerfile_name: 'Dockerfile'                       # OPTIONAL: The name of the dockerfile. Docker client does allow the Dockerfile
-                                                          # to be named differently. Defaults to Dockerfile.
-                                                          # In effect, the -f option value passed to docker build will be repository_checkout_folder/src/DockerFinal/Foo.dockerfile.
-      repository_name: 'cdpxlinux'                        # only supported ones are cdpx acr repos
-      tag: 'cidev'                                        # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
-      latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
+environment:
+  host:
+    os: 'linux'
+    flavor: 'ubuntu'
+    version: '16.04'
+  runtime:
+    provider: 'appcontainer'
+    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:1.0'
+
+version:
+  name: 'DockerProvider'
+  major: 10
+  minor: 0
+  tag: 'beta'
+  system: 'custom'
+  exclude_commit: true
+
+restore:
+  commands:
+    - !!defaultcommand
+      name: 'get go modules'
+      command: '.pipelines/restore-linux.sh'
+      fail_on_stderr: false
+
+build:
+  commands:
+    - !!defaultcommand
+      name: 'Build Docker Provider Shell Bundle'
+      command: '.pipelines/build-linux.sh'
+      fail_on_stderr: false
+
+package:
+  commands:
+    - !!dockerbuildcommand                               # REQUIRED: This maps the command data to a concrete type in the CDPX orchestrator.
+      name: 'Build Docker Image'                          # REQUIRED: All commands have a name field. All console output captured when
+                                                          #           this command runs is tagged with the value of this field.
+      context_folder: 'kubernetes/linux'                  # REQUIRED: The repository root relative path of the folder containing the Dockerfile to build.
+                                                          # In effect, the context folder will be repository_checkout_folder/src/DockerFinal.
+      dockerfile_name: 'Dockerfile'                       # OPTIONAL: The name of the dockerfile. Docker client does allow the Dockerfile
+                                                          # to be named differently. Defaults to Dockerfile.
+                                                          # In effect, the -f option value passed to docker build will be repository_checkout_folder/src/DockerFinal/Foo.dockerfile.
+      repository_name: 'cdpxlinux'                        # only supported ones are cdpx acr repos
+      tag: 'cidev'                                        # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
+      latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
diff --git a/.pipelines/pipeline.user.windows.yml b/.pipelines/pipeline.user.windows.yml
@@ -1,55 +1,55 @@
-environment:
-  host:
-    os: 'windows'
-    flavor: 'server'
-    version: '2019'
-  runtime:
-    provider: 'appcontainer'
-    image: 'cdpxwin1809.azurecr.io/user/azure-monitor/container-insights:6.0'
-    source_mode: 'map'
-
-version:
-  name: 'Certificate Generator and Out OMS plugin'
-  major: 10
-  minor: 0
-  tag: 'beta'
-  system: 'custom'
-  exclude_commit: true
-
-signing_options:
-  profile: 'azure'
-  codesign_validation_glob_pattern: 'regex|.+(?:dll|exe|sys|ps1|psm1|ps1xml|psc1|psd1|cdxml|vbs|js|wsf)$;-:file|**\linux\**' #CSV does not currently support binaries built for linux, so we exclude this folder
-
-static_analysis_options:
-  binskim_options:
-    files_to_scan:
-      - from: 'build\windows\installer\certificategenerator\bin\'
-        exclude: # exclude binaries which are referenced via dotnet packages and not built by us
-          - '**/**/**/BouncyCastle.Crypto.dll'
-          - '**/**/**/**/BouncyCastle.Crypto.dll'
-restore:
-  commands:
-    - !!defaultcommand
-       name: 'Restore dotnet packages'
-       command: '.pipelines/restore-windows.cmd'
-
-build:
-  commands:
-    - !!defaultcommand
-      name: 'Build Certificate Generator Source code and Out OMS Go plugin code'
-      command: '.pipelines/build-windows.cmd'
-      fail_on_stderr: false
-
-package:
-  commands:
-    - !!dockerbuildcommand                                # REQUIRED: This maps the command data to a concrete type in the CDPX orchestrator.
-      name: 'Build Docker Image'                          # REQUIRED: All commands have a name field. All console output captured when
-                                                          #           this command runs is tagged with the value of this field.
-      context_folder: 'kubernetes/windows'                # REQUIRED: The repository root relative path of the folder containing the Dockerfile to build.
-                                                          # In effect, the context folder will be repository_checkout_folder/src/DockerFinal.
-      dockerfile_name: 'Dockerfile'                       # OPTIONAL: The name of the dockerfile. Docker client does allow the Dockerfile
-                                                          # to be named differently. Defaults to Dockerfile.
-                                                          # In effect, the -f option value passed to docker build will be repository_checkout_folder/src/DockerFinal/Foo.dockerfile.
-      repository_name: 'cdpxwin1809'                      # only supported ones are cdpx acr repos
-      tag: 'win-cidev'                                    # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
-      latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
+environment:
+  host:
+    os: 'windows'
+    flavor: 'server'
+    version: '2019'
+  runtime:
+    provider: 'appcontainer'
+    image: 'cdpxwin1809.azurecr.io/user/azure-monitor/container-insights:6.0'
+    source_mode: 'map'
+
+version:
+  name: 'Certificate Generator and Out OMS plugin'
+  major: 10
+  minor: 0
+  tag: 'beta'
+  system: 'custom'
+  exclude_commit: true
+
+signing_options:
+  profile: 'azure'
+  codesign_validation_glob_pattern: 'regex|.+(?:dll|exe|sys|ps1|psm1|ps1xml|psc1|psd1|cdxml|vbs|js|wsf)$;-:file|**\linux\**' #CSV does not currently support binaries built for linux, so we exclude this folder
+
+static_analysis_options:
+  binskim_options:
+    files_to_scan:
+      - from: 'build\windows\installer\certificategenerator\bin\'
+        exclude: # exclude binaries which are referenced via dotnet packages and not built by us
+          - '**/**/**/BouncyCastle.Crypto.dll'
+          - '**/**/**/**/BouncyCastle.Crypto.dll'
+restore:
+  commands:
+    - !!defaultcommand
+       name: 'Restore dotnet packages'
+       command: '.pipelines/restore-windows.cmd'
+
+build:
+  commands:
+    - !!defaultcommand
+      name: 'Build Certificate Generator Source code and Out OMS Go plugin code'
+      command: '.pipelines/build-windows.cmd'
+      fail_on_stderr: false
+
+package:
+  commands:
+    - !!dockerbuildcommand                                # REQUIRED: This maps the command data to a concrete type in the CDPX orchestrator.
+      name: 'Build Docker Image'                          # REQUIRED: All commands have a name field. All console output captured when
+                                                          #           this command runs is tagged with the value of this field.
+      context_folder: 'kubernetes/windows'                # REQUIRED: The repository root relative path of the folder containing the Dockerfile to build.
+                                                          # In effect, the context folder will be repository_checkout_folder/src/DockerFinal.
+      dockerfile_name: 'Dockerfile'                       # OPTIONAL: The name of the dockerfile. Docker client does allow the Dockerfile
+                                                          # to be named differently. Defaults to Dockerfile.
+                                                          # In effect, the -f option value passed to docker build will be repository_checkout_folder/src/DockerFinal/Foo.dockerfile.
+      repository_name: 'cdpxwin1809'                      # only supported ones are cdpx acr repos
+      tag: 'win-cidev'                                    # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
+      latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
diff --git a/.pipelines/pull-from-cdpx-and-push-to-ci-acr-linux-image.sh b/.pipelines/pull-from-cdpx-and-push-to-ci-acr-linux-image.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+echo "start: pull linux agent image from cdpx and push to ciprod acr"
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           CDPXACRLinux) CDPX_ACR=$VALUE ;;
+           CDPXLinuxAgentRepositoryName) CDPX_REPO_NAME=$VALUE ;;
+           CDPXLinuxAgentImageTag) CDPX_AGENT_IMAGE_TAG=$VALUE ;;
+           CIACR) CI_ACR=$VALUE ;;
+           CIAgentRepositoryName) CI_AGENT_REPO=$VALUE ;;
+           CIRelease) CI_RELEASE=$VALUE ;;
+           CIImageTagSuffix) CI_IMAGE_TAG_SUFFIX=$VALUE ;;
+
+           *)
+    esac
+done
+
+echo "start: read appid and appsecret"
+ACR_APP_ID=$(cat ./acrappid)
+ACR_APP_SECRET=$(cat ./acrappsecret)
+echo "end: read appid and appsecret"
+
+echo "login to cdpxlinux acr:${CDPX_ACR}"
+docker login $CDPX_ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
+echo "login to cdpxlinux acr completed: ${CDPX_ACR}"
+
+echo "pull agent image from cdpxlinux acr: ${CDPX_ACR}"
+docker pull ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
+echo "pull image from cdpxlinux acr completed: ${CDPX_ACR}"
+
+echo "CI Release name is:"$CI_RELEASE
+imagetag=$CI_RELEASE$CI_IMAGE_TAG_SUFFIX
+echo "agentimagetag="$imagetag
+
+echo "CI ACR : ${CI_ACR}"
+echo "CI AGENT REPOSITORY NAME : ${CI_AGENT_REPO}"
+
+echo "tag linux agent image"
+docker tag ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+
+echo "login ciprod acr":$CI_ACR
+docker login $CI_ACR --username $ACR_APP_ID --password $ACR_APP_SECRET
+echo "login to ${CI_ACR} acr completed"
+
+echo "pushing the image to ciprod acr:${CI_ACR}"
+docker push ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+echo "pushing the image to ciprod acr completed"
+
+echo "end: pull linux agent image from cdpx and push to ciprod acr"
diff --git a/.pipelines/pull-from-cdpx-and-push-to-ci-acr-windows-image.sh b/.pipelines/pull-from-cdpx-and-push-to-ci-acr-windows-image.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+echo "start: pull windows agent image from cdpx and push to ciprod acr"
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           CDPXACRWindows) CDPX_ACR=$VALUE ;;
+           CDPXWindowsAgentRepositoryName) CDPX_REPO_NAME=$VALUE ;;
+           CDPXWindowsAgentImageTag) CDPX_AGENT_IMAGE_TAG=$VALUE ;;
+           CIACR) CI_ACR=$VALUE ;;
+           CIAgentRepositoryName) CI_AGENT_REPO=$VALUE ;;
+           CIRelease) CI_RELEASE=$VALUE ;;
+           CIImageTagSuffix) CI_IMAGE_TAG_SUFFIX=$VALUE ;;
+
+           *)
+    esac
+done
+
+echo "start: read appid and appsecret"
+ACR_APP_ID=$(cat ./acrappid )
+ACR_APP_SECRET=$(cat ./acrappsecret)
+echo "end: read appid and appsecret"
+
+echo "login to cdpxwindows acr:${CDPX_ACR}"
+docker login $CDPX_ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
+echo "login to cdpxwindows acr:${CDPX_ACR} completed"
+
+echo "pull image from cdpxwin acr: ${CDPX_ACR}"
+docker pull ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
+echo "pull image from cdpxwin acr completed: ${CDPX_ACR}"
+
+echo "CI Release name:"$CI_RELEASE
+echo "CI Image Tax suffix:"$CI_IMAGE_TAG_SUFFIX
+
+imagetag="win-"$CI_RELEASE$CI_IMAGE_TAG_SUFFIX
+echo "agentimagetag="$imagetag
+
+echo "tag windows agent image"
+docker tag ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+
+echo "login to ${CI_ACR} acr"
+docker login $CI_ACR --username $ACR_APP_ID --password $ACR_APP_SECRET
+echo "login to ${CI_ACR} acr completed"
+
+echo "pushing the image to ciprod acr"
+docker push ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+echo "pushing the image to ciprod acr completed"
+
+echo "end: pull windows agent image from cdpx and push to ciprod acr"