diff --git a/.github/workflows/pr-checker.yml b/.github/workflows/pr-checker.yml index f0cea063d..036c7edd3 100644 --- a/.github/workflows/pr-checker.yml +++ b/.github/workflows/pr-checker.yml @@ -56,8 +56,6 @@ jobs: format: 'table' severity: 'CRITICAL,HIGH' vuln-type: 'os,library' - #[vishwa] - Fix telegraf & test all for next release - see work item #https://msazure.visualstudio.com/InfrastructureInsights/_workitems/edit/13322134 - skip-files: '/usr/sbin/telegraf,/opt/telegraf' exit-code: '1' timeout: '5m0s' ignore-unfixed: true @@ -93,7 +91,7 @@ jobs: run: cd ./build/windows/ && & .\Makefile.ps1 - name: Create-docker-image run: | - cd ./kubernetes/windows/ && docker build . --file Dockerfile -t $env:IMAGETAG --build-arg IMAGE_TAG=$env:IMAGETAG_TELEMETRY + cd ./kubernetes/windows/ && docker build . --file Dockerfile -t $env:IMAGETAG --build-arg WINDOWS_VERSION=ltsc2019 --build-arg IMAGE_TAG=$env:IMAGETAG_TELEMETRY - name: List-docker-images run: docker images --digests --all - + diff --git a/.gitignore b/.gitignore index b0467519c..b6e2f8979 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ intermediate *.dll *.obj # ignore docker provider shell bundle -kubernetes/linux/Linux_ULINUX_1.0_x64_64_Release +kubernetes/linux/Linux_ULINUX_1.0_*_64_Release # ignore generated .h files for go source/plugins/go/src/*.h *_mock.go diff --git a/.pipelines/azure_pipeline_dev.yaml b/.pipelines/azure_pipeline_dev.yaml index ba8a530fc..fde666b83 100644 --- a/.pipelines/azure_pipeline_dev.yaml +++ b/.pipelines/azure_pipeline_dev.yaml @@ -4,101 +4,274 @@ # https://aka.ms/yaml trigger: -- ci_dev + batch: true + branches: + include: + - ci_dev -pool: - name: Azure-Pipelines-CI-Test-EO +pr: + branches: + include: + - ci_dev variables: armServiceConnectionName: 'ci-1es-acr-connection' subscription: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' containerRegistry: 'containerinsightsprod' repoImageName: '${{ variables.containerRegistry }}.azurecr.io/public/azuremonitor/containerinsights/cidev' + IS_PR: $[eq(variables['Build.Reason'], 'PullRequest')] -steps: -- bash: | - commit=$(git rev-parse --short HEAD) - echo "##vso[task.setvariable variable=commit;]$commit" - - datetime=$(date +'%Y%m%d%s') - echo "##vso[task.setvariable variable=datetime;]$datetime" - - cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh - - cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment artifacts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/deployment" - Contents: | - **/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/.pipelines" - Contents: | - *.sh - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/kubernetes" - Contents: | - *.yaml - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/charts" - Contents: | - **/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/test/e2e" - Contents: | - *.yaml - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: AzureCLI@2 - displayName: "Docker multi-arch linux build" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - - sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name testbuilder - docker buildx use testbuilder - - az --version - az account show - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - - docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(datetime)-$(commit) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/metadata.json --build-arg IMAGE_TAG=$(datetime)-$(commit) --push . - - docker pull ${{ variables.repoImageName }}:$(datetime)-$(commit) - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generation Task' - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)' - DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:$(datetime)-$(commit)' - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: drop +jobs: +- job: common + pool: + name: Azure-Pipelines-CI-Test-EO + steps: + - bash: | + commit=$(git rev-parse --short HEAD) + datetime=$(date +'%m%d%Y') + linuxImagetag="$datetime"-"$commit" + windowsImageTag=win-"$datetime"-"$commit" + echo "##vso[task.setvariable variable=linuxImagetag;isOutput=true]$linuxImagetag" + echo "##vso[task.setvariable variable=windowsImageTag;isOutput=true]$windowsImageTag" + + cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh + + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh + name: setup + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment artifacts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/deployment" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/.pipelines" + Contents: | + **/*.sh + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/kubernetes" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/charts" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/test/e2e" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop + +- job: build_linux + dependsOn: common + pool: + name: Azure-Pipelines-CI-Test-EO + variables: + linuxImagetag: $[ dependencies.common.outputs['setup.linuxImagetag'] ] + + steps: + - task: AzureCLI@2 + displayName: "Docker multi-arch linux build" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/linux + + sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + docker buildx create --name testbuilder + docker buildx use testbuilder + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + if [ "$(Build.Reason)" != "PullRequest" ]; then + docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) --push . + + docker pull ${{ variables.repoImageName }}:$(linuxImagetag) + else + docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) . + fi + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, true) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' + DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, false) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' + DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:$(linuxImagetag)' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop + +- job: build_windows_2019 + dependsOn: + - common + pool: + name: Azure-Pipelines-Windows-CI-Test-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2019BaseImageVersion: ltsc2019 + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + + - task: AzureCLI@2 + displayName: "Docker windows build for ltsc2019" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) --build-arg WINDOWS_VERSION=$(windows2019BaseImageVersion) --build-arg IMAGE_TAG=$(windowsImageTag) . + if ("$(Build.Reason)" -ne "PullRequest") { + docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) + } + +- job: build_windows_2022 + dependsOn: + - common + pool: + name: Azure-Pipelines-Windows-CI-Test-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2022BaseImageVersion: ltsc2022 + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + + - task: AzureCLI@2 + displayName: "Docker windows build for ltsc2022" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) --build-arg WINDOWS_VERSION=$(windows2022BaseImageVersion) --build-arg IMAGE_TAG=$(windowsImageTag) . + if ("$(Build.Reason)" -ne "PullRequest") { + docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + } + +- job: build_windows_multi_arc + dependsOn: + - common + - build_windows_2019 + - build_windows_2022 + pool: + name: Azure-Pipelines-Windows-CI-Test-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2019BaseImageVersion: ltsc2019 + windows2022BaseImageVersion: ltsc2022 + steps: + - task: AzureCLI@2 + displayName: "Docker windows build for multi-arc image" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + @{"image.name"="${{ variables.repoImageName }}:$(windowsImageTag)"} | ConvertTo-Json -Compress | Out-File -Encoding ascii $(Build.ArtifactStagingDirectory)/windows/metadata.json + + if ("$(Build.Reason)" -ne "PullRequest") { + docker manifest create ${{ variables.repoImageName }}:$(windowsImageTag) ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + docker manifest push ${{ variables.repoImageName }}:$(windowsImageTag) + } + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, true) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' + DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019, mcr.microsoft.com/windows/servercore:ltsc2022' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, false) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' + DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019, mcr.microsoft.com/windows/servercore:ltsc2022, ${{ variables.repoImageName }}:$(windowsImageTag)' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop diff --git a/.pipelines/azure_pipeline_prod.yaml b/.pipelines/azure_pipeline_prod.yaml index 6f5d8bd45..03a14d6c8 100644 --- a/.pipelines/azure_pipeline_prod.yaml +++ b/.pipelines/azure_pipeline_prod.yaml @@ -4,7 +4,15 @@ # https://aka.ms/yaml trigger: -- ci_prod + batch: true + branches: + include: + - ci_prod + +pr: + branches: + include: + - ci_prod pool: name: Azure-Pipelines-CI-Prod-EO @@ -13,92 +21,262 @@ variables: armServiceConnectionName: 'ci-1es-acr-connection-prod' subscription: '30c56c3a-54da-46ea-b004-06eb33432687' containerRegistry: 'containerinsightsbuild' - repoImageName: '${{ variables.containerRegistry }}.azurecr.io/official/linux' - -steps: -- bash: | - commit=$(git rev-parse --short HEAD) - echo "##vso[task.setvariable variable=commit;]$commit" - - datetime=$(date +'%Y%m%d%s') - echo "##vso[task.setvariable variable=datetime;]$datetime" - - cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh - - cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts - tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment artifacts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/deployment" - Contents: | - **/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/.pipelines" - Contents: | - **/*.sh - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/kubernetes" - Contents: | - *.yaml - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/charts" - Contents: | - **/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: CopyFiles@2 - displayName: "Copy ev2 deployment scripts" - inputs: - SourceFolder: "$(Build.SourcesDirectory)/test/e2e" - Contents: | - *.yaml - TargetFolder: '$(Build.ArtifactStagingDirectory)/build' - -- task: AzureCLI@2 - displayName: "Docker multi-arch linux build" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - - sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx create --name testbuilder - docker buildx use testbuilder - - az --version - az account show - az account set -s ${{ variables.subscription }} - az acr login -n ${{ variables.containerRegistry }} - - docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/metadata.json --build-arg IMAGE_TAG=ciprod-$(datetime)-$(commit) --push . - - docker pull ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit) - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generation Task' - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)' - DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit)' - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: drop \ No newline at end of file + repoImageNameLinux: '${{ variables.containerRegistry }}.azurecr.io/official/linux' + repoImageNameWindows: '${{ variables.containerRegistry }}.azurecr.io/official/windows' + IS_PR: $[eq(variables['Build.Reason'], 'PullRequest')] + +jobs: +- job: common + pool: + name: Azure-Pipelines-CI-Prod-EO + steps: + - bash: | + commit=$(git rev-parse --short HEAD) + datetime=$(date +'%m%d%Y') + linuxImagetag=ciprod-"$datetime"-"$commit" + windowsImageTag=win-ciprod-"$datetime"-"$commit" + echo "##vso[task.setvariable variable=linuxImagetag;isOutput=true]$linuxImagetag" + echo "##vso[task.setvariable variable=windowsImageTag;isOutput=true]$windowsImageTag" + + cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh + + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh + name: setup + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment artifacts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/deployment" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/.pipelines" + Contents: | + **/*.sh + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/kubernetes" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/charts" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/test/e2e" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop + +- job: build_linux + dependsOn: common + pool: + name: Azure-Pipelines-CI-Prod-EO + variables: + linuxImagetag: $[ dependencies.common.outputs['setup.linuxImagetag'] ] + + steps: + - task: AzureCLI@2 + displayName: "Docker multi-arch linux build" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/linux + + sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + docker buildx create --name testbuilder + docker buildx use testbuilder + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + if [ "$(Build.Reason)" != "PullRequest" ]; then + docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --push . + + docker pull ${{ variables.repoImageNameLinux }}:$(linuxImagetag) + else + docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json . + fi + + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, true) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' + DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, false) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux' + DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageNameLinux }}:$(linuxImagetag)' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop + +- job: build_windows_2019 + dependsOn: + - common + pool: + name: Azure-Pipelines-Windows-CI-Prod-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2019BaseImageVersion: ltsc2019 + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + + - task: AzureCLI@2 + displayName: "Docker windows build for ltsc2019" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + docker build --isolation=hyperv --tag ${{ variables.repoImageNameWindows }}:$(windowsImageTag)-$(windows2019BaseImageVersion) --build-arg WINDOWS_VERSION=$(windows2019BaseImageVersion) . + if ("$(Build.Reason)" -ne "PullRequest") { + docker push ${{ variables.repoImageNameWindows }}:$(windowsImageTag)-$(windows2019BaseImageVersion) + } + +- job: build_windows_2022 + dependsOn: + - common + pool: + name: Azure-Pipelines-Windows-CI-Prod-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2022BaseImageVersion: ltsc2022 + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + + - task: AzureCLI@2 + displayName: "Docker windows build for ltsc2022" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + docker build --isolation=hyperv --tag ${{ variables.repoImageNameWindows }}:$(windowsImageTag)-$(windows2022BaseImageVersion) --build-arg WINDOWS_VERSION=$(windows2022BaseImageVersion) . + if ("$(Build.Reason)" -ne "PullRequest") { + docker push ${{ variables.repoImageNameWindows }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + } + +- job: build_windows_multi_arc + dependsOn: + - common + - build_windows_2019 + - build_windows_2022 + pool: + name: Azure-Pipelines-Windows-CI-Prod-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2019BaseImageVersion: ltsc2019 + windows2022BaseImageVersion: ltsc2022 + steps: + - task: AzureCLI@2 + displayName: "Docker windows build for multi-arc image" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + @{"image.name"="${{ variables.repoImageNameWindows }}:$(windowsImageTag)"} | ConvertTo-Json -Compress | Out-File -Encoding ascii $(Build.ArtifactStagingDirectory)/windows/metadata.json + + if ("$(Build.Reason)" -ne "PullRequest") { + docker manifest create ${{ variables.repoImageNameWindows }}:$(windowsImageTag) ${{ variables.repoImageNameWindows }}:$(windowsImageTag)-$(windows2019BaseImageVersion) ${{ variables.repoImageNameWindows }}:$(windowsImageTag)-$(windows2022BaseImageVersion) + docker manifest push ${{ variables.repoImageNameWindows }}:$(windowsImageTag) + } + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, true) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' + DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019, mcr.microsoft.com/windows/servercore:ltsc2022' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + condition: eq(variables.IS_PR, false) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)/windows' + DockerImagesToScan: 'mcr.microsoft.com/windows/servercore:ltsc2019, mcr.microsoft.com/windows/servercore:ltsc2022, ${{ variables.repoImageName }}:$(windowsImageTag)' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 000000000..3a8089422 --- /dev/null +++ b/.trivyignore @@ -0,0 +1,16 @@ +# related to telegraf +#[vishwa] - Fix telegraf & test all for next release - see work item #https://msazure.visualstudio.com/InfrastructureInsights/_workitems/edit/13322134 +# Unfixed as of 4/28/2022 +CVE-2019-3826 +CVE-2022-27191 + +#still present in mdsd telegraf +CVE-2021-42836 + +# ruby in /usr/lib +CVE-2020-36327 +CVE-2021-43809 +CVE-2021-41816 +CVE-2021-41819 +CVE-2021-31799 +CVE-2021-28965 \ No newline at end of file diff --git a/Health/Readme.md b/Health/Readme.md deleted file mode 100644 index aed79bd26..000000000 --- a/Health/Readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# Azure Monitor for containers Health(Tab) limited preview -Azure Monitor for containers,Health(Tab) feature provides proactive health monitoring of your Kubernetes cluster to help you identify and diagnose issues. This feature is currently in limited preview, you would need to on-board to this feature manually. If you would like to be part of the limited preview, please reach out to us at [askcoin@microsoft.com](mailto:askcoin@microsoft.com). - -Learn more about the Health(Tab) feature [here](https://aka.ms/cihealthdoc) - -Learn more about how to onboard [here](https://aka.ms/cihealthob) - - diff --git a/Health/documentation.md b/Health/documentation.md deleted file mode 100644 index 6ca547567..000000000 --- a/Health/documentation.md +++ /dev/null @@ -1,93 +0,0 @@ -# Understand Kubernetes cluster health with Azure Monitor for containers - -With Azure Monitor for containers, it monitors and reports health status of the managed infrastructure components and all nodes running on any Kubernetes cluster supported by Azure Monitor for containers. This experience extends beyond the cluster health status calculated and reported on the [multi-cluster view](container-insights-analyze.md#multi-cluster-view-from-azure-monitor), where now you can understand if one or more nodes in the cluster are resource constrained, or a node or pod is unavailable that could impact a running application in the cluster based on curated metrics. - ->[!NOTE] ->The Health feature is only available to customers who are part of limited preview. If you would like to be part of Health limited preview please reach out to [askcoin@microsoft.com]. -> - -For information about how to enable Azure Monitor for containers, see [Onboard Azure Monitor for containers](container-insights-onboard.md). - ->[!NOTE] ->To support AKS Engine clusters, verify it meets the following: ->- It is using the latest version of the [HELM client](https://helm.sh/docs/using_helm/). ->- The containerized agent version is *microsoft/oms:ciprod11012019*. To upgrade the agent, see [upgrading agent on Kubernetes cluster](container-insights-manage-agent.md#how-to-upgrade-the-azure-monitor-for-containers-agent). -> - -## Overview - -In Azure Monitor for containers, the Health (preview) feature provides proactive health monitoring of your Kubernetes cluster to help you identify and diagnose issues. It gives you the ability to view significant issues detected. Monitors evaluating the health of your cluster run on the containerized agent in your cluster, and the health data is written to the **KubeHealth** table in your Log Analytics workspace. - -Kubernetes cluster health is based on a number of monitoring scenarios organized by the following Kubernetes objects and abstractions: - -- Kubernetes infrastructure - provides a rollup of the Kubernetes API server, ReplicaSets, and DaemonSets running on nodes deployed in your cluster by evaluating CPU and memory utilization, and a Pods availability - - ![Kubernetes infrastructure health rollup view](./media/container-insights-health/health-view-kube-infra-01.png) - -- Nodes - provides a rollup of the Node pools and state of individual Nodes in each pool, by evaluating CPU and memory utilization, and a Node's status as reported by Kubernetes. - - ![Nodes health rollup view](./media/container-insights-health/health-view-nodes-01.png) - -Currently, only the status of a virtual kubelet is supported. The health state for CPU and memory utilization of virtual kublet nodes is reported as **Unknown**, since a signal is not received from them. - -All monitors are shown in a hierarchical layout in the Health Hierarchy pane, where an aggregate monitor representing the Kubernetes object or abstraction (that is, Kubernetes infrastructure or Nodes) are the top-most monitor reflecting the combined health of all dependent child monitors. The key monitoring scenarios used to derive health are: - -* Evaluate CPU utilization from the node and container. -* Evaluate memory utilization from the node and container. -* Status of Pods and Nodes based on calculation of their ready state reported by Kubernetes. - -The icons used to indicate state are as follows: - -|Icon|Meaning| -|--------|-----------| -|![Green check icon indicates healthy](./media/container-insights-health/healthyicon.png)|Success, health is OK (green)| -|![Yellow triangle and exclamation mark is warning](./media/container-insights-health/warningicon.png)|Warning (yellow)| -|![Red button with white X indicates critical state](./media/container-insights-health/criticalicon.png)|Critical (red)| -|![Grayed-out icon](./media/container-insights-health/grayicon.png)|Unknown (gray)| - -## Monitor configuration - -To understand the behavior and configuration of each monitor supporting Azure Monitor for containers Health feature, see [Health monitor configuration guide](container-insights-health-monitors-config.md). - -## Sign in to the Azure portal - -Sign in to the [Azure portal](https://portal.azure.com). - -## View health of an AKS or non-AKS cluster - -Access to the Azure Monitor for containers Health (preview) feature is available directly from an AKS cluster by selecting **Insights** from the left pane in the Azure portal. Under the **Insights** section, select **Containers**. - -To view health from a non-AKS cluster, that is an AKS Engine cluster hosted on-premises or on Azure Stack, select **Azure Monitor** from the left pane in the Azure portal. Under the **Insights** section, select **Containers**. On the multi-cluster page, select the non-AKS cluster from the list. - -In Azure Monitor for containers, from the **Cluster** page, select **Health**. - -![Cluster health dashboard example](./media/container-insights-health/container-insights-health-page.png) - -## Review cluster health - -When the Health page opens, by default **Kubernetes Infrastructure** is selected in the **Health Aspect** grid. The grid summarizes current health rollup state of Kubernetes infrastructure and cluster nodes. Selecting either health aspect updates the results in the Health Hierarchy pane (that is, the middle-pane) and shows all child monitors in a hierarchical layout, displaying their current health state. To view more information about any dependent monitor, you can select one and a property pane automatically displays on the right side of the page. - -![Cluster health property pane](./media/container-insights-health/health-view-property-pane.png) - -On the property pane, you learn the following: - -- On the **Overview** tab, it shows the current state of the monitor selected, when the monitor was last calculated, and when the last state change occurred. Additional information is shown depending on the type of monitor selected in the hierarchy. - - If you select an aggregate monitor in the Health Hierarchy pane, under the **Overview** tab on the property pane it shows a rollup of the total number of child monitors in the hierarchy, and how many aggregate monitors are in a critical, warning, and healthy state. - - ![Health property pane Overview tab for aggregate monitor](./media/container-insights-health/health-overview-aggregate-monitor.png) - - If you select a unit monitor in the Health Hierarchy pane, it also shows under **Last state change** the previous samples calculated and reported by the containerized agent within the last four hours. This is based on the unit monitors calculation for comparing several consecutive values to determine its state. For example, if you selected the *Pod ready state* unit monitor, it shows the last two samples controlled by the parameter *ConsecutiveSamplesForStateTransition*. For more information, see the detailed description of [unit monitors](container-insights-health-monitors-config.md#unit-monitors). - - ![Health property pane Overview tab](./media/container-insights-health/health-overview-unit-monitor.png) - - If the time reported by **Last state change** is a day or older, it is the result of no changes in state for the monitor. However, if the last sample received for a unit monitor is more than four hours old, this likely indicates the containerized agent has not been sending data. If the agent knows that a particular resource exists, for example a Node, but it hasn't received data from the Node's CPU or memory utilization monitors (as an example), then the health state of the monitor is set to **Unknown**. - -- On the**Config** tab, it shows the default configuration parameter settings (only for unit monitors, not aggregate monitors) and their values. -- On the **Knowledge** tab, it contains information explaining the behavior of the monitor and how it evaluates for the unhealthy condition. - -Monitoring data on this page does not refresh automatically and you need to select **Refresh** at the top of the page to see the most recent health state received from the cluster. - -## Next steps - -View [log query examples](container-insights-log-search.md#search-logs-to-analyze-data) to see predefined queries and examples to evaluate or customize to alert, visualize, or analyze your clusters. diff --git a/Health/media/container-insights-health/container-insights-health-page.png b/Health/media/container-insights-health/container-insights-health-page.png deleted file mode 100644 index fa27395cb..000000000 Binary files a/Health/media/container-insights-health/container-insights-health-page.png and /dev/null differ diff --git a/Health/media/container-insights-health/criticalicon.png b/Health/media/container-insights-health/criticalicon.png deleted file mode 100644 index 0ccf87fa9..000000000 Binary files a/Health/media/container-insights-health/criticalicon.png and /dev/null differ diff --git a/Health/media/container-insights-health/grayicon.png b/Health/media/container-insights-health/grayicon.png deleted file mode 100644 index e0c494439..000000000 Binary files a/Health/media/container-insights-health/grayicon.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-overview-aggregate-monitor.png b/Health/media/container-insights-health/health-overview-aggregate-monitor.png deleted file mode 100644 index a9f7e5975..000000000 Binary files a/Health/media/container-insights-health/health-overview-aggregate-monitor.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-overview-unit-monitor.png b/Health/media/container-insights-health/health-overview-unit-monitor.png deleted file mode 100644 index 918735274..000000000 Binary files a/Health/media/container-insights-health/health-overview-unit-monitor.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-property-pane-overview.png b/Health/media/container-insights-health/health-property-pane-overview.png deleted file mode 100644 index d11dbf44c..000000000 Binary files a/Health/media/container-insights-health/health-property-pane-overview.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-view-01.png b/Health/media/container-insights-health/health-view-01.png deleted file mode 100644 index 51cf46044..000000000 Binary files a/Health/media/container-insights-health/health-view-01.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-view-kube-infra-01.png b/Health/media/container-insights-health/health-view-kube-infra-01.png deleted file mode 100644 index 6d631792e..000000000 Binary files a/Health/media/container-insights-health/health-view-kube-infra-01.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-view-nodes-01.png b/Health/media/container-insights-health/health-view-nodes-01.png deleted file mode 100644 index 11e574569..000000000 Binary files a/Health/media/container-insights-health/health-view-nodes-01.png and /dev/null differ diff --git a/Health/media/container-insights-health/health-view-property-pane.png b/Health/media/container-insights-health/health-view-property-pane.png deleted file mode 100644 index 226a7c54b..000000000 Binary files a/Health/media/container-insights-health/health-view-property-pane.png and /dev/null differ diff --git a/Health/media/container-insights-health/healthyicon.png b/Health/media/container-insights-health/healthyicon.png deleted file mode 100644 index 098ef7a21..000000000 Binary files a/Health/media/container-insights-health/healthyicon.png and /dev/null differ diff --git a/Health/media/container-insights-health/readme.md b/Health/media/container-insights-health/readme.md deleted file mode 100644 index 8b1378917..000000000 --- a/Health/media/container-insights-health/readme.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Health/media/container-insights-health/warningicon.png b/Health/media/container-insights-health/warningicon.png deleted file mode 100644 index 82c64ec36..000000000 Binary files a/Health/media/container-insights-health/warningicon.png and /dev/null differ diff --git a/Health/onboarding_instructions.md b/Health/onboarding_instructions.md deleted file mode 100644 index 4c83577b5..000000000 --- a/Health/onboarding_instructions.md +++ /dev/null @@ -1,43 +0,0 @@ -# Onboard to Azure Monitor for containers Health(Tab) limited preview - -For on-boarding to Health(Tab), you would need to complete two steps -1. Configure agent through configmap to collect health data. [Learn more about ConfigMap](https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-agent-config#configmap-file-settings-overview) -2. Access Health(Tab) in Azure Monitor for Containers Insights experience in portal with feature flag URL. [aka.ms/HealthPreview](https://aka.ms/Healthpreview) - - -## Configure agent through ConfigMap -1. If you are configuring your existing ConfigMap, append the following section in your existing ConfigMap yaml file -``` -#Append this section in your existing configmap -agent-settings: |- - # agent health model feature settings - [agent_settings.health_model] - # In the absence of this configmap, default value for enabled is false - enabled = true -``` -2. Else if you don't have ConfigMap, download the new ConfigMap from [here.](https://github.com/microsoft/Docker-Provider/blob/ci_prod/kubernetes/container-azm-ms-agentconfig.yaml) & then set `enabled =true` - -``` -#For new downloaded configmap enabled this default setting to true -agent-settings: |- - # agent health model feature settings - [agent_settings.health_model] - # In the absence of this configmap, default value for enabled is false - enabled = true -``` - - -3. Run the following kubectl command: - `kubectl apply -f ` - -Example: `kubectl apply -f container-azm-ms-agentconfig.yaml`. - -The configuration change can take a few minutes to finish before taking effect, and all omsagent pods in the cluster will restart. The restart is a rolling restart for all omsagent pods, not all restart at the same time. - - -## Access health(tab) in Azure Monitor for containers Insights experience -1. You can view Health(tab) by accessing portal through this link. [aka.ms/HealthPreview](https://aka.ms/Healthpreview). This URL includes required feature flag. - - -For any question please reachout to us at [askcoin@microsoft.com](mailto:askcoin@microsoft.com) - diff --git a/README.md b/README.md index f0fa40e53..6e51d256b 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ powershell -ExecutionPolicy bypass # switch to powershell if you are not on pow ``` ##### Developer Build optimizations -If you do not want to build the image from scratch every time you make changes during development,you can choose to build the docker images that are separated out by +If you do not want to build the image from scratch every time you make changes during development,you can choose to build the docker images that are separated out by * Base image and dependencies including agent bootstrap(setup.ps1) * Agent conf and plugin changes @@ -246,12 +246,28 @@ powershell -ExecutionPolicy bypass # switch to powershell if you are not on pow And then run the script to build the image consisting of code and conf changes. ``` .\build-and-publish-dev-docker-image.ps1 -image /: # trigger build code and image and publish docker hub or acr +By default, multi-arc docker image will be built, but if you want generate test image either with ltsc2019 or ltsc2022 base image, then you can follow the instructions below + +For building image with base image version ltsc2019 +.\build-and-publish-dev-docker-image.ps1 -image /: -windowsBaseImageVersion ltsc2019 + +For building image with base image version ltsc2022 +.\build-and-publish-dev-docker-image.ps1 -image /: -windowsBaseImageVersion ltsc2022 + + ``` -For the subsequent builds, you can just run - +For the subsequent builds, you can just run - ``` .\build-and-publish-dev-docker-image.ps1 -image /: # trigger build code and image and publish docker hub or acr +By default, multi-arc docker image will be built, but if you want generate test image either with ltsc2019 or ltsc2022 base image, then you can follow the instructions below + +For building image with base image version ltsc2019 +.\build-and-publish-dev-docker-image.ps1 -image /: -windowsBaseImageVersion ltsc2019 + +For building image with base image version ltsc2022 +.\build-and-publish-dev-docker-image.ps1 -image /: -windowsBaseImageVersion ltsc2022 ``` ###### Note - If you have changes in setup.ps1 and want to test those changes, uncomment the section consisting of setup.ps1 in the Dockerfile-dev-image file. @@ -316,18 +332,31 @@ Navigate to Kubernetes directory and update the yamls with latest docker image o For DEV and PROD branches, automatically deployed latest yaml with latest agent image (which automatically built by the azure devops pipeline) onto CIDEV and CIPROD AKS clusters in build subscription. So, you can use CIDEV and CIPROD AKS cluster to validate E2E. Similarly, you can set up build and release pipelines for your feature branch. +# Testing MSI Auth Mode Using Yaml + + 1. Enable Monitoring addon with Managed Idenity Auth Mode either using Portal or CLI or Template + 2. Deploy [ARM template](./scripts/onboarding/aks/onboarding-using-msi-auth/) with enabled = false to create DCR, DCR-A and link the workspace to Portal + > Note - Make sure to update the parameter values in existingClusterParam.json file and have enabled = false in template file + `az deployment group create --resource-group --template-file ./existingClusterOnboarding.json --parameters @./existingClusterParam.json` + 3. Get the MSI token (which is valid for 24 hrs.) value via `kubectl get secrets -n kube-system omsagent-aad-msi-token -o=jsonpath='{.data.token}'` + 4. Disable Monitoring addon via `az aks disable-addons -a monitoring -g -n ` + 5. Uncomment MSI auth related yaml lines, replace all the placeholder values, MSI token value and image tag in the omsagent.yaml + 6. Deploy the omsagent.yaml via `kubectl apply -f omsagent.yaml` + > Note: use the image toggle for release E2E validation + 7. validate E2E for LA & Metrics data flows, and other scenarios + # E2E Tests ## For executing tests 1. Deploy the omsagent.yaml with your agent image. In the yaml, make sure `ISTEST` environment variable set to `true` if its not set already -2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests +2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests > Note: Service Principal requires reader role on log analytics workspace and cluster resource to query LA and metrics ``` - cd ~/Docker-Provider/test/e2e # based on your repo path - kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace - kubectl get po -n sonobuoy # to check the pods and jobs associated to tests - ``` + cd ~/Docker-Provider/test/e2e # based on your repo path + kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace + kubectl get po -n sonobuoy # to check the pods and jobs associated to tests + ``` 3. Download (sonobuoy)[https://github.com/vmware-tanzu/sonobuoy/releases] on your dev box to view the results of the tests ``` results=$(sonobuoy retrieve) # downloads tar file which has logs and test results @@ -338,9 +367,9 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent ## For adding new tests 1. Add the test python file with your test code under `tests` directory -2. Build the docker image, recommended to use ACR & MCR +2. Build the docker image, recommended to use ACR & MCR ``` - cd ~/Docker-Provider/test/e2e/src # based on your repo path + cd ~/Docker-Provider/test/e2e/src # based on your repo path docker login -u -p # login to acr docker build -f ./core/Dockerfile -t /: . docker push /: diff --git a/Rakefile b/Rakefile deleted file mode 100644 index 4ac2b1137..000000000 --- a/Rakefile +++ /dev/null @@ -1,9 +0,0 @@ -require "rake/testtask" - -task default: "test" - -Rake::TestTask.new do |task| - task.libs << "test" - task.pattern = "./test/unit-tests/plugins/health/*_spec.rb" - task.warning = false -end diff --git a/ReleaseNotes.md b/ReleaseNotes.md index d0e5bad22..176cbc2b8 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -11,6 +11,45 @@ additional questions or comments. Note : The agent version(s) below has dates (ciprod), which indicate the agent build dates (not release dates) +### 5/19/2022 - +##### Version microsoft/oms:ciprod05192022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05192022 (linux) +##### Version microsoft/oms:win-ciprod05192022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod05192022 (windows) +##### Code change log +- Linux Agent + - PodReadyPercentage metric bug fix + - add cifs & fuse file systems to ignore list + - CA Cert Fix for Mariner Hosts in Air Gap + - Disk usage metrics will no longer be collected for the paths "/mnt/containers" and "/mnt/docker" +- Windows Agent + - Ruby version upgrade from 2.6.5.1 to 2.7.5.1 + - Added Support for Windows Server 2022 + - Multi-Arch Image to support both Windows 2019 and Windows 2022 +- Common (Linux & Windows Agent) + - Telegraf version update from 1.20.3 to 1.22.2 to fix the vulnerabilitis + - Removal of Health feature as part of deprecation plan + - AAD Auth MSI feature support for Arc K8s (not usable externally yet) + - MSI onboarding ARM template updates for both AKS & Arc K8s + - Fixed the bug related to windows metrics in MSI mode for AKS + - Configmap updates for log collection settings for v2 schema +- Misc + - Improvements related to CI/CD Multi-arc image + - Do trivy rootfs checks + - Disable push to ACR for PR and PR updates + - Enable batch builds + - Scope Dev/Prod pipelines to respective branches + - Shorten datetime component of image tag + - Troubleshooting script updates for MSI onboarding + - Instructions for testing of agent in MSI auth mode + - Add CI Windows Build to MultiArch Dev pipeline + - Updates related to building of Multi-arc image for windows in Build Pipeline and local dev builds + - Test yamls to test container logs and prometheus scraping on both WS2019 & WS2022 + - Arc K8s conformance test updates + - Script to collect the Agent logs for troubleshooting + - Force run trivy stage for Linux + - Fix docker msi download link in windows install-build-pre-requisites.ps1 script + - Added Onboarding templates for legacy auth for internal testing + - Update the Build pipelines to have separate phase for Windows + ### 3/17/2022 - ##### Version microsoft/oms:ciprod03172022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022 (linux) ##### Version microsoft/oms:win-ciprod03172022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03172022 (windows) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..766e6f887 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/msrc/pgp-key-msrc). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/msrc/cvd). + + diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb index 052bb5a5d..ebe1e3982 100644 --- a/build/common/installer/scripts/tomlparser-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-agent-config.rb @@ -12,7 +12,6 @@ @configMapMountPath = "/etc/config/settings/agent-settings" @configSchemaVersion = "" -@enable_health_model = false # 250 Node items (15KB per node) account to approximately 4MB @nodesChunkSize = 250 @@ -89,10 +88,6 @@ def parseConfigMap def populateSettingValuesFromConfigMap(parsedConfig) begin if !parsedConfig.nil? && !parsedConfig[:agent_settings].nil? - if !parsedConfig[:agent_settings][:health_model].nil? && !parsedConfig[:agent_settings][:health_model][:enabled].nil? - @enable_health_model = parsedConfig[:agent_settings][:health_model][:enabled] - puts "enable_health_model = #{@enable_health_model}" - end chunk_config = parsedConfig[:agent_settings][:chunk_config] if !chunk_config.nil? nodesChunkSize = chunk_config[:NODES_CHUNK_SIZE] @@ -179,7 +174,6 @@ def populateSettingValuesFromConfigMap(parsedConfig) end rescue => errorStr puts "config::error:Exception while reading config settings for agent configuration setting - #{errorStr}, using defaults" - @enable_health_model = false end end @@ -194,14 +188,12 @@ def populateSettingValuesFromConfigMap(parsedConfig) if (File.file?(@configMapMountPath)) ConfigParseErrorLogger.logError("config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version") end - @enable_health_model = false end # Write the settings to file, so that they can be set as environment variables file = File.open("agent_config_env_var", "w") if !file.nil? - file.write("export AZMON_CLUSTER_ENABLE_HEALTH_MODEL=#{@enable_health_model}\n") file.write("export NODES_CHUNK_SIZE=#{@nodesChunkSize}\n") file.write("export PODS_CHUNK_SIZE=#{@podsChunkSize}\n") file.write("export EVENTS_CHUNK_SIZE=#{@eventsChunkSize}\n") diff --git a/build/common/installer/scripts/tomlparser-prom-agent-config.rb b/build/common/installer/scripts/tomlparser-prom-agent-config.rb index be9d08e59..664691a44 100644 --- a/build/common/installer/scripts/tomlparser-prom-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-prom-agent-config.rb @@ -84,7 +84,6 @@ def populateSettingValuesFromConfigMap(parsedConfig) if (File.file?(@configMapMountPath)) ConfigParseErrorLogger.logError("config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version") end - @enable_health_model = false end # Write the settings to file, so that they can be set as environment variables diff --git a/build/linux/installer/conf/container.conf b/build/linux/installer/conf/container.conf index 093c9ef12..438c0891d 100644 --- a/build/linux/installer/conf/container.conf +++ b/build/linux/installer/conf/container.conf @@ -33,16 +33,6 @@ @log_level debug - - @type cadvisor_health_node - @log_level debug - - - - @type cadvisor_health_container - @log_level debug - - #custom_metrics_mdm filter plugin @type cadvisor2mdm @@ -107,34 +97,6 @@ keepalive true - - @type health_forward - send_timeout 60s - recover_wait 10s - hard_timeout 60s - transport tcp - ignore_network_errors_at_startup true - expire_dns_cache 600s - - @type file - overflow_action drop_oldest_chunk - path /var/opt/microsoft/docker-cimprov/state/out_health_forward*.buffer - chunk_limit_size 3m - flush_interval 20s - retry_max_times 10 - retry_max_interval 5m - retry_wait 5s - - - host "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_HOST']}" - port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" - - - @type file - path /var/opt/microsoft/docker-cimprov/state/fluent_forward_failed.buffer - - - @type mdm @log_level debug diff --git a/build/linux/installer/conf/kube.conf b/build/linux/installer/conf/kube.conf index a1c8bf928..53040e2f9 100644 --- a/build/linux/installer/conf/kube.conf +++ b/build/linux/installer/conf/kube.conf @@ -1,11 +1,3 @@ - #fluent forward plugin - - @type forward - port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" - bind 0.0.0.0 - chunk_size_limit 4m - - #Kubernetes pod inventory @type kube_podinventory @@ -38,14 +30,6 @@ @log_level debug - #Kubernetes health - - @type kube_health - tag kubehealth.ReplicaSet - run_interval 60 - @log_level debug - - #cadvisor perf- Windows nodes @type win_cadvisor_perf @@ -82,11 +66,6 @@ @log_level info - #health model aggregation filter - - @type health_model_builder - - #kubepodinventory @type forward @@ -357,29 +336,3 @@ retry_mdm_post_wait_minutes 30 - - #kubehealth - - @type forward - @log_level debug - send_timeout 30 - connect_timeout 30 - heartbeat_type none - - host 0.0.0.0 - port "#{ENV['MDSD_FLUENT_SOCKET_PORT']}" - - - @type file - path /var/opt/microsoft/docker-cimprov/state/kubehealth*.buffer - overflow_action drop_oldest_chunk - chunk_limit_size 4m - queue_limit_length 20 - flush_interval 20s - retry_max_times 10 - retry_wait 5s - retry_max_interval 5m - flush_thread_count 5 - - keepalive true - diff --git a/build/linux/installer/conf/telegraf.conf b/build/linux/installer/conf/telegraf.conf index 9f213e3e8..6ee1c472b 100644 --- a/build/linux/installer/conf/telegraf.conf +++ b/build/linux/installer/conf/telegraf.conf @@ -445,7 +445,7 @@ # mount_points = ["/"] ## Ignore mount points by filesystem type. - ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs", "cifs", "fuse"] fieldpass = ["free", "used", "used_percent"] taginclude = ["device","path","hostName"] # Below due to Bug - https://github.com/influxdata/telegraf/issues/5615 diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index 9fc7ce08f..7b176d030 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -57,10 +57,6 @@ MAINTAINER: 'Microsoft Corporation' /opt/test.json; build/linux/installer/conf/test.json; 644; root; root - -/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json; build/linux/installer/conf/healthmonitorconfig.json; 644; root; root -/etc/opt/microsoft/docker-cimprov/health/health_model_definition.json; build/linux/installer/conf/health_model_definition.json; 644; root; root - /etc/fluent/plugin/lib/application_insights/version.rb; source/plugins/ruby/lib/application_insights/version.rb; 644; root; root /etc/fluent/plugin/lib/application_insights/rack/track_request.rb; source/plugins/ruby/lib/application_insights/rack/track_request.rb; 644; root; root /etc/fluent/plugin/lib/application_insights/unhandled_exception.rb; source/plugins/ruby/lib/application_insights/unhandled_exception.rb; 644; root; root @@ -106,32 +102,6 @@ MAINTAINER: 'Microsoft Corporation' /etc/fluent/plugin/lib/application_insights/channel/event.rb; source/plugins/ruby/lib/application_insights/channel/event.rb; 644; root; root /etc/fluent/plugin/lib/application_insights.rb; source/plugins/ruby/lib/application_insights.rb; 644; root; root -/etc/fluent/plugin/health/aggregate_monitor.rb; source/plugins/ruby/health/aggregate_monitor.rb; 644; root; root -/etc/fluent/plugin/health/agg_monitor_id_labels.rb; source/plugins/ruby/health/agg_monitor_id_labels.rb; 644; root; root -/etc/fluent/plugin/health/aggregate_monitor_state_finalizer.rb; source/plugins/ruby/health/aggregate_monitor_state_finalizer.rb; 644; root; root -/etc/fluent/plugin/health/cluster_health_state.rb; source/plugins/ruby/health/cluster_health_state.rb; 644; root; root -/etc/fluent/plugin/health/health_container_cpu_memory_aggregator.rb; source/plugins/ruby/health/health_container_cpu_memory_aggregator.rb; 644; root; root -/etc/fluent/plugin/health/health_container_cpu_memory_record_formatter.rb; source/plugins/ruby/health/health_container_cpu_memory_record_formatter.rb; 644; root; root -/etc/fluent/plugin/health/health_hierarchy_builder.rb; source/plugins/ruby/health/health_hierarchy_builder.rb; 644; root; root -/etc/fluent/plugin/health/health_kubernetes_resources.rb; source/plugins/ruby/health/health_kubernetes_resources.rb; 644; root; root -/etc/fluent/plugin/health/health_kube_api_down_handler.rb; source/plugins/ruby/health/health_kube_api_down_handler.rb; 644; root; root -/etc/fluent/plugin/health/health_missing_signal_generator.rb; source/plugins/ruby/health/health_missing_signal_generator.rb; 644; root; root -/etc/fluent/plugin/health/health_model_buffer.rb; source/plugins/ruby/health/health_model_buffer.rb; 644; root; root -/etc/fluent/plugin/health/health_model_builder.rb; source/plugins/ruby/health/health_model_builder.rb; 644; root; root -/etc/fluent/plugin/health/health_model_constants.rb; source/plugins/ruby/health/health_model_constants.rb; 644; root; root -/etc/fluent/plugin/health/parent_monitor_provider.rb; source/plugins/ruby/health/parent_monitor_provider.rb; 644; root; root -/etc/fluent/plugin/health/health_model_definition_parser.rb; source/plugins/ruby/health/health_model_definition_parser.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_helpers.rb; source/plugins/ruby/health/health_monitor_helpers.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_optimizer.rb; source/plugins/ruby/health/health_monitor_optimizer.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_provider.rb; source/plugins/ruby/health/health_monitor_provider.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_record.rb; source/plugins/ruby/health/health_monitor_record.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_state.rb; source/plugins/ruby/health/health_monitor_state.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_telemetry.rb; source/plugins/ruby/health/health_monitor_telemetry.rb; 644; root; root -/etc/fluent/plugin/health/health_monitor_utils.rb; source/plugins/ruby/health/health_monitor_utils.rb; 644; root; root -/etc/fluent/plugin/health/health_signal_reducer.rb; source/plugins/ruby/health/health_signal_reducer.rb; 644; root; root -/etc/fluent/plugin/health/monitor_factory.rb; source/plugins/ruby/health/monitor_factory.rb; 644; root; root -/etc/fluent/plugin/health/monitor_set.rb; source/plugins/ruby/health/monitor_set.rb; 644; root; root -/etc/fluent/plugin/health/unit_monitor.rb; source/plugins/ruby/health/unit_monitor.rb; 644; root; root /etc/fluent/plugin/ApplicationInsightsUtility.rb; source/plugins/ruby/ApplicationInsightsUtility.rb; 644; root; root /etc/fluent/plugin/arc_k8s_cluster_identity.rb; source/plugins/ruby/arc_k8s_cluster_identity.rb; 644; root; root @@ -164,19 +134,14 @@ MAINTAINER: 'Microsoft Corporation' /etc/fluent/plugin/in_kube_podinventory.rb; source/plugins/ruby/in_kube_podinventory.rb; 644; root; root /etc/fluent/plugin/KubernetesApiClient.rb; source/plugins/ruby/KubernetesApiClient.rb; 644; root; root /etc/fluent/plugin/in_kube_events.rb; source/plugins/ruby/in_kube_events.rb; 644; root; root -/etc/fluent/plugin/in_kube_health.rb; source/plugins/ruby/in_kube_health.rb; 644; root; root /etc/fluent/plugin/in_kube_pvinventory.rb; source/plugins/ruby/in_kube_pvinventory.rb; 644; root; root /etc/fluent/plugin/in_kubestate_deployments.rb; source/plugins/ruby/in_kubestate_deployments.rb; 644; root; root /etc/fluent/plugin/in_kubestate_hpa.rb; source/plugins/ruby/in_kubestate_hpa.rb; 644; root; root -/etc/fluent/plugin/filter_cadvisor_health_container.rb; source/plugins/ruby/filter_cadvisor_health_container.rb; 644; root; root -/etc/fluent/plugin/filter_cadvisor_health_node.rb; source/plugins/ruby/filter_cadvisor_health_node.rb; 644; root; root /etc/fluent/plugin/filter_cadvisor2mdm.rb; source/plugins/ruby/filter_cadvisor2mdm.rb; 644; root; root -/etc/fluent/plugin/filter_health_model_builder.rb; source/plugins/ruby/filter_health_model_builder.rb; 644; root; root /etc/fluent/plugin/filter_inventory2mdm.rb; source/plugins/ruby/filter_inventory2mdm.rb; 644; root; root /etc/fluent/plugin/filter_telegraf2mdm.rb; source/plugins/ruby/filter_telegraf2mdm.rb; 644; root; root -/etc/fluent/plugin/out_health_forward.rb; source/plugins/ruby/out_health_forward.rb; 644; root; root /etc/fluent/plugin/out_mdm.rb; source/plugins/ruby/out_mdm.rb; 644; root; root @@ -195,7 +160,6 @@ MAINTAINER: 'Microsoft Corporation' /etc/opt/microsoft; 755; root; root; sysdir /etc/opt/microsoft/docker-cimprov; 755; root; root /etc/opt/microsoft/docker-cimprov/conf; 755; root; root -/etc/opt/microsoft/docker-cimprov/health; 755; root; root /opt/microsoft; 755; root; root; sysdir /opt/microsoft/docker-cimprov; 755; root; root @@ -217,7 +181,6 @@ MAINTAINER: 'Microsoft Corporation' /etc/fluent; 755; root; root; sysdir /etc/fluent/plugin; 755; root; root; sysdir -/etc/fluent/plugin/health; 755; root; root; sysdir /etc/fluent/plugin/lib; 755; root; root; sysdir /etc/fluent/plugin/lib/application_insights; 755; root; root; sysdir /etc/fluent/plugin/lib/application_insights/channel; 755; root; root; sysdir @@ -270,15 +233,6 @@ chmod 666 /var/opt/microsoft/docker-cimprov/log/filter_inventory2mdm.log touch /var/opt/microsoft/docker-cimprov/log/mdm_metrics_generator.log chmod 666 /var/opt/microsoft/docker-cimprov/log/mdm_metrics_generator.log - -touch /var/opt/microsoft/docker-cimprov/log/health_monitors.log -chmod 666 /var/opt/microsoft/docker-cimprov/log/health_monitors.log - - -touch /var/opt/microsoft/docker-cimprov/log/filter_health_model_builder.log -chmod 666 /var/opt/microsoft/docker-cimprov/log/filter_health_model_builder.log - - touch /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log chmod 666 /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log diff --git a/build/version b/build/version index 95d20e931..19787cb72 100644 --- a/build/version +++ b/build/version @@ -2,11 +2,11 @@ # Build Version Information -CONTAINER_BUILDVERSION_MAJOR=16 +CONTAINER_BUILDVERSION_MAJOR=17 CONTAINER_BUILDVERSION_MINOR=0 CONTAINER_BUILDVERSION_PATCH=0 CONTAINER_BUILDVERSION_BUILDNR=0 -CONTAINER_BUILDVERSION_DATE=20210611 +CONTAINER_BUILDVERSION_DATE=20220519 CONTAINER_BUILDVERSION_STATUS=Developer_Build #-------------------------------- End of File ----------------------------------- diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index 0ff1e3387..8e9f4847f 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.9.2 +version: 2.9.3 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml index b7482b8b5..c61d4b83c 100644 --- a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -19,8 +19,16 @@ metadata: name: container-insights-clusteridentityrequest namespace: azure-arc spec: - audience: https://monitoring.azure.com/ + {{- if eq (.Values.Azure.Cluster.Cloud | lower) "azurepubliccloud" }} + audience: https://monitor.azure.com/ + {{- else if eq (.Values.Azure.Cluster.Cloud | lower) "azurechinacloud" }} + audience: https://monitor.azure.cn/ + {{- else if eq (.Values.Azure.Cluster.Cloud | lower) "azureusgovernmentcloud" }} + audience: https://monitor.azure.us/ + {{- else }} + audience: https://monitor.azure.com/ + {{- end }} {{- if not (empty .Values.Azure.Extension.Name) }} - resourceId: {{ .Values.Azure.Extension.Name }} + resourceId: {{ .Values.Azure.Extension.Name }} {{- end }} {{- end }} diff --git a/charts/azuremonitor-containers/templates/omsagent-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-crd.yaml deleted file mode 100644 index 46c5341cc..000000000 --- a/charts/azuremonitor-containers/templates/omsagent-crd.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if semverCompare "<1.19-0" .Capabilities.KubeVersion.Version }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: healthstates.azmon.container.insights - namespace: kube-system -spec: - group: azmon.container.insights - version: v1 - scope: Namespaced - names: - plural: healthstates - kind: HealthState -{{- else }} -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: healthstates.azmon.container.insights - namespace: kube-system -spec: - group: azmon.container.insights - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - state: - type: string - scope: Namespaced - names: - plural: healthstates - kind: HealthState -{{- end }} \ No newline at end of file diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml index b581a324a..ef72b385b 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml @@ -69,6 +69,8 @@ spec: {{- else if ne .Values.Azure.Cluster.ResourceId "" }} - name: AKS_RESOURCE_ID value: {{ .Values.Azure.Cluster.ResourceId | quote }} + - name: USING_AAD_MSI_AUTH + value: {{ .Values.omsagent.useAADAuth | quote }} {{- if ne .Values.Azure.Cluster.Region "" }} - name: AKS_REGION value: {{ .Values.Azure.Cluster.Region | quote }} diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index 3b48c26c4..5bd8bdf79 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -37,6 +37,16 @@ spec: serviceAccountName: omsagent {{- end }} containers: +{{- if and (ne .Values.Azure.Cluster.ResourceId "") (.Values.omsagent.useAADAuth) }} + - name: addon-token-adapter + imagePullPolicy: IfNotPresent + env: + - name: AZMON_COLLECT_ENV + value: "false" + - name: TOKEN_NAMESPACE + value: "azure-arc" +{{- .Values.Azure.Identity.MSIAdapterYaml | nindent 7 }} +{{- end }} - name: omsagent {{- if eq (.Values.omsagent.domain | lower) "opinsights.azure.cn" }} image: "mcr.azk8s.cn/azuremonitor/containerinsights/ciprod:{{ .Values.omsagent.image.tag }}" @@ -57,6 +67,8 @@ spec: {{- else if ne .Values.Azure.Cluster.ResourceId "" }} - name: AKS_RESOURCE_ID value: {{ .Values.Azure.Cluster.ResourceId | quote }} + - name: USING_AAD_MSI_AUTH + value: {{ .Values.omsagent.useAADAuth | quote }} {{- if ne .Values.Azure.Cluster.Region "" }} - name: AKS_REGION value: {{ .Values.Azure.Cluster.Region | quote }} @@ -159,6 +171,8 @@ spec: {{- else if ne .Values.Azure.Cluster.ResourceId "" }} - name: AKS_RESOURCE_ID value: {{ .Values.Azure.Cluster.ResourceId | quote }} + - name: USING_AAD_MSI_AUTH + value: {{ .Values.omsagent.useAADAuth | quote }} {{- if ne .Values.Azure.Cluster.Region "" }} - name: AKS_REGION value: {{ .Values.Azure.Cluster.Region | quote }} diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index 55b1f4a8d..a0abb0f57 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -33,6 +33,16 @@ spec: serviceAccountName: omsagent {{- end }} containers: +{{- if and (ne .Values.Azure.Cluster.ResourceId "") (.Values.omsagent.useAADAuth) }} + - name: addon-token-adapter + imagePullPolicy: IfNotPresent + env: + - name: AZMON_COLLECT_ENV + value: "false" + - name: TOKEN_NAMESPACE + value: "azure-arc" +{{- .Values.Azure.Identity.MSIAdapterYaml | nindent 7 }} +{{- end }} - name: omsagent {{- if eq (.Values.omsagent.domain | lower) "opinsights.azure.cn" }} image: "mcr.azk8s.cn/azuremonitor/containerinsights/ciprod:{{ .Values.omsagent.image.tag }}" @@ -53,6 +63,8 @@ spec: {{- else if ne .Values.Azure.Cluster.ResourceId "" }} - name: AKS_RESOURCE_ID value: {{ .Values.Azure.Cluster.ResourceId | quote }} + - name: USING_AAD_MSI_AUTH + value: {{ .Values.omsagent.useAADAuth | quote }} {{- if ne .Values.Azure.Cluster.Region "" }} - name: AKS_REGION value: {{ .Values.Azure.Cluster.Region | quote }} @@ -84,9 +96,6 @@ spec: protocol: TCP - containerPort: 25224 protocol: UDP - - containerPort: 25227 - protocol: TCP - name: in-rs-tcp volumeMounts: - mountPath: /var/run/host name: docker-sock diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index d9bca069d..f8d33df97 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -28,13 +28,9 @@ rules: - apiGroups: ["apps", "extensions", "autoscaling"] resources: ["replicasets", "deployments", "horizontalpodautoscalers"] verbs: ["list"] -- apiGroups: ["azmon.container.insights"] - resources: ["healthstates"] - verbs: ["get", "create", "patch"] - apiGroups: ["clusterconfig.azure.com"] resources: ["azureclusteridentityrequests", "azureclusteridentityrequests/status"] - resourceNames: ["container-insights-clusteridentityrequest"] - verbs: ["get", "create", "patch"] + verbs: ["get", "create", "patch", "list", "update", "delete"] - nonResourceURLs: ["/metrics"] verbs: ["get"] #arc k8s extension model grants access as part of the extension msi diff --git a/charts/azuremonitor-containers/templates/omsagent-rs-configmap.yaml b/charts/azuremonitor-containers/templates/omsagent-rs-configmap.yaml index fc7c471f8..7fa85bfdc 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rs-configmap.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rs-configmap.yaml @@ -4,13 +4,6 @@ apiVersion: v1 data: kube.conf: | # Fluentd config file for OMS Docker - cluster components (kubeAPI) - #fluent forward plugin - - type forward - port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" - bind 0.0.0.0 - chunk_size_limit 4m - #Kubernetes pod inventory @@ -44,14 +37,6 @@ data: log_level debug - #Kubernetes health - - type kubehealth - tag kubehealth.ReplicaSet - run_interval 60 - log_level debug - - #cadvisor perf- Windows nodes type wincadvisorperf @@ -87,11 +72,6 @@ data: log_level info - #health model aggregation filter - - type filter_health_model_builder - - type out_oms log_level debug @@ -243,21 +223,6 @@ data: retry_mdm_post_wait_minutes 30 - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubehealth*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - type out_oms log_level debug diff --git a/charts/azuremonitor-containers/templates/omsagent-service.yaml b/charts/azuremonitor-containers/templates/omsagent-service.yaml deleted file mode 100644 index 00e6a1d3b..000000000 --- a/charts/azuremonitor-containers/templates/omsagent-service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: healthmodel-replicaset-service - namespace: kube-system -spec: - selector: - rsName: "omsagent-rs" - ports: - - protocol: TCP - port: 25227 - targetPort: in-rs-tcp diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 4460f7756..480e7040c 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -7,6 +7,7 @@ ## Values of under Azure are being populated by Azure Arc K8s RP during the installation of the extension Azure: Cluster: + Cloud: Region: ResourceId: Extension: @@ -21,10 +22,10 @@ Azure: omsagent: image: repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" - tag: "ciprod03172022" - tagWindows: "win-ciprod03172022" + tag: "ciprod05192022" + tagWindows: "win-ciprod05192022" pullPolicy: IfNotPresent - dockerProviderVersion: "16.0.0-0" + dockerProviderVersion: "17.0.0-0" agentVersion: "azure-mdsd-1.17.0" winAgentVersion: "0.0.0-0" # there is no base agent version for windows agent @@ -45,6 +46,9 @@ omsagent: # if set to true additional agent workflow logs will be emitted which are used for e2e and arc k8s conformance testing ISTEST: false + # This flag used to determine whether to use AAD MSI auth or not for Arc K8s cluster + useAADAuth: false + ## To get your workspace id and key do the following ## You can create a Azure Loganalytics workspace from portal.azure.com and get its ID & PRIMARY KEY from 'Advanced Settings' tab in the Ux. diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/Parameters/ContainerInsights.Windows.Parameters.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Parameters/ContainerInsights.Windows.Parameters.json new file mode 100644 index 000000000..cfa945e5d --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Parameters/ContainerInsights.Windows.Parameters.json @@ -0,0 +1,68 @@ +{ + "$schema": "http://schema.express.azure.com/schemas/2015-01-01-alpha/RolloutParameters.json", + "contentVersion": "1.0.0.0", + "wait": [ + { + "name": "waitSdpBakeTime", + "properties": { + "duration": "PT24H" + } + } + ], + "shellExtensions": [ + { + "name": "PushAgentToACR", + "type": "ShellExtensionType", + "properties": { + "maxexecutiontime": "PT1H" + }, + "package": { + "reference": { + "path": "artifacts.tar.gz" + } + }, + "launch": { + "command": [ + "/bin/bash", + "pushAgentToAcr.sh" + ], + "environmentVariables": [ + { + "name": "ACR_NAME", + "value": "__ACR_NAME__" + }, + { + "name": "AGENT_RELEASE", + "value": "__AGENT_RELEASE__" + }, + { + "name": "AGENT_IMAGE_TAG_SUFFIX", + "value": "__AGENT_IMAGE_TAG_SUFFIX__" + }, + { + "name": "AGENT_IMAGE_FULL_PATH", + "value": "public/azuremonitor/containerinsights/__AGENT_RELEASE__:win-__AGENT_RELEASE____AGENT_IMAGE_TAG_SUFFIX__" + }, + { + "name": "CDPX_REGISTRY", + "value": "__CDPX_WINDOWS_REGISTRY__" + }, + { + "name": "CDPX_REPO_NAME", + "value": "__CDPX_WINDOWS_REPO_NAME__" + }, + { + "name": "CDPX_TAG", + "value": "__CDPX_WINDOWS_TAG__" + } + ], + "identity": { + "type": "userAssigned", + "userAssignedIdentities": [ + "__MANAGED_IDENTITY__" + ] + } + } + } + ] + } \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json index 250878590..91bc42ba1 100644 --- a/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json @@ -2,8 +2,8 @@ "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json", "ContentVersion": "1.0.0.0", "RolloutMetadata": { - "ServiceModelPath": "ServiceModels//Public.ServiceModel.json", - "ScopeBindingsPath": "ScopeBindings//Public.ScopeBindings.json", + "ServiceModelPath": "ServiceModels//Public.ServiceModel.json", + "ScopeBindingsPath": "ScopeBindings//Public.ScopeBindings.json", "Name": "ContainerInsightsAgent", "RolloutType": "Major", "BuildSource": { @@ -15,7 +15,7 @@ "Email": { "To": "omscontainers@microsoft.com" } - } + } }, "OrchestratedSteps": [ { @@ -24,6 +24,13 @@ "targetName": "PushLinuxAgent", "actions": [ "Shell/PushAgentToACR" ], "dependsOn": [ ] + }, + { + "name": "PushWindowsAgent", + "targetType": "ServiceResource", + "targetName": "PushWindowsAgent", + "actions": [ "Shell/PushAgentToACR" ], + "dependsOn": [ ] } ] } \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh index d39cedde0..2738e110e 100644 --- a/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh @@ -19,7 +19,14 @@ if [ $? -ne 0 ]; then echo "-e error unable to get list of mcr tags for azuremonitor/containerinsights/ciprod repository" exit 1 fi -TAG_EXISTS=$(echo $MCR_TAG_RESULT | jq '.tags | contains(["'"$AGENT_RELEASE$AGENT_IMAGE_TAG_SUFFIX"'"])') + +if [[ "$AGENT_IMAGE_FULL_PATH" == *"win-"* ]]; then + echo "checking windows tags" + TAG_EXISTS=$(echo $MCR_TAG_RESULT | jq '.tags | contains(["'"win-$AGENT_RELEASE$AGENT_IMAGE_TAG_SUFFIX"'"])') +else + echo "checking linux tags" + TAG_EXISTS=$(echo $MCR_TAG_RESULT | jq '.tags | contains(["'"$AGENT_RELEASE$AGENT_IMAGE_TAG_SUFFIX"'"])') +fi if $TAG_EXISTS; then echo "-e error ${AGENT_IMAGE_TAG_SUFFIX} already exists in mcr. make sure the image tag is unique" @@ -69,4 +76,4 @@ if [ $? -eq 0 ]; then else echo "-e error failed to retag and push image to destination ACR" exit 1 -fi \ No newline at end of file +fi diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json index c3b00340a..ef08c3fc2 100644 --- a/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json @@ -28,24 +28,29 @@ ] } ], - "ServiceResourceGroups": [ + "ServiceResourceGroups": [ { "AzureResourceGroupName": "ContainerInsights-MultiArch-Agent-Release", "Location": "eastus2", "InstanceOf": "CI-Agent-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "Global" } - ], + ], "ServiceResources": [ { "Name": "PushLinuxAgent", "InstanceOf": "ShellExtension", "RolloutParametersPath": "Parameters\\ContainerInsights.Linux.Parameters.json" + }, + { + "Name": "PushWindowsAgent", + "InstanceOf": "ShellExtension", + "RolloutParametersPath": "Parameters\\ContainerInsights.Windows.Parameters.json" } ] - } + } ] - } \ No newline at end of file + } diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 328acb201..5e8aa187a 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -42,6 +42,12 @@ data: # When the setting is set to false, only the kube events with !normal event type will be collected enabled = false # When this is enabled (enabled = true), all kube events including normal events will be collected + #[log_collection_settings.schema] + # In the absence of this configmap, default value for containerlog_schema_version is "v1" + # Supported values for this setting are "v1","v2" + # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema + # containerlog_schema_version = "v2" + prometheus-data-collection-settings: |- # Custom Prometheus metrics data collection settings diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index becbe1157..6f68f664e 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -17,7 +17,7 @@ ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd logrotate.conf $tmpdir/ -ARG IMAGE_TAG=ciprod03172022 +ARG IMAGE_TAG=ciprod05192022 ENV AGENT_VERSION ${IMAGE_TAG} WORKDIR ${tmpdir} diff --git a/kubernetes/linux/Dockerfile.multiarch b/kubernetes/linux/Dockerfile.multiarch index df8b04d19..3fdc0431c 100644 --- a/kubernetes/linux/Dockerfile.multiarch +++ b/kubernetes/linux/Dockerfile.multiarch @@ -29,11 +29,27 @@ RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl COPY --from=builder /src/kubernetes/linux/Linux_ULINUX_1.0_*_64_Release/docker-cimprov-*.*.*-*.*.sh $tmpdir/ COPY kubernetes/linux/setup.sh kubernetes/linux/main.sh kubernetes/linux/defaultpromenvvariables kubernetes/linux/defaultpromenvvariables-rs kubernetes/linux/defaultpromenvvariables-sidecar kubernetes/linux/mdsd.xml kubernetes/linux/envmdsd kubernetes/linux/logrotate.conf $tmpdir/ -ARG IMAGE_TAG=ciprod03172022 +ARG IMAGE_TAG=ciprod05192022 ENV AGENT_VERSION ${IMAGE_TAG} WORKDIR ${tmpdir} RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh ${TARGETARCH} -CMD [ "/opt/main.sh" ] +# Do vulnerability scan in a seperate stage to avoid adding layer +FROM base_image AS vulnscan +COPY --from=aquasec/trivy:latest /usr/local/bin/trivy /usr/local/bin/trivy +COPY .trivyignore .trivyignore +RUN trivy rootfs --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM --skip-files "/usr/local/bin/trivy" / +RUN trivy rootfs --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM /usr/lib +RUN trivy rootfs --exit-code 1 --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM --skip-files "/usr/local/bin/trivy" / > /dev/null 2>&1 && trivy rootfs --exit-code 1 --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM /usr/lib > /dev/null 2>&1 + +# Revert to base layer before vulnscan +FROM base_image AS ContainerInsights +# force the trivy stage to run +# docker buildx (BUILDKIT) does not build stages which do not affect the final stage +# by copying over a file we create a dependency +# see: https://github.com/docker/build-push-action/issues/377 +COPY --from=vulnscan /usr/local/bin/trivy /usr/local/bin/trivy +RUN rm /usr/local/bin/trivy +CMD [ "/opt/main.sh" ] diff --git a/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh b/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh index 580b158c9..638236507 100755 --- a/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh +++ b/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh @@ -138,7 +138,7 @@ echo "source code base directory: $baseDir" echo "build directory for docker provider: $buildDir" echo "docker file directory: $dockerFileDir" -if [ "$multi" -eq "1" ]; then +if [ -n "$multi" ] && [ "$multi" -eq "1" ]; then echo "building multiarch" cd $baseDir docker buildx build --platform linux/arm64/v8,linux/amd64 -t $image --build-arg IMAGE_TAG=$imageTag -f $linuxDir/Dockerfile.multiarch --push . diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 997f624e2..2b25b044c 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -288,6 +288,30 @@ fi export CLOUD_ENVIRONMENT=$CLOUD_ENVIRONMENT echo "export CLOUD_ENVIRONMENT=$CLOUD_ENVIRONMENT" >> ~/.bashrc +# Copying over CA certs for airgapped clouds. This is needed for Mariner vs Ubuntu hosts. +# We are unable to tell if the host is Mariner or Ubuntu, +# so both /anchors/ubuntu and /anchors/mariner are mounted in the yaml. +# One will have the certs and the other will be empty. +# These need to be copied to a different location for Mariner vs Ubuntu containers. +# OS_ID here is the container distro. +# Adding Mariner now even though the elif will never currently evaluate. +if [ $CLOUD_ENVIRONMENT == "usnat" ] || [ $CLOUD_ENVIRONMENT == "ussec" ]; then + OS_ID=$(cat /etc/os-release | grep ^ID= | cut -d '=' -f2 | tr -d '"' | tr -d "'") + if [ $OS_ID == "mariner" ]; then + cp /anchors/ubuntu/* /etc/pki/ca-trust/source/anchors + cp /anchors/mariner/* /etc/pki/ca-trust/source/anchors + update-ca-trust + else + if [ $OS_ID != "ubuntu" ]; then + echo "Error: The ID in /etc/os-release is not ubuntu or mariner. Defaulting to ubuntu." + fi + cp /anchors/ubuntu/* /usr/local/share/ca-certificates/ + cp /anchors/mariner/* /usr/local/share/ca-certificates/ + update-ca-certificates + cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem + fi +fi + #consisten naming conventions with the windows export DOMAIN=$domain echo "export DOMAIN=$DOMAIN" >> ~/.bashrc diff --git a/kubernetes/linux/mdsd.xml b/kubernetes/linux/mdsd.xml index de14240aa..6391d4403 100644 --- a/kubernetes/linux/mdsd.xml +++ b/kubernetes/linux/mdsd.xml @@ -140,19 +140,6 @@ - - - - - - - - - - - - - @@ -231,7 +218,6 @@ - @@ -341,11 +327,6 @@ - - - - - @@ -443,12 +424,5 @@ - - - - ]]> - - - diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 00d5bc0fa..c478af0e5 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -40,10 +40,10 @@ sudo apt-get install jq=1.5+dfsg-2 -y #used to setcaps for ruby process to read /proc/env sudo apt-get install libcap2-bin -y -wget https://dl.influxdata.com/telegraf/releases/telegraf-1.20.3_linux_$ARCH.tar.gz -tar -zxvf telegraf-1.20.3_linux_$ARCH.tar.gz +wget https://dl.influxdata.com/telegraf/releases/telegraf-1.22.2_linux_$ARCH.tar.gz +tar -zxvf telegraf-1.22.2_linux_$ARCH.tar.gz -mv /opt/telegraf-1.20.3/usr/bin/telegraf /opt/telegraf +mv /opt/telegraf-1.22.2/usr/bin/telegraf /opt/telegraf chmod 544 /opt/telegraf @@ -79,11 +79,3 @@ sudo apt-get remove ruby2.7-dev gcc make -y # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files # in /var/log at the same time, the rotation doesn't happen correctly and then the *.1 file is forever logged to. rm /etc/logrotate.d/alternatives /etc/logrotate.d/apt /etc/logrotate.d/azure-mdsd /etc/logrotate.d/rsyslog - -#Remove gemfile.lock for http_parser gem 0.6.0 -#see - https://github.com/fluent/fluentd/issues/3374 https://github.com/tmm1/http_parser.rb/issues/70 -if [ -e "/var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock" ]; then - #rename - echo "Renaming unused gemfile.lock for http_parser 0.6.0" - mv /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/renamed_Gemfile_lock.renamed -fi diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index c8324370b..4e021e1b8 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -27,9 +27,11 @@ rules: - apiGroups: ["apps", "extensions", "autoscaling"] resources: ["replicasets", "deployments", "horizontalpodautoscalers"] verbs: ["list"] - - apiGroups: ["azmon.container.insights"] - resources: ["healthstates"] - verbs: ["get", "create", "patch"] + # Uncomment below lines for MSI Auth Mode testing + # - apiGroups: [""] + # resources: ["secrets"] + # resourceNames: [ "omsagent-aad-msi-token" ] + # verbs: ["get", "watch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] --- @@ -51,14 +53,6 @@ apiVersion: v1 data: kube.conf: |- # Fluentd config file for OMS Docker - cluster components (kubeAPI) - #fluent forward plugin - - type forward - port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" - bind 0.0.0.0 - chunk_size_limit 4m - - #Kubernetes pod inventory type kubepodinventory @@ -91,14 +85,6 @@ data: log_level debug - #Kubernetes health - - type kubehealth - tag kubehealth.ReplicaSet - run_interval 60 - log_level debug - - #cadvisor perf- Windows nodes type wincadvisorperf @@ -135,11 +121,6 @@ data: log_level info - #health model aggregation filter - - type filter_health_model_builder - - type out_oms log_level debug @@ -291,21 +272,6 @@ data: retry_mdm_post_wait_minutes 30 - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 4m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubehealth*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 5s - max_retry_wait 5m - - type out_oms log_level debug @@ -336,6 +302,16 @@ data: WSID: "VALUE_WSID" KEY: "VALUE_KEY" --- +# Uncomment below lines for MSI Auth Mode testing +# apiVersion: v1 +# kind: Secret +# metadata: +# name: omsagent-aad-msi-token +# namespace: kube-system +# type: Opaque +# data: +# token: "VALUE_MSI_TOKEN" +# --- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -358,7 +334,7 @@ spec: tier: node annotations: agentVersion: "azure-mdsd-1.17.0" - dockerProviderVersion: "16.0.0-0" + dockerProviderVersion: "17.0.0-0" schema-versions: "v1" spec: serviceAccountName: omsagent @@ -367,8 +343,43 @@ spec: - name: ndots value: "3" containers: + # Uncomment below lines for MSI Auth Mode testing + # - name: addon-token-adapter + # command: + # - /addon-token-adapter + # args: + # - --secret-namespace=kube-system + # - --secret-name=omsagent-aad-msi-token + # - --token-server-listening-port=8888 + # - --health-server-listening-port=9999 + # # Make sure this matching with version in AKS RP side + # image: mcr.microsoft.com/aks/msi/addon-token-adapter:master.220318.3 + # imagePullPolicy: IfNotPresent + # env: + # - name: AZMON_COLLECT_ENV + # value: "false" + # livenessProbe: + # httpGet: + # path: /healthz + # port: 9999 + # initialDelaySeconds: 10 + # periodSeconds: 60 + # resources: + # limits: + # cpu: 500m + # memory: 500Mi + # requests: + # cpu: 100m + # memory: 100Mi + # securityContext: + # capabilities: + # drop: + # - ALL + # add: + # - NET_ADMIN + # - NET_RAW - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05192022" imagePullPolicy: IfNotPresent resources: limits: @@ -406,8 +417,9 @@ spec: value: "VALUE_USER_ASSIGNED_IDENTITY_CLIENT_ID_VALUE" - name: AZMON_CONTAINERLOGS_ONEAGENT_REGIONS value: "koreacentral,norwayeast,eastus2" - - name: USING_AAD_MSI_AUTH - value: "false" + # Uncomment below lines for MSI Auth Mode testing + # - name: USING_AAD_MSI_AUTH + # value: "true" securityContext: privileged: true ports: @@ -454,7 +466,7 @@ spec: timeoutSeconds: 15 #Only in sidecar scraping mode - name: omsagent-prometheus - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05192022" imagePullPolicy: IfNotPresent resources: limits: @@ -487,8 +499,9 @@ spec: # Update this with the user assigned msi client id for omsagent - name: USER_ASSIGNED_IDENTITY_CLIENT_ID value: "VALUE_USER_ASSIGNED_IDENTITY_CLIENT_ID_VALUE" - - name: USING_AAD_MSI_AUTH - value: "false" + # Uncomment below lines for MSI Auth Mode testing + # - name: USING_AAD_MSI_AUTH + # value: "true" securityContext: privileged: true volumeMounts: @@ -597,13 +610,48 @@ spec: rsName: "omsagent-rs" annotations: agentVersion: "azure-mdsd-1.17.0" - dockerProviderVersion: "16.0.0-0" + dockerProviderVersion: "17.0.0-0" schema-versions: "v1" spec: serviceAccountName: omsagent containers: + # Uncomment below lines for MSI Auth Mode testing + # - name: addon-token-adapter + # command: + # - /addon-token-adapter + # args: + # - --secret-namespace=kube-system + # - --secret-name=omsagent-aad-msi-token + # - --token-server-listening-port=8888 + # - --health-server-listening-port=9999 + # # Make sure this matching with version in AKS RP side + # image: mcr.microsoft.com/aks/msi/addon-token-adapter:master.220318.3 + # imagePullPolicy: IfNotPresent + # env: + # - name: AZMON_COLLECT_ENV + # value: "false" + # livenessProbe: + # httpGet: + # path: /healthz + # port: 9999 + # initialDelaySeconds: 10 + # periodSeconds: 60 + # resources: + # limits: + # cpu: 500m + # memory: 500Mi + # requests: + # cpu: 100m + # memory: 100Mi + # securityContext: + # capabilities: + # drop: + # - ALL + # add: + # - NET_ADMIN + # - NET_RAW - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05192022" imagePullPolicy: IfNotPresent resources: limits: @@ -635,8 +683,9 @@ spec: # Add the below environment variable to true only in sidecar enabled regions, else set it to false - name: SIDECAR_SCRAPING_ENABLED value: "true" - - name: USING_AAD_MSI_AUTH - value: "false" + # Uncomment below lines for MSI Auth Mode testing + # - name: USING_AAD_MSI_AUTH + # value: "true" securityContext: privileged: true ports: @@ -644,9 +693,6 @@ spec: protocol: TCP - containerPort: 25224 protocol: UDP - - containerPort: 25227 - protocol: TCP - name: in-rs-tcp volumeMounts: - mountPath: /var/run/host name: docker-sock @@ -766,7 +812,7 @@ spec: tier: node-win annotations: agentVersion: "0.0.0-0" - dockerProviderVersion: "16.0.0-0" + dockerProviderVersion: "17.0.0-0" schema-versions: "v1" spec: serviceAccountName: omsagent @@ -776,7 +822,7 @@ spec: value: "3" containers: - name: omsagent-win - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03172022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod05192022" imagePullPolicy: IfNotPresent resources: limits: @@ -818,6 +864,9 @@ spec: # Add this only for clouds that require cert bootstrapping # - name: REQUIRES_CERT_BOOTSTRAP # value: "true" + # Uncomment below lines for MSI Auth Mode testing + # - name: USING_AAD_MSI_AUTH + # value: "true" volumeMounts: - mountPath: C:\ProgramData\docker\containers name: docker-windows-containers @@ -840,6 +889,10 @@ spec: - mountPath: C:\etc\kubernetes\host name: azure-json-path readOnly: true + # Uncomment below lines for MSI Auth Mode testing + # - mountPath: C:\etc\IMDS-access-token + # name: imds-token + # readOnly: true livenessProbe: exec: command: @@ -897,39 +950,8 @@ spec: secret: secretName: omsagent-adx-secret optional: true ---- -kind: Service -apiVersion: v1 -metadata: - name: healthmodel-replicaset-service - namespace: kube-system -spec: - selector: - rsName: "omsagent-rs" - ports: - - protocol: TCP - port: 25227 - targetPort: in-rs-tcp ---- -# this is for versions >=1.19, for versions <1.19 we continue to use v1beta1 -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: healthstates.azmon.container.insights - namespace: kube-system -spec: - group: azmon.container.insights - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - state: - type: string - scope: Namespaced - names: - plural: healthstates - kind: HealthState + # Uncomment below lines for MSI Auth Mode testing + # - name: imds-token + # secret: + # secretName: omsagent-aad-msi-token + diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 3aa4054e1..383652e0e 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -1,16 +1,18 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +# Supported values of windows version are ltsc2019 or ltsc2022 which are being passed by the build script or build pipeline +ARG WINDOWS_VERSION= +FROM mcr.microsoft.com/windows/servercore:${WINDOWS_VERSION} MAINTAINER OMSContainers@microsoft.com LABEL vendor=Microsoft\ Corp \ com.microsoft.product="Azure Monitor for containers" -ARG IMAGE_TAG=win-ciprod03172022 +ARG IMAGE_TAG=win-ciprod05192022 # Do not split this into multiple RUN! # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 2.6.5.1 --params "'/InstallDir:C:\ruby26'" \ -&& choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby26\msys64'" \ +RUN choco install -y ruby --version 2.7.5.1 --params "'/InstallDir:C:\ruby27'" \ +&& choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby27\msys64'" \ && choco install -y vim # gangams - optional MSYS2 update via ridk failing in merged docker file so skipping that since we dont need optional update @@ -30,7 +32,7 @@ RUN refreshenv \ && gem sources --clear-all # Remove gem cache and chocolatey -RUN powershell -Command "Remove-Item -Force C:\ruby26\lib\ruby\gems\2.6.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" +RUN powershell -Command "Remove-Item -Force C:\ruby27\lib\ruby\gems\2.7.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" SHELL ["powershell"] diff --git a/kubernetes/windows/Dockerfile-dev-base-image b/kubernetes/windows/Dockerfile-dev-base-image index 501fead89..a2f67b7c5 100644 --- a/kubernetes/windows/Dockerfile-dev-base-image +++ b/kubernetes/windows/Dockerfile-dev-base-image @@ -1,4 +1,5 @@ -FROM mcr.microsoft.com/windows/servercore:ltsc2019 +ARG WINDOWS_VERSION= +FROM mcr.microsoft.com/windows/servercore:${WINDOWS_VERSION} MAINTAINER OMSContainers@microsoft.com LABEL vendor=Microsoft\ Corp \ com.microsoft.product="Azure Monitor for containers" diff --git a/kubernetes/windows/dockerbuild/build-and-publish-dev-docker-image.ps1 b/kubernetes/windows/dockerbuild/build-and-publish-dev-docker-image.ps1 index b87132218..f62851eac 100644 --- a/kubernetes/windows/dockerbuild/build-and-publish-dev-docker-image.ps1 +++ b/kubernetes/windows/dockerbuild/build-and-publish-dev-docker-image.ps1 @@ -7,7 +7,8 @@ #> param( [Parameter(mandatory = $true)] - [string]$image + [string]$image, + [string]$windowsBaseImageVersion="" # Supported values are ltsc2019 or ltsc2022. Default is multi-arc image unless this value specified ) $currentdir = $PSScriptRoot @@ -55,10 +56,50 @@ Write-Host "changing directory to DockerFile dir: $dockerFileDir" Set-Location -Path $dockerFileDir $updateImage = ${imagerepo} + ":" + ${imageTag} -Write-Host "STAT:Triggering docker image build: $image" -docker build -t $updateImage --build-arg IMAGE_TAG=$imageTag -f Dockerfile-dev-image . -Write-Host "END:Triggering docker image build: $updateImage" +if ([string]::IsNullOrEmpty($windowsBaseImageVersion)) { + Write-Host "START:Triggering multi-arc docker image build for ltsc2019 & ltsc2022: $image" -Write-Host "STAT:pushing docker image : $updateImage" -docker push $updateImage -Write-Host "EnD:pushing docker image : $updateImage" + $WINDOWS_VERSION="ltsc2019" + $updateImageLTSC2019 = ${imagerepo} + ":" + ${imageTag} + "-" + ${WINDOWS_VERSION} + Write-Host "START:Triggering docker image build for ltsc2019: $updateImageLTSC2019" + docker build --isolation=hyperv -t $updateImageLTSC2019 --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --build-arg IMAGE_TAG=$imageTag -f Dockerfile-dev-image . + Write-Host "END:Triggering docker image build for ltsc2019: $updateImageLTSC2019" + + $WINDOWS_VERSION="ltsc2022" + $updateImageLTSC2022 = ${imagerepo} + ":" + ${imageTag} + "-" + ${WINDOWS_VERSION} + Write-Host "START:Triggering docker image build for ltsc2022: $updateImageLTSC2022" + docker build --isolation=hyperv -t $updateImageLTSC2022 --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --build-arg IMAGE_TAG=$imageTag -f Dockerfile-dev-image . + Write-Host "END:Triggering docker image build for ltsc2022: $updateImageLTSC2022" + + Write-Host "START:pushing docker image with base image ltsc2019 : $updateImageLTSC2019" + docker push $updateImageLTSC2019 + Write-Host "END:pushing docker image : $updateImageLTSC2019" + + Write-Host "START:pushing docker image with base image ltsc2022 : $updateImageLTSC2022" + docker push $updateImageLTSC2022 + Write-Host "END:pushing docker image : $updateImageLTSC2022" + + Write-Host "START:Triggering manigest for multi-arc docker image: $updateImage" + docker manifest create $updateImage $updateImageLTSC2019 $updateImageLTSC2022 + docker manifest push $updateImage + Write-Host "END:Triggering manifest for multi-arc docker image: $updateImage" + + Write-Host "END:Triggering multi-arc docker image build for ltsc2019 & ltsc2022: $image" + +} else { + + if (($windowsBaseImageVersion -eq "ltsc2019") -or ($windowsBaseImageVersion -eq "ltsc2022")) { + Write-Host "Provided baseimage version valid and supported: ${windowsBaseImageVersion}" + } else { + Write-Host "Provided baseimage version neither valid nor supported: ${windowsBaseImageVersion}" -ForegroundColor Red + exit 1 + } + + Write-Host "STAT:Triggering docker image build: $image with base image version: $windowsBaseImageVersion" + docker build -t $updateImage --build-arg WINDOWS_VERSION=$windowsBaseImageVersion --build-arg IMAGE_TAG=$imageTag -f Dockerfile-dev-image . + Write-Host "END:Triggering docker image build: $updateImage" + + Write-Host "STAT:pushing docker image : $updateImage" + docker push $updateImage + Write-Host "EnD:pushing docker image : $updateImage" +} diff --git a/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 b/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 index c1f655882..a5c78cc72 100644 --- a/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 +++ b/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 @@ -7,7 +7,8 @@ #> param( [Parameter(mandatory = $true)] - [string]$image + [string]$image, + [string]$windowsBaseImageVersion="" # Supported values are ltsc2019 or ltsc2022. Default is multi-arc image unless this value specified ) $currentdir = $PSScriptRoot @@ -55,10 +56,49 @@ Write-Host "changing directory to DockerFile dir: $dockerFileDir" Set-Location -Path $dockerFileDir $updateImage = ${imagerepo} + ":" + ${imageTag} -Write-Host "STAT:Triggering docker image build: $image" -docker build -t $updateImage --build-arg IMAGE_TAG=$imageTag . -Write-Host "END:Triggering docker image build: $updateImage" +if ([string]::IsNullOrEmpty($windowsBaseImageVersion)) { + Write-Host "START:Triggering multi-arc docker image build for ltsc2019 & ltsc2022: $image" + + $WINDOWS_VERSION="ltsc2019" + $updateImageLTSC2019 = ${imagerepo} + ":" + ${imageTag} + "-" + ${WINDOWS_VERSION} + Write-Host "START:Triggering docker image build for ltsc2019: $updateImageLTSC2019" + docker build --isolation=hyperv -t $updateImageLTSC2019 --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --build-arg IMAGE_TAG=$imageTag . + Write-Host "END:Triggering docker image build for ltsc2019: $updateImageLTSC2019" + + $WINDOWS_VERSION="ltsc2022" + $updateImageLTSC2022 = ${imagerepo} + ":" + ${imageTag} + "-" + ${WINDOWS_VERSION} + Write-Host "START:Triggering docker image build for ltsc2022: $updateImageLTSC2022" + docker build --isolation=hyperv -t $updateImageLTSC2022 --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --build-arg IMAGE_TAG=$imageTag . + Write-Host "END:Triggering docker image build for ltsc2022: $updateImageLTSC2022" + + Write-Host "START:pushing docker image with base image ltsc2019 : $updateImageLTSC2019" + docker push $updateImageLTSC2019 + Write-Host "END:pushing docker image : $updateImageLTSC2019" + + Write-Host "START:pushing docker image with base image ltsc2022 : $updateImageLTSC2022" + docker push $updateImageLTSC2022 + Write-Host "END:pushing docker image : $updateImageLTSC2022" + + Write-Host "START:Triggering manigest for multi-arc docker image: $updateImage" + docker manifest create $updateImage $updateImageLTSC2019 $updateImageLTSC2022 + docker manifest push $updateImage + Write-Host "END:Triggering manifest for multi-arc docker image: $updateImage" + + Write-Host "END:Triggering multi-arc docker image build for ltsc2019 & ltsc2022: $image" +} else { + if (($windowsBaseImageVersion -eq "ltsc2019") -or ($windowsBaseImageVersion -eq "ltsc2022")) { + Write-Host "Provided baseimage version valid and supported: ${windowsBaseImageVersion}" + } else { + Write-Host "Provided baseimage version neither valid nor supported: ${windowsBaseImageVersion}" -ForegroundColor Red + exit 1 + } + + Write-Host "START:Triggering docker image build: $image with baseImage version: ${windowsBaseImageVersion}" + docker build --isolation=hyperv -t $updateImage --build-arg WINDOWS_VERSION=$windowsBaseImageVersion --build-arg IMAGE_TAG=$imageTag . + Write-Host "END:Triggering docker image build: $updateImage" + + Write-Host "START:pushing docker image : $updateImage" + docker push $updateImage + Write-Host "END:pushing docker image : $updateImage" +} -Write-Host "STAT:pushing docker image : $updateImage" -docker push $updateImage -Write-Host "EnD:pushing docker image : $updateImage" diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 6482daed9..2dd1ddd01 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -302,10 +302,18 @@ function Set-EnvironmentVariables { Write-Host "Failed to set environment variable AGENT_VERSION for target 'machine' since it is either null or empty" } + $kubernetesPort = [System.Environment]::GetEnvironmentVariable("KUBERNETES_PORT_443_TCP_PORT", "process") + if (![string]::IsNullOrEmpty($kubernetesPort)) { + [System.Environment]::SetEnvironmentVariable("KUBERNETES_PORT_443_TCP_PORT", $kubernetesPort, "machine") + Write-Host "Successfully set environment variable KUBERNETES_PORT_443_TCP_PORT - $($kubernetesPort) for target 'machine'..." + } + else { + Write-Host "Failed to set environment variable KUBERNETES_PORT_443_TCP_PORT for target 'machine' since it is either null or empty" + } + # run config parser ruby /opt/omsagentwindows/scripts/ruby/tomlparser.rb .\setenv.ps1 - #Parse the configmap to set the right environment variables for agent config. ruby /opt/omsagentwindows/scripts/ruby/tomlparser-agent-config.rb .\setagentenv.ps1 diff --git a/kubernetes/windows/setup.ps1 b/kubernetes/windows/setup.ps1 index 857f9f690..2fd429e43 100644 --- a/kubernetes/windows/setup.ps1 +++ b/kubernetes/windows/setup.ps1 @@ -1,3 +1,7 @@ +# speed up Invoke-WebRequest +# https://stackoverflow.com/questions/28682642/powershell-why-is-using-invoke-webrequest-much-slower-than-a-browser-download +$ProgressPreference = 'SilentlyContinue' + Write-Host ('Creating folder structure') New-Item -Type Directory -Path /installation -ErrorAction SilentlyContinue @@ -31,7 +35,7 @@ Write-Host ('Finished Installing Fluentbit') Write-Host ('Installing Telegraf'); try { - $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.20.3_windows_amd64.zip' + $telegrafUri='https://dl.influxdata.com/telegraf/releases/telegraf-1.22.2_windows_amd64.zip' Invoke-WebRequest -Uri $telegrafUri -OutFile /installation/telegraf.zip Expand-Archive -Path /installation/telegraf.zip -Destination /installation/telegraf Move-Item -Path /installation/telegraf/*/* -Destination /opt/telegraf/ -ErrorAction SilentlyContinue @@ -66,10 +70,3 @@ Remove-Item /installation -Recurse #Remove gemfile.lock for http_parser gem 0.6.0 #see - https://github.com/fluent/fluentd/issues/3374 https://github.com/tmm1/http_parser.rb/issues/70 - -$gemfile = "\ruby26\lib\ruby\gems\2.6.0\gems\http_parser.rb-0.6.0\Gemfile.lock" -$gemfileFullPath = $Env:SYSTEMDRIVE + "\" + $gemfile -If (Test-Path -Path $gemfile ) { - Write-Host ("Renaming unused gemfile.lock for http_parser 0.6.0") - Rename-Item -Path $gemfileFullPath -NewName "renamed_Gemfile_lock.renamed" -} \ No newline at end of file diff --git a/scripts/build/windows/install-build-pre-requisites.ps1 b/scripts/build/windows/install-build-pre-requisites.ps1 index 750a7b18b..235f6ace9 100644 --- a/scripts/build/windows/install-build-pre-requisites.ps1 +++ b/scripts/build/windows/install-build-pre-requisites.ps1 @@ -13,7 +13,7 @@ function Install-Go { exit 1 } - $url = "https://dl.google.com/go/go1.15.14.windows-amd64.msi" + $url = "https://go.dev/dl/go1.15.14.windows-amd64.msi" $output = Join-Path -Path $tempGo -ChildPath "go1.15.14.windows-amd64.msi" Write-Host("downloading go msi into directory path : " + $output + " ...") Invoke-WebRequest -Uri $url -OutFile $output -ErrorAction Stop @@ -102,7 +102,7 @@ function Install-DotNetCoreSDK() { # install dotNet core sdk Write-Host("installing .net core sdk 3.1 ...") - Start-Process msiexec.exe -Wait -ArgumentList '/I ', $output, '/quiet' + Start-Process -Wait $output -ArgumentList " /q /norestart" Write-Host("installing .net core sdk 3.1 completed") } @@ -121,7 +121,7 @@ function Install-Docker() { exit 1 } - $url = "https://download.docker.com/win/stable/Docker%20Desktop%20Installer.exe" + $url = "https://desktop.docker.com/win/main/amd64/Docker%20Desktop%20Installer.exe" $output = Join-Path -Path $dockerTemp -ChildPath "docker-desktop-installer.exe" Write-Host("downloading docker-desktop-installer: " + $dockerTemp + " ...") Invoke-WebRequest -Uri $url -OutFile $output -ErrorAction Stop @@ -129,10 +129,14 @@ function Install-Docker() { # install docker Write-Host("installing docker for desktop ...") - Start-Process msiexec.exe -Wait -ArgumentList '/I ', $output, '/quiet' + Start-Process $output -Wait -ArgumentList 'install --quiet' Write-Host("installing docker for desktop completed") } +# speed up Invoke-WebRequest +# https://stackoverflow.com/questions/28682642/powershell-why-is-using-invoke-webrequest-much-slower-than-a-browser-download +$ProgressPreference = 'SilentlyContinue' + Write-Host "Install GO 1.15.14 version" Install-Go Write-Host "Install Build dependencies" diff --git a/scripts/dcr-onboarding/ci-extension-dcr-streams.md b/scripts/dcr-onboarding/ci-extension-dcr-streams.md index cbac41838..23e929d7c 100644 --- a/scripts/dcr-onboarding/ci-extension-dcr-streams.md +++ b/scripts/dcr-onboarding/ci-extension-dcr-streams.md @@ -156,21 +156,7 @@ stage: to review tags: agent ``` -# 12 - KubeHealth -``` -stream-id: Microsoft-KubeHealth -data-type: KUBE_HEALTH_BLOB -intelligence-pack: ContainerInsights -solutions: ContainerInsights -platform: Any -la-table-name: KubeHealth -alias-stream-id: Microsoft-KubeHealth -contact-alias: OMScontainers@microsoft.com -stage: to review -tags: agent -``` - -# 13 - Perf +# 12 - Perf ``` > Note - This stream already exists stream-id: Microsoft-Perf diff --git a/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json b/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json index 28996f4a1..c42a1d074 100644 --- a/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json +++ b/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json @@ -14,10 +14,10 @@ "description": "Location of the AKS resource e.g. \"East US\"" } }, - "aksResourceTagValues": { + "resourceTagValues": { "type": "object", "metadata": { - "description": "Existing all tags on AKS Cluster Resource" + "description": "Existing or new tags to use on AKS, ContainerInsights and DataCollectionRule Resources" } }, "workspaceLocation": { @@ -31,12 +31,6 @@ "metadata": { "description": "Full Resource ID of the log analitycs workspace that will be used for data destination. For example /subscriptions/00000000-0000-0000-0000-0000-00000000/resourceGroups/ResourceGroupName/providers/Microsoft.operationalinsights/workspaces/ws_xyz" } - }, - "dcrResourceTagValues": { - "type": "object", - "metadata": { - "description": "Existing or new tags on DCR Resource" - } } }, "variables": { @@ -70,7 +64,7 @@ "apiVersion": "2019-11-01-preview", "name": "[variables('dcrName')]", "location": "[parameters('workspaceLocation')]", - "tags": "[parameters('dcrResourceTagValues')]", + "tags": "[parameters('resourceTagValues')]", "kind": "Linux", "properties": { "dataSources": { @@ -184,7 +178,7 @@ "name": "[variables('clusterName')]", "type": "Microsoft.ContainerService/managedClusters", "location": "[parameters('aksResourceLocation')]", - "tags": "[parameters('aksResourceTagValues')]", + "tags": "[parameters('resourceTagValues')]", "apiVersion": "2018-03-31", "properties": { "mode": "Incremental", diff --git a/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterParam.json b/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterParam.json index 31f0f9c49..e0f9a643f 100644 --- a/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterParam.json +++ b/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterParam.json @@ -8,20 +8,13 @@ "aksResourceLocation": { "value": "" }, - "aksResourceTagValues": { - "value": { - "": "", - "": "", - "": "" - } - }, "workspaceResourceId": { "value": "/subscriptions//resourceGroups//providers/Microsoft.OperationalInsights/workspaces/" }, "workspaceLocation": { "value": "" }, - "dcrResourceTagValues": { + "resourceTagValues": { "value": { "": "", "": "", diff --git a/scripts/onboarding/templates/arc-k8s-extension-msi-auth/existingClusterOnboarding.json b/scripts/onboarding/templates/arc-k8s-extension-msi-auth/existingClusterOnboarding.json new file mode 100644 index 000000000..a4a4e3453 --- /dev/null +++ b/scripts/onboarding/templates/arc-k8s-extension-msi-auth/existingClusterOnboarding.json @@ -0,0 +1,224 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterResourceId": { + "type": "string", + "metadata": { + "description": "Resource Id of the Azure Arc Connected Cluster" + } + }, + "clusterRegion": { + "type": "string", + "metadata": { + "description": "Location of the Azure Arc Connected Cluster Resource e.g. \"eastus\"" + } + }, + "workspaceResourceId": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Resource ID" + } + }, + "workspaceRegion": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Workspace region e.g. \"eastus\"" + } + }, + "workspaceDomain": { + "type": "string", + "allowedValues": [ + "opinsights.azure.com", + "opinsights.azure.cn", + "opinsights.azure.us", + "opinsights.azure.eaglex.ic.gov", + "opinsights.azure.microsoft.scloud" + ], + "defaultValue": "opinsights.azure.com", + "metadata": { + "description": "Azure Monitor Log Analytics Workspace Domain e.g. opinsights.azure.com" + } + }, + "resourceTagValues": { + "type": "object", + "metadata": { + "description": "Existing or new tags to use on Arc K8s ContainerInsights extension resources" + } + } + }, + "variables": { + "clusterSubscriptionId": "[split(parameters('clusterResourceId'),'/')[2]]", + "clusterResourceGroup": "[split(parameters('clusterResourceId'),'/')[4]]", + "clusterName": "[split(parameters('clusterResourceId'),'/')[8]]", + "clusterLocation": "[replace(parameters('clusterRegion'),' ', '')]", + "workspaceSubscriptionId": "[split(parameters('workspaceResourceId'),'/')[2]]", + "workspaceResourceGroup": "[split(parameters('workspaceResourceId'),'/')[4]]", + "dcrName": "[Concat('MSCI', '-', variables('clusterName'), '-', variables('clusterLocation'))]", + "associationName": "ContainerInsightsExtension", + "dataCollectionRuleId": "[resourceId(variables('workspaceSubscriptionId'), variables('workspaceResourceGroup'), 'Microsoft.Insights/dataCollectionRules', variables('dcrName'))]" + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "name": "[Concat('arc-k8s-monitoring-msi-dcr', '-', uniqueString(variables('dcrName')))]", + "apiVersion": "2017-05-10", + "subscriptionId": "[variables('workspaceSubscriptionId')]", + "resourceGroup": "[variables('workspaceResourceGroup')]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Insights/dataCollectionRules", + "apiVersion": "2019-11-01-preview", + "name": "[variables('dcrName')]", + "location": "[parameters('workspaceRegion')]", + "tags": "[parameters('resourceTagValues')]", + "kind": "Linux", + "properties": { + "dataSources": { + "extensions": [ + { + "name": "ContainerInsightsExtension", + "streams": [ + "Microsoft-Perf", + "Microsoft-ContainerInventory", + "Microsoft-ContainerLog", + "Microsoft-ContainerLogV2", + "Microsoft-ContainerNodeInventory", + "Microsoft-KubeEvents", + "Microsoft-KubeMonAgentEvents", + "Microsoft-KubeNodeInventory", + "Microsoft-KubePodInventory", + "Microsoft-KubePVInventory", + "Microsoft-KubeServices", + "Microsoft-InsightsMetrics" + ], + "extensionName": "ContainerInsights" + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "[parameters('workspaceResourceId')]", + "name": "ciworkspace" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-Perf", + "Microsoft-ContainerInventory", + "Microsoft-ContainerLog", + "Microsoft-ContainerLogV2", + "Microsoft-ContainerNodeInventory", + "Microsoft-KubeEvents", + "Microsoft-KubeMonAgentEvents", + "Microsoft-KubeNodeInventory", + "Microsoft-KubePodInventory", + "Microsoft-KubePVInventory", + "Microsoft-KubeServices", + "Microsoft-InsightsMetrics" + ], + "destinations": [ + "ciworkspace" + ] + } + ] + } + } + ] + }, + "parameters": {} + } + }, + { + "type": "Microsoft.Resources/deployments", + "name": "[Concat('arc-k8s-monitoring-msi-dcra', '-', uniqueString(parameters('clusterResourceId')))]", + "apiVersion": "2017-05-10", + "subscriptionId": "[variables('clusterSubscriptionId')]", + "resourceGroup": "[variables('clusterResourceGroup')]", + "dependsOn": [ + "[Concat('arc-k8s-monitoring-msi-dcr', '-', uniqueString(variables('dcrName')))]" + ], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Kubernetes/connectedClusters/providers/dataCollectionRuleAssociations", + "name": "[concat(variables('clusterName'),'/microsoft.insights/', variables('associationName'))]", + "apiVersion": "2019-11-01-preview", + "properties": { + "description": "Association of data collection rule. Deleting this association will break the data collection for this AKS Cluster.", + "dataCollectionRuleId": "[variables('dataCollectionRuleId')]" + } + } + ] + }, + "parameters": {} + } + }, + { + "type": "Microsoft.Resources/deployments", + "name": "[Concat('arc-k8s-ci-extension', '-', uniqueString(parameters('clusterResourceId')))]", + "apiVersion": "2019-05-01", + "subscriptionId": "[split(parameters('clusterResourceId'),'/')[2]]", + "resourceGroup": "[split(parameters('clusterResourceId'),'/')[4]]", + "dependsOn": [ + "[Concat('arc-k8s-monitoring-msi-dcra', '-', uniqueString(parameters('clusterResourceId')))]" + ], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2021-09-01", + "name": "azuremonitor-containers", + "location": "[parameters('clusterRegion')]", + "identity": { + "type": "systemassigned" + }, + "properties": { + "extensionType": "Microsoft.AzureMonitor.Containers", + "configurationSettings": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]", + "omsagent.domain": "[parameters('workspaceDomain')]", + "omsagent.useAADAuth": "true" + }, + "configurationProtectedSettings": { + "omsagent.secret.wsid": "[reference(parameters('workspaceResourceId'), '2015-03-20').customerId]", + "omsagent.secret.key": "[listKeys(parameters('workspaceResourceId'), '2015-03-20').primarySharedKey]" + }, + "autoUpgradeMinorVersion": true, + "releaseTrain": "Stable", + "scope": { + "Cluster": { + "releaseNamespace": "azuremonitor-containers" + } + } + }, + "scope": "[concat('Microsoft.Kubernetes/connectedClusters/', split(parameters('clusterResourceId'),'/')[8])]" + } + ] + } + } + } + ] +} diff --git a/scripts/onboarding/templates/arc-k8s-extension-msi-auth/existingClusterParam.json b/scripts/onboarding/templates/arc-k8s-extension-msi-auth/existingClusterParam.json new file mode 100644 index 000000000..8cd17ceb3 --- /dev/null +++ b/scripts/onboarding/templates/arc-k8s-extension-msi-auth/existingClusterParam.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterResourceId": { + "value": "/subscriptions//resourceGroups//providers/Microsoft.Kubernetes/connectedClusters/" + }, + "clusterRegion": { + "value": "" + }, + "workspaceResourceId": { + "value": "/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/" + }, + "workspaceRegion": { + "value": "" + }, + "workspaceDomain": { + "value": "" + }, + "resourceTagValues": { + "value": { + "": "", + "": "", + "": "" + } + } + } +} diff --git a/scripts/preview/health/HealthAgentOnboarding.ps1 b/scripts/preview/health/HealthAgentOnboarding.ps1 deleted file mode 100644 index 9ce8eca74..000000000 --- a/scripts/preview/health/HealthAgentOnboarding.ps1 +++ /dev/null @@ -1,432 +0,0 @@ -<# - .DESCRIPTION - Upgrades the Kubernetes cluster that has been onboarded to monitoring to a version of the agent - that generates health monitor signals - 1. Installs necessary powershell modules - 2. Onboards Container Insights solution to the supplied LA workspace if not already onboarded - 3. Updates the cluster metadata to link the LA workspace ID to the cluster - .PARAMETER aksResourceId - Name of the cluster configured on the OMSAgent - .PARAMETER loganalyticsWorkspaceResourceId - Azure ResourceId of the log analytics workspace Id - .PARAMETER aksResourceLocation - Resource location of the AKS cluster resource -#> -param( - [Parameter(mandatory = $true)] - [string]$aksResourceId, - [Parameter(mandatory = $true)] - [string]$aksResourceLocation, - [Parameter(mandatory = $true)] - [string]$logAnalyticsWorkspaceResourceId -) - - -$OptOutLink = "https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-optout" - -# checks the required Powershell modules exist and if not exists, request the user permission to install -$azAccountModule = Get-Module -ListAvailable -Name Az.Accounts -$azResourcesModule = Get-Module -ListAvailable -Name Az.Resources -$azOperationalInsights = Get-Module -ListAvailable -Name Az.OperationalInsights -$azAks = Get-Module -ListAvailable -Name Az.Aks - -if (($null -eq $azAccountModule) -or ($null -eq $azResourcesModule) -or ($null -eq $azOperationalInsights)) { - - $currentPrincipal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent()) - - if ($currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { - Write-Host("Running script as an admin...") - Write-Host("") - } - else { - Write-Host("Please re-launch the script with elevated administrator") -ForegroundColor Red - Stop-Transcript - exit - } - - $message = "This script will try to install the latest versions of the following Modules : ` - Az.Resources, Az.Accounts, Az.Aks and Az.OperationalInsights using the command` - `'Install-Module {Insert Module Name} -Repository PSGallery -Force -AllowClobber -ErrorAction Stop -WarningAction Stop' - `If you do not have the latest version of these Modules, this troubleshooting script may not run." - $question = "Do you want to Install the modules and run the script or just run the script?" - - $choices = New-Object Collections.ObjectModel.Collection[Management.Automation.Host.ChoiceDescription] - $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Yes, Install and run')) - $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Continue without installing the Module')) - $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Quit')) - - $decision = $Host.UI.PromptForChoice($message, $question, $choices, 0) - - switch ($decision) { - 0 { - - if ($null -eq $azResourcesModule) { - try { - Write-Host("Installing Az.Resources...") - Install-Module Az.Resources -Repository PSGallery -Force -AllowClobber -ErrorAction Stop - } - catch { - Write-Host("Close other powershell logins and try installing the latest modules forAz.Accounts in a new powershell window: eg. 'Install-Module Az.Accounts -Repository PSGallery -Force'") -ForegroundColor Red - exit - } - } - - if ($null -eq $azAccountModule) { - try { - Write-Host("Installing Az.Accounts...") - Install-Module Az.Accounts -Repository PSGallery -Force -AllowClobber -ErrorAction Stop - } - catch { - Write-Host("Close other powershell logins and try installing the latest modules forAz.Accounts in a new powershell window: eg. 'Install-Module Az.Accounts -Repository PSGallery -Force'") -ForegroundColor Red - exit - } - } - - if ($null -eq $azOperationalInsights) { - try { - - Write-Host("Installing AzureRM.OperationalInsights...") - Install-Module Az.OperationalInsights -Repository PSGallery -Force -AllowClobber -ErrorAction Stop - } - catch { - Write-Host("Close other powershell logins and try installing the latest modules for AzureRM.OperationalInsights in a new powershell window: eg. 'Install-Module AzureRM.OperationalInsights -Repository PSGallery -Force'") -ForegroundColor Red - exit - } - } - if ($null -eq $azAks) { - try { - - Write-Host("Installing Az.Aks...") - Install-Module Az.Aks -Repository PSGallery -Force -AllowClobber -ErrorAction Stop - } - catch { - Write-Host("Close other powershell logins and try installing the latest modules for AzureRM.OperationalInsights in a new powershell window: eg. 'Install-Module AzureRM.OperationalInsights -Repository PSGallery -Force'") -ForegroundColor Red - exit - } - } - - - } - 1 { - - if ($null -eq $azResourcesModule) { - try { - Import-Module Az.Resources -ErrorAction Stop - } - catch { - Write-Host("Could not import Az.Resources...") -ForegroundColor Red - Write-Host("Close other powershell logins and try installing the latest modules for Az.Resources in a new powershell window: eg. 'Install-Module Az.Resources -Repository PSGallery -Force'") -ForegroundColor Red - Stop-Transcript - exit - } - } - if ($null -eq $azAccountModule) { - try { - Import-Module Az.Accounts -ErrorAction Stop - } - catch { - Write-Host("Could not import Az.Accounts...") -ForegroundColor Red - Write-Host("Close other powershell logins and try installing the latest modules for Az.Accounts in a new powershell window: eg. 'Install-Module Az.Accounts -Repository PSGallery -Force'") -ForegroundColor Red - Stop-Transcript - exit - } - } - - if ($null -eq $azAccountModule) { - try { - Import-Module Az.OperationalInsights - } - catch { - Write-Host("Could not import Az.OperationalInsights... Please reinstall this Module") -ForegroundColor Red - Stop-Transcript - exit - } - } - - } - 2 { - Write-Host("") - Stop-Transcript - exit - } - } -} - -if ([string]::IsNullOrEmpty($logAnalyticsWorkspaceResourceId)) { - Write-Host("logAnalyticsWorkspaceResourceId should not be NULL or empty") -ForegroundColor Red - exit -} - -if (($logAnalyticsWorkspaceResourceId -match "/providers/Microsoft.OperationalInsights/workspaces") -eq $false) { - Write-Host("logAnalyticsWorkspaceResourceId should be valid Azure Resource Id format") -ForegroundColor Red - exit -} - -$workspaceResourceDetails = $logAnalyticsWorkspaceResourceId.Split("/") - -if ($workspaceResourceDetails.Length -ne 9) { - Write-Host("logAnalyticsWorkspaceResourceId should be valid Azure Resource Id format") -ForegroundColor Red - exit -} - -$workspaceSubscriptionId = $workspaceResourceDetails[2] -$workspaceSubscriptionId = $workspaceSubscriptionId.Trim() -$workspaceResourceGroupName = $workspaceResourceDetails[4] -$workspaceResourceGroupName = $workspaceResourceGroupName.Trim() -$workspaceName = $workspaceResourceDetails[8] -$workspaceResourceGroupName = $workspaceResourceGroupName.Trim() - -$aksResourceDetails = $aksResourceId.Split("/") -$clusterResourceGroupName = $aksResourceDetails[4].Trim() -$clusterSubscriptionId = $aksResourceDetails[2].Trim() -$clusterName = $aksResourceDetails[8].Trim() - -Write-Host("LogAnalytics Workspace SubscriptionId : '" + $workspaceSubscriptionId + "' ") -ForegroundColor Green - -try { - Write-Host("") - Write-Host("Trying to get the current Az login context...") - $account = Get-AzContext -ErrorAction Stop - Write-Host("Successfully fetched current AzContext context...") -ForegroundColor Green - Write-Host("") -} -catch { - Write-Host("") - Write-Host("Could not fetch AzContext..." ) -ForegroundColor Red - Write-Host("") -} - - -if ($null -eq $account.Account) { - try { - Write-Host("Please login...") - Connect-AzAccount -subscriptionid $clusterSubscriptionId - } - catch { - Write-Host("") - Write-Host("Could not select subscription with ID : " + $clusterSubscriptionId + ". Please make sure the ID you entered is correct and you have access to the cluster" ) -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit - } -} - -Write-Host("Checking if cluster is onboarded to Container Monitoring") -if ($account.Subscription.Id -eq $clusterSubscriptionId) { - Write-Host("Subscription: $clusterSubscriptionId is already selected. Account details: ") - $account -} -else { - try { - Write-Host("Current Subscription:") - $account - Write-Host("Changing to workspace subscription: $clusterSubscriptionId") - Set-AzContext -SubscriptionId $clusterSubscriptionId - - } - catch { - Write-Host("") - Write-Host("Could not select subscription with ID : " + $workspaceSubscriptionId + ". Please make sure the ID you entered is correct and you have access to the cluster" ) -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit - } -} - -try { - $resources = Get-AzResource -ResourceGroupName $clusterResourceGroupName -Name $clusterName -ResourceType "Microsoft.ContainerService/managedClusters" -ExpandProperties -ErrorAction Stop -WarningAction Stop - $clusterResource = $resources[0] - - $props = ($clusterResource.Properties | ConvertTo-Json).toLower() | ConvertFrom-Json - - if ($true -eq $props.addonprofiles.omsagent.enabled -and $null -ne $props.addonprofiles.omsagent.config) { - Write-Host("Your cluster is already onboarded to Azure monitor for containers. Please refer to the following documentation to opt-out and re-run this script again:") -ForegroundColor Red; - Write-Host("") - Write-Host($OptOutLink) -ForegroundColor Red - Write-Host("") - throw - } - - Write-Host("Setting context to the current cluster") - Import-AzAksCredential -Id $aksResourceId -Force - $omsagentCount = kubectl get pods -n kube-system | Select-String omsagent - if ($null -eq $omsagentCount) { - Write-Host ("OmsAgent is not running. Proceeding to do custom onboarding for Health Agent") - } - else { - Write-Host ("Cluster is not enabled for Monitoring. But detected omsagent pods. Please wait for 30 min to ensure that omsagent containers are completely stopped and re-run this script") -ForegroundColor Red - Stop-Transcript - exit - } -} -catch { - Write-Host("Error when checking if cluster is already onboarded") - exit -} - - -if ($account.Subscription.Id -eq $workspaceSubscriptionId) { - Write-Host("Subscription: $workspaceSubscriptionId is already selected. Account details: ") - $account -} -else { - try { - Write-Host("Current Subscription:") - $account - Write-Host("Changing to workspace subscription: $workspaceSubscriptionId") - Set-AzContext -SubscriptionId $workspaceSubscriptionId - } - catch { - Write-Host("") - Write-Host("Could not select subscription with ID : " + $workspaceSubscriptionId + ". Please make sure the ID you entered is correct and you have access to the cluster" ) -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit - } -} - -$WorkspaceInformation = Get-AzOperationalInsightsWorkspace -ResourceGroupName $workspaceResourceGroupName -Name $workspaceName -ErrorAction Stop -$key = (Get-AzOperationalInsightsWorkspaceSharedKeys -ResourceGroupName $workspaceResourceGroupName -Name $workspaceName).PrimarySharedKey -$wsid = $WorkspaceInformation.CustomerId -$base64EncodedKey = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($key)) -$base64EncodedWsId = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($wsid)) -Write-Host("Successfully verified specified logAnalyticsWorkspaceResourceId valid and exists...") -ForegroundColor Green -$WorkspaceLocation = $WorkspaceInformation.Location - -if ($null -eq $WorkspaceLocation) { - Write-Host("") - Write-Host("Cannot fetch workspace location. Please try again...") -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit -} - -try { - $WorkspaceIPDetails = Get-AzOperationalInsightsIntelligencePacks -ResourceGroupName $workspaceResourceGroupName -WorkspaceName $workspaceName -ErrorAction Stop - Write-Host("Successfully fetched workspace IP details...") -ForegroundColor Green - Write-Host("") -} -catch { - Write-Host("") - Write-Host("Failed to get the list of solutions onboarded to the workspace. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit -} - -try { - $ContainerInsightsIndex = $WorkspaceIPDetails.Name.IndexOf("ContainerInsights"); - Write-Host("Successfully located ContainerInsights solution") -ForegroundColor Green - Write-Host("") -} -catch { - Write-Host("Failed to get ContainerInsights solution details from the workspace") -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit -} - -$isSolutionOnboarded = $WorkspaceIPDetails.Enabled[$ContainerInsightsIndex] - -if ($false -eq $isSolutionOnboarded) { - - $DeploymentName = "ContainerInsightsSolutionOnboarding-" + ((Get-Date).ToUniversalTime()).ToString('MMdd-HHmm') - $Parameters = @{ } - $Parameters.Add("workspaceResourceId", $logAnalyticsWorkspaceResourceID) - $Parameters.Add("workspaceRegion", $WorkspaceLocation) - $Parameters - - try { - New-AzResourceGroupDeployment -Name $DeploymentName ` - -ResourceGroupName $workspaceResourceGroupName ` - -TemplateUri https://raw.githubusercontent.com/microsoft/Docker-Provider/ci_prod/scripts/onboarding/templates/azuremonitor-containerSolution.json ` - -TemplateParameterObject $Parameters -ErrorAction Stop` - - - Write-Host("Successfully added Container Insights Solution") -ForegroundColor Green - - } - catch { - Write-Host ("Template deployment failed with an error: '" + $Error[0] + "' ") -ForegroundColor Red - Write-Host("Please contact us by emailing askcoin@microsoft.com for help") -ForegroundColor Red - } - -} - -Write-Host("Successfully added Container Insights Solution to workspace " + $workspaceName) -ForegroundColor Green - -try { - $Parameters = @{ } - $Parameters.Add("aksResourceId", $aksResourceId) - $Parameters.Add("aksResourceLocation", $aksResourceLocation) - $Parameters.Add("workspaceResourceId", $logAnalyticsWorkspaceResourceId) - $DeploymentName = "ClusterHealthOnboarding-" + ((Get-Date).ToUniversalTime()).ToString('MMdd-HHmm') - $Parameters - - Write-Host " Onboarding cluster to provided LA workspace " - - if ($account.Subscription.Id -eq $clusterSubscriptionId) { - Write-Host("Subscription: $clusterSubscriptionId is already selected. Account details: ") - $account - } - else { - try { - Write-Host("Current Subscription:") - $account - Write-Host("Changing to subscription: $clusterSubscriptionId") - Set-AzContext -SubscriptionId $clusterSubscriptionId - } - catch { - Write-Host("") - Write-Host("Could not select subscription with ID : " + $workspaceSubscriptionId + ". Please make sure the ID you entered is correct and you have access to the cluster" ) -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit - } - } - - Write-Host("Enabling Custom Monitoring using template deployment") - New-AzResourceGroupDeployment -Name $DeploymentName ` - -ResourceGroupName $clusterResourceGroupName ` - -TemplateUri https://raw.githubusercontent.com/Microsoft/OMS-docker/dilipr/onboardHealth/health/customOnboarding.json ` - -TemplateParameterObject $Parameters -ErrorAction Stop` - - Write-Host("") - - Write-Host("Successfully custom onboarded cluster to Monitoring") -ForegroundColor Green - - Write-Host("") -} -catch { - Write-Host ("Template deployment failed with an error: '" + $Error[0] + "' ") -ForegroundColor Red - exit - #Write-Host("Please contact us by emailing askcoin@microsoft.com for help") -ForegroundColor Red -} - -$desktopPath = "~" -if (-not (test-path $desktopPath/deployments) ) { - Write-Host "$($desktopPath)/deployments doesn't exist, creating it" - mkdir $desktopPath/deployments | out-null -} -else { - Write-Host "$($desktopPath)/deployments exists, no need to create it" -} -try { - - $aksResourceDetails = $aksResourceId.Split("/") - if ($aksResourceDetails.Length -ne 9) { - Write-Host("aksResourceDetails should be valid Azure Resource Id format") -ForegroundColor Red - exit - } - $clusterName = $aksResourceDetails[8].Trim() - $clusterResourceGroupName = $aksResourceDetails[4].Trim() - Import-AzAksCredential -Id $aksResourceId -Force - Invoke-WebRequest https://raw.githubusercontent.com/microsoft/OMS-docker/dilipr/mergeHealthToCiFeature/health/omsagent-template.yaml -OutFile $desktopPath/omsagent-template.yaml - - (Get-Content -Path $desktopPath/omsagent-template.yaml -Raw) -replace 'VALUE_AKS_RESOURCE_ID', $aksResourceId -replace 'VALUE_AKS_REGION', $aksResourceLocation -replace 'VALUE_WSID', $base64EncodedWsId -replace 'VALUE_KEY', $base64EncodedKey -replace 'VALUE_ACS_RESOURCE_NAME', $acsResourceName | Set-Content $desktopPath/deployments/omsagent-$clusterName.yaml - kubectl apply -f $desktopPath/deployments/omsagent-$clusterName.yaml - Write-Host "Successfully onboarded to health model omsagent" -ForegroundColor Green -} -catch { - Write-Host ("Agent deployment failed with an error: '" + $Error[0] + "' ") -ForegroundColor Red -} diff --git a/scripts/preview/health/HealthOnboarding.md b/scripts/preview/health/HealthOnboarding.md deleted file mode 100644 index 5e4db2b9b..000000000 --- a/scripts/preview/health/HealthOnboarding.md +++ /dev/null @@ -1,40 +0,0 @@ -## Overview -The following documentation outlines the steps required to upgrade an existing cluster onboarded to a Log Analytics workspace running the omsagent, to an agent running the workflow that generates health monitor signals into the same workspace. - -### Onboarding using a script (AKS) -We have a handy [script](https://github.com/Microsoft/OMS-docker/blob/dilipr/kubeHealth/health/HealthAgentOnboarding.ps1) which can onboard your AKS clusters to a version of the agent that can generate the health model. Read on to find out more - -#### Script Prerequisites -* script should run in an elevated command prompt -* kubectl should have been installed and be present in the path - -#### What does the script do: -* Installs necessary powershell modules -* Onboards Container Insights solution to the supplied LA workspace if not already onboarded -* Updates the cluster metadata to link the LA workspace ID to the cluster -* Installs the new agent that generates health monitor signals (using kubectl) - -#### Script Execution -* Download the script from [here](https://github.com/Microsoft/OMS-docker/blob/dilipr/kubeHealth/health/HealthAgentOnboarding.ps1) -* Run the script: - .\HealthAgentOnboarding.ps1 -aksResourceId -aksResourceLocation - -logAnalyticsWorkspaceResourceId (e.g./subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/dilipr-health-preview/providers/Microsoft.OperationalInsights/workspaces/dilipr-health-preview) - * Please make sure the right location of the AKS cluster is passed in to the script (without spaces e.g. eastus, southcentralus) - -#### Viewing the health model -* Navigate to -* There should be a new tab named "Health" in Cluster Insights -* Note: It might take about 15-20 min after the script runs for the data to show up in the Insights Page of the Cluster - - -### AKS Engine Onboarding -If your cluster is already onboarded to Monitoring, proceed directly to step 4 and continue from there on. -1. Add Container Insights Solution to your workspace using the instructions [here](http://aka.ms/coinhelmdoc) -2. Tag your AKS-Engine cluster appropriately using the instructions [here](http://aka.ms/coin-acs-tag-doc) -3. Set the current k8s context to be your AKS Engine cluster (the kube-config should refer to your AKS-Engine cluster) -4. Download the [omsagent-template-aks-engine.yaml](https://github.com/microsoft/OMS-docker/blob/dilipr/kubeHealth/health/omsagent-template-aks-engine.yaml) file to your local machine -5. Update the Values of VALUE_ACS_RESOURCE_NAME, VALUE_WSID {base 64 encoded workspace id} and VALUE_KEY {base 64 encoded workspace key}. See [here](https://github.com/Azure/aks-engine/blob/master/examples/addons/container-monitoring/README.md) on instructions to get the Workspace ID and Key of the file downloaded in Step 4 above -6. Run kubectl delete on the file {kubectl delete -f path_to_file_in_step_4} -7. Run kubectl apply on the file {kubectl apply -f path_to_file_in_step_4} - - diff --git a/scripts/preview/health/customOnboarding.json b/scripts/preview/health/customOnboarding.json deleted file mode 100644 index ecccc2ea7..000000000 --- a/scripts/preview/health/customOnboarding.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "aksResourceId": { - "type": "string", - "metadata": { - "description": "AKS Cluster resource id" - } - }, - "aksResourceLocation": { - "type": "string", - "metadata": { - "description": "Location of the AKS resource e.g. \"East US\"" - } - }, - "workspaceResourceId": { - "type": "string", - "metadata": { - "description": "Azure Monitor Log Analytics Resource ID" - } - } - }, - "resources": [ - { - "name": "[split(parameters('aksResourceId'),'/')[8]]", - "type": "Microsoft.ContainerService/managedClusters", - "location": "[parameters('aksResourceLocation')]", - "apiVersion": "2018-03-31", - "properties": { - "mode": "Incremental", - "id": "[parameters('aksResourceId')]", - "addonProfiles": { - "omsagent": { - "enabled": false, - "config": { - "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/scripts/preview/health/omsagent-template-aks-engine.yaml b/scripts/preview/health/omsagent-template-aks-engine.yaml deleted file mode 100644 index 5e063fd54..000000000 --- a/scripts/preview/health/omsagent-template-aks-engine.yaml +++ /dev/null @@ -1,586 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: omsagent - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: omsagent-reader -rules: - - apiGroups: [""] - resources: ["pods", "events", "nodes", "namespaces", "services"] - verbs: ["list", "get", "watch"] - - apiGroups: ["extensions"] - resources: ["deployments"] - verbs: ["list"] - - nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: omsagentclusterrolebinding -subjects: - - kind: ServiceAccount - name: omsagent - namespace: kube-system -roleRef: - kind: ClusterRole - name: omsagent-reader - apiGroup: rbac.authorization.k8s.io ---- -kind: ConfigMap -apiVersion: v1 -data: - kube.conf: |- - # Fluentd config file for OMS Docker - cluster components (kubeAPI) - #fluent forward plugin - - type forward - port 25235 - bind 0.0.0.0 - - - #Kubernetes pod inventory - - type kubepodinventory - tag oms.containerinsights.KubePodInventory - run_interval 60s - log_level debug - - - #Kubernetes events - - type kubeevents - tag oms.containerinsights.KubeEvents - run_interval 60s - log_level debug - - - #Kubernetes logs - - type kubelogs - tag oms.api.KubeLogs - run_interval 60s - - - #Kubernetes services - - type kubeservices - tag oms.containerinsights.KubeServices - run_interval 60s - log_level debug - - - #Kubernetes Nodes - - type kubenodeinventory - tag oms.containerinsights.KubeNodeInventory - run_interval 60s - log_level debug - - - #Kubernetes perf - - type kubeperf - tag oms.api.KubePerf - run_interval 60s - log_level debug - - - #Kubernetes health - - type kubehealth - tag kubehealth.ReplicaSet - run_interval 60s - log_level debug - - - #cadvisor perf- Windows nodes - - type wincadvisorperf - tag oms.api.wincadvisorperf - run_interval 60s - log_level debug - - - - type filter_inventory2mdm - log_level info - - - # custom_metrics_mdm filter plugin for perf data from windows nodes - - type filter_cadvisor2mdm - metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes - log_level info - - #health model aggregation filter - - type filter_health_model_builder - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubepods*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 5m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeevents*.buffer - buffer_queue_limit 10 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms_api - log_level debug - buffer_chunk_limit 10m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_kubernetes_logs*.buffer - buffer_queue_limit 10 - flush_interval 20s - retry_limit 10 - retry_wait 30s - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeservices*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/state/out_oms_kubenodes*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms - log_level debug - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_containernodeinventory*.buffer - buffer_queue_limit 20 - flush_interval 20s - retry_limit 10 - retry_wait 15s - max_retry_wait 9m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_mdm - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_mdm_*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - retry_mdm_post_wait_minutes 60 - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_wincadvisorperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_mdm - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_mdm_cdvisorperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - retry_mdm_post_wait_minutes 60 - - - - type out_oms_api - log_level debug - buffer_chunk_limit 10m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_kubehealth*.buffer - buffer_queue_limit 10 - flush_interval 20s - retry_limit 10 - retry_wait 30s - -metadata: - name: omsagent-rs-config - namespace: kube-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: omsagent-secret - namespace: kube-system -type: Opaque -data: - #BASE64 ENCODED (Both WSID & KEY) INSIDE DOUBLE QUOTE ("") - WSID: "VALUE_WSID" - KEY: "VALUE_KEY" ---- -kind: Service -apiVersion: v1 -metadata: - name: replicaset-service - namespace: kube-system -spec: - selector: - rsName: "omsagent-rs" - ports: - - protocol: TCP - port: 25235 - targetPort: in-rs-tcp ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: azurefile -provisioner: kubernetes.io/azure-file -mountOptions: - - dir_mode=0777 - - file_mode=0777 - - uid=1000 - - gid=1000 -parameters: - skuName: Standard_LRS ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:azure-cloud-provider -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system:azure-cloud-provider -roleRef: - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io - name: system:azure-cloud-provider -subjects: - - kind: ServiceAccount - name: persistent-volume-binder - namespace: kube-system ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: azurefile - namespace: kube-system -spec: - accessModes: - - ReadWriteMany - storageClassName: azurefile - resources: - requests: - storage: 10Mi ---- -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: omsagent - namespace: kube-system -spec: - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - dsName: "omsagent-ds" - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "6.0.0-1" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - containers: - - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 150m - memory: 600Mi - requests: - cpu: 75m - memory: 225Mi - env: - # - name: AKS_RESOURCE_ID - # value: "VALUE_AKS_RESOURCE_ID" - # - name: AKS_REGION - # value: "VALUE_AKS_REGION" - # Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - - name: ACS_RESOURCE_NAME - value: "VALUE_ACS_RESOURCE_NAME" - - name: DISABLE_KUBE_SYSTEM_LOG_COLLECTION - value: "true" - - name: CONTROLLER_TYPE - value: "DaemonSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - volumeMounts: - - mountPath: /hostfs - name: host-root - readOnly: true - - mountPath: /var/run/host - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /var/lib/docker/containers - name: containerlog-path - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - nodeSelector: - beta.kubernetes.io/os: linux - # Tolerate a NoSchedule taint on master that ACS Engine sets. - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "true" - effect: "NoSchedule" - volumes: - - name: host-root - hostPath: - path: / - - name: docker-sock - hostPath: - path: /var/run - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: containerlog-path - hostPath: - path: /var/lib/docker/containers - - name: azure-json-path - hostPath: - path: /etc/kubernetes - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: omsagent-rs - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - rsName: "omsagent-rs" - strategy: - type: RollingUpdate - template: - metadata: - labels: - rsName: "omsagent-rs" - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "6.0.0-1" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - containers: - - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 150m - memory: 500Mi - requests: - cpu: 50m - memory: 175Mi - env: - # - name: AKS_RESOURCE_ID - # value: "VALUE_AKS_RESOURCE_ID" - # - name: AKS_REGION - # value: "VALUE_AKS_REGION" - # Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - - name: ACS_RESOURCE_NAME - value: "aks-engine-health" - - name: CONTROLLER_TYPE - value: "ReplicaSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - - containerPort: 25235 - protocol: TCP - name: in-rs-tcp - volumeMounts: - - mountPath: /var/run/host - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /var/lib/docker/containers - name: containerlog-path - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - readOnly: true - - mountPath: /etc/config - name: omsagent-rs-config - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - - mountPath: "/mnt/azure" - name: azurefile-pv - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - nodeSelector: - beta.kubernetes.io/os: linux - kubernetes.io/role: agent - volumes: - - name: docker-sock - hostPath: - path: /var/run - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: containerlog-path - hostPath: - path: /var/lib/docker/containers - - name: azure-json-path - hostPath: - path: /etc/kubernetes - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: omsagent-rs-config - configMap: - name: omsagent-rs-config - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true - - name: azurefile-pv - persistentVolumeClaim: - claimName: azurefile diff --git a/scripts/preview/health/omsagent-template.yaml b/scripts/preview/health/omsagent-template.yaml deleted file mode 100644 index e58e9c33f..000000000 --- a/scripts/preview/health/omsagent-template.yaml +++ /dev/null @@ -1,586 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: omsagent - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: omsagent-reader -rules: -- apiGroups: [""] - resources: ["pods", "events", "nodes", "namespaces", "services"] - verbs: ["list", "get", "watch"] -- apiGroups: ["extensions"] - resources: ["deployments"] - verbs: ["list"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: omsagentclusterrolebinding -subjects: - - kind: ServiceAccount - name: omsagent - namespace: kube-system -roleRef: - kind: ClusterRole - name: omsagent-reader - apiGroup: rbac.authorization.k8s.io ---- -kind: ConfigMap -apiVersion: v1 -data: - kube.conf: |- - # Fluentd config file for OMS Docker - cluster components (kubeAPI) - #fluent forward plugin - - type forward - port 25235 - bind 0.0.0.0 - - - #Kubernetes pod inventory - - type kubepodinventory - tag oms.containerinsights.KubePodInventory - run_interval 60s - log_level debug - - - #Kubernetes events - - type kubeevents - tag oms.containerinsights.KubeEvents - run_interval 60s - log_level debug - - - #Kubernetes logs - - type kubelogs - tag oms.api.KubeLogs - run_interval 60s - - - #Kubernetes services - - type kubeservices - tag oms.containerinsights.KubeServices - run_interval 60s - log_level debug - - - #Kubernetes Nodes - - type kubenodeinventory - tag oms.containerinsights.KubeNodeInventory - run_interval 60s - log_level debug - - - #Kubernetes perf - - type kubeperf - tag oms.api.KubePerf - run_interval 60s - log_level debug - - - #Kubernetes health - - type kubehealth - tag kubehealth.ReplicaSet - run_interval 60s - log_level debug - - - #cadvisor perf- Windows nodes - - type wincadvisorperf - tag oms.api.wincadvisorperf - run_interval 60s - log_level debug - - - - type filter_inventory2mdm - log_level info - - - # custom_metrics_mdm filter plugin for perf data from windows nodes - - type filter_cadvisor2mdm - metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes - log_level info - - #health model aggregation filter - - type filter_health_model_builder - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubepods*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 5m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeevents*.buffer - buffer_queue_limit 10 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms_api - log_level debug - buffer_chunk_limit 10m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_kubernetes_logs*.buffer - buffer_queue_limit 10 - flush_interval 20s - retry_limit 10 - retry_wait 30s - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeservices*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/state/out_oms_kubenodes*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_oms - log_level debug - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_containernodeinventory*.buffer - buffer_queue_limit 20 - flush_interval 20s - retry_limit 10 - retry_wait 15s - max_retry_wait 9m - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_kubeperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_mdm - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_mdm_*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - retry_mdm_post_wait_minutes 60 - - - - type out_oms - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_wincadvisorperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - - - - type out_mdm - log_level debug - num_threads 5 - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_mdm_cdvisorperf*.buffer - buffer_queue_limit 20 - buffer_queue_full_action drop_oldest_chunk - flush_interval 20s - retry_limit 10 - retry_wait 30s - max_retry_wait 9m - retry_mdm_post_wait_minutes 60 - - - - type out_oms_api - log_level debug - buffer_chunk_limit 10m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_api_kubehealth*.buffer - buffer_queue_limit 10 - flush_interval 20s - retry_limit 10 - retry_wait 30s - -metadata: - name: omsagent-rs-config - namespace: kube-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: omsagent-secret - namespace: kube-system -type: Opaque -data: - #BASE64 ENCODED (Both WSID & KEY) INSIDE DOUBLE QUOTE ("") - WSID: "VALUE_WSID" - KEY: "VALUE_KEY" ---- -kind: Service -apiVersion: v1 -metadata: - name: replicaset-service - namespace: kube-system -spec: - selector: - rsName: "omsagent-rs" - ports: - - protocol: TCP - port: 25235 - targetPort: in-rs-tcp ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: azurefile -provisioner: kubernetes.io/azure-file -mountOptions: - - dir_mode=0777 - - file_mode=0777 - - uid=1000 - - gid=1000 -parameters: - skuName: Standard_LRS ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:azure-cloud-provider -rules: -- apiGroups: [''] - resources: ['secrets'] - verbs: ['get','create'] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system:azure-cloud-provider -roleRef: - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io - name: system:azure-cloud-provider -subjects: -- kind: ServiceAccount - name: persistent-volume-binder - namespace: kube-system ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: azurefile - namespace: kube-system -spec: - accessModes: - - ReadWriteMany - storageClassName: azurefile - resources: - requests: - storage: 10Mi ---- -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: omsagent - namespace: kube-system -spec: - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - dsName: "omsagent-ds" - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "6.0.0-1" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - containers: - - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 150m - memory: 600Mi - requests: - cpu: 75m - memory: 225Mi - env: - - name: AKS_RESOURCE_ID - value: "VALUE_AKS_RESOURCE_ID" - - name: AKS_REGION - value: "VALUE_AKS_REGION" - # Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - # - name: ACS_RESOURCE_NAME - # value: "my_acs_cluster_name" - - name: DISABLE_KUBE_SYSTEM_LOG_COLLECTION - value: "true" - - name: CONTROLLER_TYPE - value: "DaemonSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - volumeMounts: - - mountPath: /hostfs - name: host-root - readOnly: true - - mountPath: /var/run/host - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /var/lib/docker/containers - name: containerlog-path - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - nodeSelector: - beta.kubernetes.io/os: linux - # Tolerate a NoSchedule taint on master that ACS Engine sets. - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "true" - effect: "NoSchedule" - volumes: - - name: host-root - hostPath: - path: / - - name: docker-sock - hostPath: - path: /var/run - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: containerlog-path - hostPath: - path: /var/lib/docker/containers - - name: azure-json-path - hostPath: - path: /etc/kubernetes - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: omsagent-rs - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - rsName: "omsagent-rs" - strategy: - type: RollingUpdate - template: - metadata: - labels: - rsName: "omsagent-rs" - annotations: - agentVersion: "1.10.0.1" - dockerProviderVersion: "6.0.0-1" - schema-versions: "v1" - spec: - serviceAccountName: omsagent - containers: - - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 150m - memory: 500Mi - requests: - cpu: 50m - memory: 175Mi - env: - - name: AKS_RESOURCE_ID - value: "VALUE_AKS_RESOURCE_ID" - - name: AKS_REGION - value: "VALUE_AKS_REGION" - #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - # - name: ACS_RESOURCE_NAME - # value: "aks-engine-health" - - name: CONTROLLER_TYPE - value: "ReplicaSet" - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - - containerPort: 25235 - protocol: TCP - name: in-rs-tcp - volumeMounts: - - mountPath: /var/run/host - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /var/lib/docker/containers - name: containerlog-path - - mountPath: /etc/kubernetes/host - name: azure-json-path - - mountPath: /etc/omsagent-secret - name: omsagent-secret - readOnly: true - - mountPath : /etc/config - name: omsagent-rs-config - - mountPath: /etc/config/settings - name: settings-vol-config - readOnly: true - - mountPath: "/mnt/azure" - name: azurefile-pv - livenessProbe: - exec: - command: - - /bin/bash - - -c - - /opt/livenessprobe.sh - initialDelaySeconds: 60 - periodSeconds: 60 - nodeSelector: - beta.kubernetes.io/os: linux - kubernetes.io/role: agent - volumes: - - name: docker-sock - hostPath: - path: /var/run - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: containerlog-path - hostPath: - path: /var/lib/docker/containers - - name: azure-json-path - hostPath: - path: /etc/kubernetes - - name: omsagent-secret - secret: - secretName: omsagent-secret - - name: omsagent-rs-config - configMap: - name: omsagent-rs-config - - name: settings-vol-config - configMap: - name: container-azm-ms-agentconfig - optional: true - - name: azurefile-pv - persistentVolumeClaim: - claimName: azurefile diff --git a/scripts/preview/health/optouttemplate.json b/scripts/preview/health/optouttemplate.json deleted file mode 100644 index b036aba24..000000000 --- a/scripts/preview/health/optouttemplate.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "aksResourceId": { - "type": "string", - "metadata": { - "description": "AKS Cluster resource id" - } - }, - "aksResourceLocation": { - "type": "string", - "metadata": { - "description": "Location of the AKS resource e.g. \"East US\"" - } - } - }, - "resources": [ - { - "name": "[split(parameters('aksResourceId'),'/')[8]]", - "type": "Microsoft.ContainerService/managedClusters", - "location": "[parameters('aksResourceLocation')]", - "apiVersion": "2018-03-31", - "properties": { - "mode": "Incremental", - "id": "[parameters('aksResourceId')]", - "addonProfiles": { - "omsagent": { - "enabled": false, - "config": null - } - } - } - } - ] -} diff --git a/scripts/troubleshoot/LogCollection/AgentLogCollection.sh b/scripts/troubleshoot/LogCollection/AgentLogCollection.sh new file mode 100644 index 000000000..42a351e6e --- /dev/null +++ b/scripts/troubleshoot/LogCollection/AgentLogCollection.sh @@ -0,0 +1,226 @@ +#!/bin/bash +# +# Copyright (c) Microsoft Corporation. +# +# This script will collect all logs from the replicaset agent pod and a random daemonset pod, also collect onboard logs with processes +# +# Author Nina Li + +Red='\033[0;31m' +Cyan='\033[0;36m' +NC='\033[0m' # No Color + +init() +{ + echo -e "Preparing for log collection..." | tee -a Tool.log + + if ! cmd="$(type -p kubectl)" || [[ -z $cmd ]]; then + echo -e "${Red}Command kubectl not found, please install it firstly, exit...${NC}" + cd .. + rm -rf $output_path + exit + fi + + if ! cmd="$(type -p tar)" || [[ -z $cmd ]]; then + echo -e "${Red}Command tar not found, please install it firstly, exit...${NC}" + cd .. + rm -rf $output_path + exit + fi + + cmd=`kubectl get nodes 2>&1` + if [[ $cmd == *"refused"* ]];then + echo -e "${Red}Fail to connect your AKS, please fisrlty connect to cluster by command: az aks get-credentials --resource-group myResourceGroup --name myAKSCluster${NC}" + cd .. + rm -rf $output_path + exit + fi + + cmd=`kubectl get nodes | sed 1,1d | awk '{print $2}'` + for node in $cmd + do + if [ `echo $node | tr -s '[:upper:]' '[:lower:]'` != "ready" ]; then + kubectl get nodes + echo -e "${Red} One or more AKS node is not ready, please start this node firstly for log collection, exit...${NC}" + cd .. + rm -rf $output_path + exit + fi + done + echo -e "Prerequistes check is done, all good" | tee -a Tool.log + + echo -e "Saving cluster information" | tee -a Tool.log + + cmd=`kubectl cluster-info 2>&1` + if [[ $cmd == *"refused"* ]];then + echo -e "${Red}Fail to get cluster info, please check your AKS status fistly, exit...${NC}" + cd .. + rm -rf $output_path + exit + else + echo $cmd >> Tool.log + echo -e "cluster info saved to Tool.log" | tee -a Tool.log + fi + +} + +ds_logCollection() +{ + echo -e "Collecting logs from ${ds_pod}..." | tee -a Tool.log + kubectl describe pod ${ds_pod} --namespace=kube-system > describe_${ds_pod}.txt + kubectl logs ${ds_pod} --container omsagent --namespace=kube-system > logs_${ds_pod}.txt + kubectl logs ${ds_pod} --container omsagent-prometheus --namespace=kube-system > logs_${ds_pod}_prom.txt + kubectl exec ${ds_pod} -n kube-system --request-timeout=10m -- ps -ef > process_${ds_pod}.txt + + cmd=`kubectl exec ${ds_pod} -n kube-system -- ls /var/opt/microsoft 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/var/opt/microsoft not exist on ${ds_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-daemonset --namespace=kube-system --container omsagent > /dev/null + kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-prom-daemonset --namespace=kube-system --container omsagent-prometheus > /dev/null + kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-daemonset-mdsd --namespace=kube-system --container omsagent > /dev/null + kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-prom-daemonset-mdsd --namespace=kube-system --container omsagent-prometheus > /dev/null + fi + + kubectl exec ${ds_pod} --namespace=kube-system -- ls /var/opt/microsoft/docker-cimprov/state/ContainerInventory > containerID_${ds_pod}.txt 2>&1 + + cmd=`kubectl exec ${ds_pod} -n kube-system -- ls /etc/fluent 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/etc/fluent not exist on ${ds_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${ds_pod}:/etc/fluent/container.conf omsagent-daemonset/container_${ds_pod}.conf --namespace=kube-system --container omsagent > /dev/null + kubectl cp ${ds_pod}:/etc/fluent/container.conf omsagent-prom-daemonset/container_${ds_pod}_prom.conf --namespace=kube-system --container omsagent-prometheus > /dev/null + fi + + cmd=`kubectl exec ${ds_pod} -n kube-system -- ls /etc/opt/microsoft/docker-cimprov 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/etc/opt/microsoft/docker-cimprov not exist on ${ds_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${ds_pod}:/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf omsagent-daemonset/td-agent-bit.conf --namespace=kube-system --container omsagent > /dev/null + kubectl cp ${ds_pod}:/etc/opt/microsoft/docker-cimprov/telegraf.conf omsagent-daemonset/telegraf.conf --namespace=kube-system --container omsagent > /dev/null + kubectl cp ${ds_pod}:/etc/opt/microsoft/docker-cimprov/telegraf.conf omsagent-prom-daemonset/telegraf.conf --namespace=kube-system --container omsagent-prometheus > /dev/null + kubectl cp ${ds_pod}:/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf omsagent-prom-daemonset/td-agent-bit.conf --namespace=kube-system --container omsagent-prometheus > /dev/null + fi + echo -e "Complete log collection from ${ds_pod}!" | tee -a Tool.log +} + +win_logCollection() +{ + echo -e "Collecting logs from ${ds_win_pod}, windows pod will take several minutes for log collection, please dont exit forcely..." | tee -a Tool.log + kubectl describe pod ${ds_win_pod} --namespace=kube-system > describe_${ds_win_pod}.txt + kubectl logs ${ds_win_pod} --container omsagent-win --namespace=kube-system > logs_${ds_win_pod}.txt + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell Get-Process > process_${ds_win_pod}.txt + + cmd=`kubectl exec ${ds_win_pod} -n kube-system -- powershell ls /etc 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/etc/ not exist on ${ds_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${ds_win_pod}:/etc/fluent-bit omsagent-win-daemonset-fbit --namespace=kube-system > /dev/null + kubectl cp ${ds_win_pod}:/etc/telegraf/telegraf.conf omsagent-win-daemonset-fbit/telegraf.conf --namespace=kube-system > /dev/null + + echo -e "${Cyan}If your log size are too large, log collection of windows node may fail. You can reduce log size by re-creating windows pod ${NC}" + # for some reason copying logs out of /etc/omsagentwindows doesn't work (gives a permission error), but exec then cat does work. + # kubectl cp ${ds_win_pod}:/etc/omsagentwindows omsagent-win-daemonset --namespace=kube-system + mkdir -p omsagent-win-daemonset + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/kubernetes_perf_log.txt > omsagent-win-daemonset/kubernetes_perf_log.txt + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/appinsights_error.log > omsagent-win-daemonset/appinsights_error.log + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/filter_cadvisor2mdm.log > omsagent-win-daemonset/filter_cadvisor2mdm.log + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/fluent-bit-out-oms-runtime.log > omsagent-win-daemonset/fluent-bit-out-oms-runtime.log + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/kubernetes_client_log.txt > omsagent-win-daemonset/kubernetes_client_log.txt + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/mdm_metrics_generator.log > omsagent-win-daemonset/mdm_metrics_generator.log + kubectl exec ${ds_win_pod} -n kube-system --request-timeout=10m -- powershell cat /etc/omsagentwindows/out_oms.conf > omsagent-win-daemonset/out_oms.conf + fi + + echo -e "Complete log collection from ${ds_win_pod}!" | tee -a Tool.log +} + +rs_logCollection() +{ + echo -e "Collecting logs from ${rs_pod}..." + kubectl describe pod ${rs_pod} --namespace=kube-system > describe_${rs_pod}.txt + kubectl logs ${rs_pod} --container omsagent --namespace=kube-system > logs_${rs_pod}.txt + kubectl exec ${rs_pod} -n kube-system --request-timeout=10m -- ps -ef > process_${rs_pod}.txt + + cmd=`kubectl exec ${rs_pod} -n kube-system -- ls /var/opt/microsoft 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/var/opt/microsoft not exist on ${rs_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${rs_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-replicaset --namespace=kube-system > /dev/null + kubectl cp ${rs_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-replicaset-mdsd --namespace=kube-system > /dev/null + fi + + cmd=`kubectl exec ${rs_pod} -n kube-system -- ls /etc/fluent 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/etc/fluent not exist on ${rs_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${rs_pod}:/etc/fluent/kube.conf omsagent-replicaset/kube_${rs_pod}.conf --namespace=kube-system --container omsagent > /dev/null + fi + + cmd=`kubectl exec ${rs_pod} -n kube-system -- ls /etc/opt/microsoft/docker-cimprov 2>&1` + if [[ $cmd == *"cannot access"* ]];then + echo -e "${Red}/etc/opt/microsoft/docker-cimprov not exist on ${rs_pod}${NC}" | tee -a Tool.log + else + kubectl cp ${rs_pod}:/etc/opt/microsoft/docker-cimprov/td-agent-bit-rs.conf omsagent-replicaset/td-agent-bit.conf --namespace=kube-system --container omsagent > /dev/null + kubectl cp ${rs_pod}:/etc/opt/microsoft/docker-cimprov/telegraf-rs.conf omsagent-replicaset/telegraf-rs.conf --namespace=kube-system --container omsagent > /dev/null + fi + echo -e "Complete log collection from ${rs_pod}!" | tee -a Tool.log +} + +other_logCollection() +{ + echo -e "Collecting onboard logs..." + export deploy=$(kubectl get deployment --namespace=kube-system | grep -E omsagent | head -n 1 | awk '{print $1}') + if [ -z "$deploy" ];then + echo -e "${Red}there is not omsagent deployment, skipping log collection of deployment${NC}" | tee -a Tool.log + else + kubectl get deployment $deploy --namespace=kube-system -o yaml > deployment_${deploy}.txt + fi + + export config=$(kubectl get configmaps --namespace=kube-system | grep -E container-azm-ms-agentconfig | head -n 1 | awk '{print $1}') + if [ -z "$config" ];then + echo -e "${Red}configMap named container-azm-ms-agentconfig is not found, if you created configMap for omsagent, please manually save your custom configMap of omsagent by command: kubectl get configmaps --namespace=kube-system -o yaml > configMap.yaml${NC}" | tee -a Tool.log + else + kubectl get configmaps $config --namespace=kube-system -o yaml > ${config}.yaml + fi + + kubectl get nodes > node.txt + echo -e "Complete onboard log collection!" | tee -a Tool.log +} + +#main +output_path="AKSInsights-logs.$(date +%s).`hostname`" +mkdir -p $output_path +cd $output_path + +init + +export ds_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-[a-z0-9]{5} | head -n 1) +if [ -z "$ds_pod" ];then + echo -e "${Red}daemonset pod do not exist, skipping log collection for daemonset pod${NC}" | tee -a Tool.log +else + ds_logCollection +fi + +export ds_win_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-win-[a-z0-9]{5} | head -n 1) +if [ -z "$ds_win_pod" ];then + echo -e "${Cyan} windows agent pod do not exist, skipping log collection for windows agent pod ${NC}" | tee -a Tool.log +else + win_logCollection +fi + +export rs_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-rs-[a-z0-9]{5} | head -n 1) +if [ -z "$rs_pod" ];then + echo -e "${Red}replicaset pod do not exist, skipping log collection for replicaset pod ${NC}" | tee -a Tool.log +else + rs_logCollection +fi + +other_logCollection + +cd .. +echo +echo -e "Archiving logs..." +tar -czf $output_path.tgz $output_path +rm -rf $output_path + +echo "log files have been written to ${output_path}.tgz in current folder" diff --git a/scripts/troubleshoot/LogCollection/README.md b/scripts/troubleshoot/LogCollection/README.md new file mode 100644 index 000000000..9c867837b --- /dev/null +++ b/scripts/troubleshoot/LogCollection/README.md @@ -0,0 +1,46 @@ +# Container Insights Log collector + +This tool will collect: +* agent logs from linux ds and rs pods; +* agent logs from windows pod if enabled; +* cluster/node info, pod deployment, configMap, process logs etc.. + +## Prerequisites +* kubectl: az aks install-cli +* tar (installed by default) +* all nodes should be running on AKS +* AKS Insights are enabled: https://docs.microsoft.com/en-us/azure/azure-monitor/containers/container-insights-onboard + +Otherwise, script will report error message and exit. + +## How to run +``` +az login --use-device-code # login to azure +az account set --subscription +az aks get-credentials --resource-group --name --file ~/ClusterKubeConfig +export KUBECONFIG=~/ClusterKubeConfig + +wget https://raw.githubusercontent.com/microsoft/Docker-Provider/ci_dev/scripts/troubleshoot/LogCollection/AgentLogCollection.sh && bash ./AgentLogCollection.sh +``` + +Output: +``` +Preparing for log collection... +Prerequistes check is done, all good +Saving cluster information +cluster info saved to Tool.log +Collecting logs from omsagent-5kwzn... +Defaulted container "omsagent" out of: omsagent, omsagent-prometheus +Complete log collection from omsagent-5kwzn! +Collecting logs from omsagent-win-krcpv, windows pod will take several minutes for log collection, please dont exit forcely... +If your log size are too large, log collection of windows node may fail. You can reduce log size by re-creating windows pod +Complete log collection from omsagent-win-krcpv! +Collecting logs from omsagent-rs-6fc95c45cf-qjsdb... +Complete log collection from omsagent-rs-6fc95c45cf-qjsdb! +Collecting onboard logs... +configMap named container-azm-ms-agentconfig is not found, if you created configMap for omsagent, please use command to save your custom configMap of omsagent: kubectl get configmaps --namespace=kube-system -o yaml > configMap.yaml +Complete onboard log collection! + +Archiving logs... +log files have been written to AKSInsights-logs.1649655490.ubuntu1804.tgz in current folder +``` diff --git a/scripts/troubleshoot/README.md b/scripts/troubleshoot/README.md index 650a5df6f..c68e12913 100644 --- a/scripts/troubleshoot/README.md +++ b/scripts/troubleshoot/README.md @@ -50,8 +50,8 @@ You can use the troubleshooting script provided [here](https://raw.githubusercon Steps: - Open powershell using the [cloudshell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview) in the azure portal. > Note: This script supported on any Powershell supported environment: Windows and Non-Windows. - For Linux, refer [Install-Powershell-On-Linux](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-linux?view=powershell-6) and - For Mac OS, refer [install-powershell-core-on-mac](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6) how to install powershell + For Linux, refer [Install-Powershell-On-Linux](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-linux?view=powershell-7) and + For Mac OS, refer [install-powershell-core-on-mac](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-7) how to install powershell - Make sure that you're using powershell (selected by default) - Run the following command to change home directory - `cd ~` - Run the following command to download the script - `curl -LO https://raw.githubusercontent.com/microsoft/Docker-Provider/ci_prod/scripts/troubleshoot/TroubleshootError.ps1` diff --git a/scripts/troubleshoot/TroubleshootError.ps1 b/scripts/troubleshoot/TroubleshootError.ps1 index 6d97c53d5..8dcded546 100644 --- a/scripts/troubleshoot/TroubleshootError.ps1 +++ b/scripts/troubleshoot/TroubleshootError.ps1 @@ -38,6 +38,8 @@ if (($null -eq $ClusterResourceId) -or ($ClusterResourceId.Split("/").Length -ne exit 1 } +$UseAADAuth = $false +$ClusterRegion = "" $isClusterAndWorkspaceInDifferentSubs = $false $ClusterType = "AKS" if ($ClusterResourceId.ToLower().Contains("microsoft.containerservice/openshiftmanagedclusters") -eq $true) { @@ -50,8 +52,9 @@ $azResourcesModule = Get-Module -ListAvailable -Name Az.Resources $azOperationalInsights = Get-Module -ListAvailable -Name Az.OperationalInsights $azAksModule = Get-Module -ListAvailable -Name Az.Aks $azARGModule = Get-Module -ListAvailable -Name Az.ResourceGraph +$azMonitorModule = Get-Module -ListAvailable -Name Az.Monitor -if (($null -eq $azAksModule) -or ($null -eq $azARGModule) -or ($null -eq $azAccountModule) -or ($null -eq $azResourcesModule) -or ($null -eq $azOperationalInsights)) { +if (($null -eq $azAksModule) -or ($null -eq $azARGModule) -or ($null -eq $azAccountModule) -or ($null -eq $azResourcesModule) -or ($null -eq $azOperationalInsights) -or ($null -eq $azMonitorModule)) { $isWindowsMachine = $true if ($PSVersionTable -and $PSVersionTable.PSEdition -contains "core") { @@ -75,7 +78,7 @@ if (($null -eq $azAksModule) -or ($null -eq $azARGModule) -or ($null -eq $azAcco } $message = "This script will try to install the latest versions of the following Modules : ` - Az.Ak,Az.ResourceGraph, Az.Resources, Az.Accounts and Az.OperationalInsights using the command` + Az.Ak,Az.ResourceGraph, Az.Resources, Az.Accounts, Az.OperationalInsights and Az.Monitor using the command` `'Install-Module {Insert Module Name} -Repository PSGallery -Force -AllowClobber -ErrorAction Stop -WarningAction Stop' `If you do not have the latest version of these Modules, this troubleshooting script may not run." $question = "Do you want to Install the modules and run the script or just run the script?" @@ -149,6 +152,18 @@ if (($null -eq $azAksModule) -or ($null -eq $azARGModule) -or ($null -eq $azAcco } } + if ($null -eq $azMonitorModule) { + try { + + Write-Host("Installing Az.Monitor...") + Install-Module Az.Monitor -Repository PSGallery -Force -AllowClobber -ErrorAction Stop + } + catch { + Write-Host("Close other powershell logins and try installing the latest modules for Az.OperationalInsights in a new powershell window: eg. 'Install-Module Az.Monitor -Repository PSGallery -Force'") -ForegroundColor Red + Stop-Transcript + exit 1 + } + } } 1 { if ($null -eq $azARGModule) { @@ -209,6 +224,16 @@ if (($null -eq $azAksModule) -or ($null -eq $azARGModule) -or ($null -eq $azAcco } } + if ($null -eq $azMonitorModule) { + try { + Import-Module Az.Monitor -ErrorAction Stop + } + catch { + Write-Host("Could not import Az.Monitor... Please reinstall this Module") -ForegroundColor Red + Stop-Transcript + exit 1 + } + } } 2 { Write-Host("") @@ -265,12 +290,16 @@ $ResourceGroupName = $ClusterResourceId.split("/")[4] $ClusterName = $ClusterResourceId.split("/")[8] # -# Subscription existance and access check +# Subscription existence and access check # if ($null -eq $account.Account) { try { Write-Host("Please login...") - Login-AzAccount -subscriptionid $ClusterSubscriptionId + if ($isWindowsMachine) { + Login-AzAccount -subscriptionid $ClusterSubscriptionId + } else { + Login-AzAccount -subscriptionid $ClusterSubscriptionId -UseDeviceAuthentication + } } catch { Write-Host("") @@ -331,26 +360,29 @@ try { } else { Write-Host("Successfully checked '" + $ClusterType + "' Cluster details...") -ForegroundColor Green + $ClusterRegion = $ResourceDetailsArray.Location Write-Host("") foreach ($ResourceDetail in $ResourceDetailsArray) { if ($ResourceDetail.ResourceType -eq "Microsoft.ContainerService/managedClusters") { - #gangams: profile can be different casing so convert properties to lowecase and extract it - $props = ($ResourceDetail.Properties | ConvertTo-Json).toLower() | ConvertFrom-Json; + $addonProfiles = ($ResourceDetail.Properties.addonProfiles | ConvertTo-Json).toLower() | ConvertFrom-Json - if ($null -eq $props.addonprofiles.omsagent.config) { + if (($nul -eq $addonProfiles) -or ($null -eq $addonProfiles.omsagent) -or ($null -eq $addonProfiles.omsagent.config)) { Write-Host("Your cluster isn't onboarded to Azure monitor for containers. Please refer to the following documentation to onboard:") -ForegroundColor Red; + $clusterProperies = ($ResourceDetail.Properties | ConvertTo-Json) + Write-Host("Cluster Properties found: " + $clusterProperies) -ForegroundColor Red; Write-Host($AksOptInLink) -ForegroundColor Red; Write-Host(""); Stop-Transcript exit 1 } - $omsagentconfig = $props.addonprofiles.omsagent.config; + $LogAnalyticsWorkspaceResourceID = $addonProfiles.omsagent.config.loganalyticsworkspaceresourceid + $isAADAuth = $addonProfiles.omsagent.config.useaadauth + if ($true -eq $isAADAuth) { + $UseAADAuth = $true + } - #gangams - figure out betterway to do this - $omsagentconfig = $omsagentconfig.Trim("{", "}"); - $LogAnalyticsWorkspaceResourceID = $omsagentconfig.split("=")[1]; - Write-Host("AKS Cluster ResourceId: '" + $ResourceDetail.ResourceId + "' "); + Write-Host("AKS Cluster ResourceId: '" + $ResourceDetail.ResourceId + "', LogAnalyticsWorkspaceResourceId: '" + $LogAnalyticsWorkspaceResourceID + "', UseAADAuth: '" + $UseAADAuth + "'"); break } } @@ -389,15 +421,15 @@ catch { } -if ("AKS" -eq $ClusterType ) { +if (("AKS" -eq $ClusterType ) -and ($false -eq $UseAADAuth)) { Write-Host("Currently checking if the cluster is onboarded to custom metrics for Azure monitor for containers..."); #Pre requisite - need cluster spn object Id try { - $clusterDetails = Get-AzAks -Id $ClusterResourceId -ErrorVariable clusterFetchError -ErrorAction SilentlyContinue + $clusterDetails = Get-AzAksCluster -Id $ClusterResourceId -ErrorVariable clusterFetchError -ErrorAction SilentlyContinue Write-Host($clusterDetails | Format-List | Out-String) # Check to see if SP exists, if it does use it. Else use MSI - if ($clusterDetails.ServicePrincipalProfile -ne $null -and $clusterDetails.ServicePrincipalProfile.ClientId -ne $null -and $clusterDetails.ServicePrincipalProfile.ClientId -ne "") { + if ($clusterDetails.ServicePrincipalProfile -ne $null -and $clusterDetails.ServicePrincipalProfile.ClientId -ne $null -and $clusterDetails.ServicePrincipalProfile.ClientId -ne "msi") { Write-Host('Attempting to provide permissions to service principal...') -ForegroundColor Green $clusterSpnClientID = $clusterDetails.ServicePrincipalProfile.ClientId $isServicePrincipal = $true @@ -625,77 +657,144 @@ else { $WorkspacePricingTier = $WorkspaceInformation.sku Write-Host("Pricing tier of the configured LogAnalytics workspace: '" + $WorkspacePricingTier + "' ") -ForegroundColor Green - try { - $WorkspaceIPDetails = Get-AzOperationalInsightsIntelligencePacks -ResourceGroupName $workspaceResourceGroupName -WorkspaceName $workspaceName -ErrorAction Stop -WarningAction silentlyContinue - Write-Host("Successfully fetched workspace IP details...") -ForegroundColor Green - Write-Host("") - } - catch { - Write-Host("") - Write-Host("Failed to get the list of solutions onboarded to the workspace. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit 1 - } - - try { - $ContainerInsightsIndex = $WorkspaceIPDetails.Name.IndexOf("ContainerInsights") - Write-Host("Successfully located ContainerInsights solution") -ForegroundColor Green - Write-Host("") - } - catch { - Write-Host("Failed to get ContainerInsights solution details from the workspace") -ForegroundColor Red - Write-Host("") - Stop-Transcript - exit 1 - } - - $isSolutionOnboarded = $WorkspaceIPDetails.Enabled[$ContainerInsightsIndex] - if ($isSolutionOnboarded) { - if ($WorkspacePricingTier -eq "Free") { - Write-Host("Pricing tier of the configured LogAnalytics workspace is Free so you may need to upgrade to pricing tier to non-Free") -ForegroundColor Yellow - } - } - else { + if ($true -eq $UseAADAuth) { # - # Check contributor access to WS + # Check existence of the ContainerInsightsExtension DCR # - $message = "Detected that there is a workspace associated with this cluster, but workspace - '" + $workspaceName + "' in subscription '" + $workspaceSubscriptionId + "' IS NOT ONBOARDED with container health solution." - Write-Host($message) - $question = " Do you want to onboard container health to the workspace?" - - $choices = New-Object Collections.ObjectModel.Collection[Management.Automation.Host.ChoiceDescription] - $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Yes')) - $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&No')) - - $decision = $Host.UI.PromptForChoice($message, $question, $choices, 0) - - if ($decision -eq 0) { - Write-Host("Deploying template to onboard container health : Please wait...") - - $DeploymentName = "ContainerInsightsSolutionOnboarding-" + ((Get-Date).ToUniversalTime()).ToString('MMdd-HHmm') - $Parameters = @{ } - $Parameters.Add("workspaceResourceId", $LogAnalyticsWorkspaceResourceID) - $Parameters.Add("workspaceRegion", $WorkspaceLocation) - $Parameters - try { - New-AzResourceGroupDeployment -Name $DeploymentName ` - -ResourceGroupName $workspaceResourceGroupName ` - -TemplateUri https://raw.githubusercontent.com/microsoft/Docker-Provider/ci_prod/scripts/onboarding/templates/azuremonitor-containerSolution.json ` - -TemplateParameterObject $Parameters -ErrorAction Stop` + try { + $dcrRuleName = "MSCI-" + $ClusterName + "-" + $ClusterRegion + $dcrRule = Get-AzDataCollectionRule -ResourceGroupName $workspaceResourceGroupName -RuleName $dcrRuleName -ErrorAction Stop -WarningAction silentlyContinue + Write-Host("Successfully fetched Data Collection Rule...") -ForegroundColor Green + $extensionNameInDCR = $dcrRule.DataSources.Extensions.ExtensionName + if ($extensionNameInDCR -eq "ContainerInsights") { + $laDestinations = $dcrRule.Destinations.LogAnalytics + if (($null -ne $laDestinations) -and ($laDestinations.Length -gt 0) -and ($LogAnalyticsWorkspaceResourceID -eq $laDestinations[0].WorkspaceResourceId)) { + Write-Host("Successfully validated Data Collection Rule is valid...") -ForegroundColor Green + } else { + Write-Host("") + Write-Host("Data Collection Rule: '" + $dcrRuleName + "' found has Log Analytics(LA) workspace which different from the Log Analytics workspace in Monitoring addon.") -ForegroundColor Red + $laWorkspaceResIdInDCR = $laDestinations[0].WorkspaceResourceId + Write-Host("LA workspace found in Data Collection Rule: '" + $laWorkspaceResIdInDCR + "' but where as LA workspace in Monitoring Addon: '" + $LogAnalyticsWorkspaceResourceID + "'.") -ForegroundColor Red + Write-Host("") + Stop-Transcript + exit 1 + } + } else { + Write-Host("") + Write-Host("Data Collection Rule: '" + $dcrRuleName + "' found is not valid ContainerInsights extension DCR.") -ForegroundColor Red Write-Host("") - Write-Host("Successfully added Container Insights Solution") -ForegroundColor Green + Stop-Transcript + exit 1 + } + } + catch { + Write-Host("") + Write-Host("Failed to get the data collection Rule: '" + $dcrRuleName + "'. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red + Write-Host("If DataCollectionRule :'" + $dcrRuleName + "' has been deleted accidentally, disable and enable Monitoring addon back to get this fixed.") -ForegroundColor Red + Write-Host("") + Stop-Transcript + exit 1 + } + # + # Check existence of the ContainerInsightsExtension DCR-A on the cluster resource + # + try { + $dcrAssociation = Get-AzDataCollectionRuleAssociation -TargetResourceId $ClusterResourceId -AssociationName "ContainerInsightsExtension" -ErrorAction Stop -WarningAction silentlyContinue + Write-Host("Successfully fetched ContainerInsightsExtension Data Collection Rule Association ...") -ForegroundColor Green + if ($null -eq $dcrAssociation) { + Write-Host("") + Write-Host("ContainerInsightsExtension Data Collection Rule Association doenst exist.") -ForegroundColor Red Write-Host("") + Stop-Transcript + exit 1 } - catch { - Write-Host ("Template deployment failed with an error: '" + $Error[0] + "' ") -ForegroundColor Red - Write-Host("Please contact us by emailing askcoin@microsoft.com for help") -ForegroundColor Red + } + catch { + Write-Host("") + Write-Host("Failed to get the data collection Rule Association. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red + Write-Host("If ContainerInsightsExtension DataCollectionRule Association has been deleted accidentally, disable and enable Monitoring addon back to get this fixed.") -ForegroundColor Red + Write-Host("") + Stop-Transcript + exit 1 + } + } + else { + + try { + $WorkspaceIPDetails = Get-AzOperationalInsightsIntelligencePacks -ResourceGroupName $workspaceResourceGroupName -WorkspaceName $workspaceName -ErrorAction Stop -WarningAction silentlyContinue + Write-Host("Successfully fetched workspace IP details...") -ForegroundColor Green + Write-Host("") + } + catch { + Write-Host("") + Write-Host("Failed to get the list of solutions onboarded to the workspace. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red + Write-Host("") + Stop-Transcript + exit 1 + } + + try { + $ContainerInsightsIndex = $WorkspaceIPDetails.Name.IndexOf("ContainerInsights") + Write-Host("Successfully located ContainerInsights solution") -ForegroundColor Green + Write-Host("") + } + catch { + Write-Host("Failed to get ContainerInsights solution details from the workspace") -ForegroundColor Red + Write-Host("") + Stop-Transcript + exit 1 + } + + + $isSolutionOnboarded = $WorkspaceIPDetails.Enabled[$ContainerInsightsIndex] + if ($isSolutionOnboarded) { + if ($WorkspacePricingTier -eq "Free") { + Write-Host("Pricing tier of the configured LogAnalytics workspace is Free so you may need to upgrade to pricing tier to non-Free") -ForegroundColor Yellow } } else { - Write-Host("The container health solution isn't onboarded to your cluster. This required for the monitoring to work. Please contact us by emailing askcoin@microsoft.com if you need any help on this") -ForegroundColor Red + # + # Check contributor access to WS + # + $message = "Detected that there is a workspace associated with this cluster, but workspace - '" + $workspaceName + "' in subscription '" + $workspaceSubscriptionId + "' IS NOT ONBOARDED with container health solution." + Write-Host($message) + $question = " Do you want to onboard container health to the workspace?" + + $choices = New-Object Collections.ObjectModel.Collection[Management.Automation.Host.ChoiceDescription] + $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Yes')) + $choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&No')) + + $decision = $Host.UI.PromptForChoice($message, $question, $choices, 0) + + if ($decision -eq 0) { + Write-Host("Deploying template to onboard container health : Please wait...") + + $DeploymentName = "ContainerInsightsSolutionOnboarding-" + ((Get-Date).ToUniversalTime()).ToString('MMdd-HHmm') + $Parameters = @{ } + $Parameters.Add("workspaceResourceId", $LogAnalyticsWorkspaceResourceID) + $Parameters.Add("workspaceRegion", $WorkspaceLocation) + $Parameters + try { + New-AzResourceGroupDeployment -Name $DeploymentName ` + -ResourceGroupName $workspaceResourceGroupName ` + -TemplateUri https://raw.githubusercontent.com/microsoft/Docker-Provider/ci_prod/scripts/onboarding/templates/azuremonitor-containerSolution.json ` + -TemplateParameterObject $Parameters -ErrorAction Stop` + + Write-Host("") + Write-Host("Successfully added Container Insights Solution") -ForegroundColor Green + + Write-Host("") + } + catch { + Write-Host ("Template deployment failed with an error: '" + $Error[0] + "' ") -ForegroundColor Red + Write-Host("Please contact us by emailing askcoin@microsoft.com for help") -ForegroundColor Red + } + } + else { + Write-Host("The container health solution isn't onboarded to your cluster. This required for the monitoring to work. Please contact us by emailing askcoin@microsoft.com if you need any help on this") -ForegroundColor Red + } } } } @@ -737,17 +836,11 @@ if ("AKS" -eq $ClusterType ) { Write-Host("Getting Kubeconfig of the cluster...") Import-AzAksCredential -Id $ClusterResourceId -Force -ErrorAction Stop - Write-Host("Successful got the Kubeconfig of the cluster.") - - Write-Host("Get current context of the k8s cluster") - $clusterContext = kubectl config current-context - Write-Host $clusterContext + Write-Host("Successfully got the Kubeconfig of the cluster.") - if ($clusterContext -ne $ClusterName) { - Write-Host("Switch to cluster context to:", $ClusterName ) - kubectl config use-context $ClusterName - Write-Host("Successfully switche current context of the k8s cluster to:", $ClusterName) - } + Write-Host("Switch to cluster context to:", $ClusterName ) + kubectl config use-context $ClusterName + Write-Host("Successfully switched current context of the k8s cluster to:", $ClusterName) Write-Host("Check whether the omsagent replicaset pod running correctly ...") $rsPod = kubectl get deployments omsagent-rs -n kube-system -o json | ConvertFrom-Json @@ -875,11 +968,16 @@ if ("AKS" -eq $ClusterType ) { Write-Host("Checking agent version...") try { - Write-Host("KubeConfig: " + $KubeConfig) $omsagentInfo = kubectl get pods -n kube-system -o json -l rsName=omsagent-rs | ConvertFrom-Json - $omsagentImage = $omsagentInfo.items.spec.containers.image.split(":")[1] - + for ($index = 0; $index -le $omsagentInfo.items.spec.containers.Length; $index++) + { + $containerName = $omsagentInfo.items.spec.containers[$index].Name + if ($containerName -eq "omsagent") { + $omsagentImage = $omsagentInfo.items.spec.containers[$index].image.split(":")[1] + break + } + } Write-Host('The version of the omsagent running on your cluster is ' + $omsagentImage) Write-Host('You can encounter problems with your cluster if your omsagent version isnt on the latest version. Please go to https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-manage-agent and validate that you have the latest omsagent version running.') -ForegroundColor Yellow } catch { diff --git a/source/plugins/go/src/go.mod b/source/plugins/go/src/go.mod index 9f30afab1..1960f82b8 100644 --- a/source/plugins/go/src/go.mod +++ b/source/plugins/go/src/go.mod @@ -13,6 +13,7 @@ require ( github.com/tinylib/msgp v1.1.2 github.com/ugorji/go v1.1.2-0.20180813092308-00b869d2f4a5 gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3 + k8s.io/api v0.21.0 k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.21.0 ) diff --git a/source/plugins/go/src/ingestion_token_utils.go b/source/plugins/go/src/ingestion_token_utils.go index 896930005..81039f966 100644 --- a/source/plugins/go/src/ingestion_token_utils.go +++ b/source/plugins/go/src/ingestion_token_utils.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -12,6 +13,9 @@ import ( "strconv" "strings" "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const IMDSTokenPathForWindows = "c:/etc/imds-access-token/token" // only used in windows @@ -29,6 +33,12 @@ var IngestionAuthToken string var IngestionAuthTokenExpiration int64 var AMCSRedirectedEndpoint string = "" +// Arc k8s MSI related +const ArcK8sClusterConfigCRDAPIVersion = "clusterconfig.azure.com/v1beta1" +const ArcK8sClusterIdentityResourceName = "container-insights-clusteridentityrequest" +const ArcK8sClusterIdentityResourceNameSpace = "azure-arc" +const ArcK8sMSITokenSecretNameSpace = "azure-arc" + type IMDSResponse struct { AccessToken string `json:"access_token"` ClientID string `json:"client_id"` @@ -66,8 +76,8 @@ type AgentConfiguration struct { } `json:"channels"` Extensionconfigurations struct { Containerinsights []struct { - ID string `json:"id"` - Originids []string `json:"originIds"` + ID string `json:"id"` + Originids []string `json:"originIds"` Outputstreams struct { LinuxPerfBlob string `json:"LINUX_PERF_BLOB"` ContainerInventoryBlob string `json:"CONTAINER_INVENTORY_BLOB"` @@ -75,7 +85,6 @@ type AgentConfiguration struct { ContainerinsightsContainerlogv2 string `json:"CONTAINERINSIGHTS_CONTAINERLOGV2"` ContainerNodeInventoryBlob string `json:"CONTAINER_NODE_INVENTORY_BLOB"` KubeEventsBlob string `json:"KUBE_EVENTS_BLOB"` - KubeHealthBlob string `json:"KUBE_HEALTH_BLOB"` KubeMonAgentEventsBlob string `json:"KUBE_MON_AGENT_EVENTS_BLOB"` KubeNodeInventoryBlob string `json:"KUBE_NODE_INVENTORY_BLOB"` KubePodInventoryBlob string `json:"KUBE_POD_INVENTORY_BLOB"` @@ -94,14 +103,47 @@ type IngestionTokenResponse struct { Ingestionauthtoken string `json:"ingestionAuthToken"` } +type ContainerInsightsIdentityRequest struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata struct { + Annotations struct { + MetaHelmShReleaseName string `json:"meta.helm.sh/release-name"` + MetaHelmShReleaseNamespace string `json:"meta.helm.sh/release-namespace"` + } `json:"annotations"` + CreationTimestamp time.Time `json:"creationTimestamp"` + Generation int `json:"generation"` + Labels struct { + AppKubernetesIoManagedBy string `json:"app.kubernetes.io/managed-by"` + } `json:"labels"` + Name string `json:"name"` + Namespace string `json:"namespace"` + ResourceVersion string `json:"resourceVersion"` + SelfLink string `json:"selfLink"` + UID string `json:"uid"` + } `json:"metadata"` + Spec struct { + Audience string `json:"audience"` + ResourceID string `json:"resourceId"` + } `json:"spec"` + Status struct { + ExpirationTime time.Time `json:"expirationTime"` + TokenReference struct { + DataName string `json:"dataName"` + SecretName string `json:"secretName"` + } `json:"tokenReference"` + } `json:"status"` +} + func getAccessTokenFromIMDS() (string, int64, error) { Log("Info getAccessTokenFromIMDS: start") useIMDSTokenProxyEndPoint := os.Getenv("USE_IMDS_TOKEN_PROXY_END_POINT") imdsAccessToken := "" + var expiration int64 var responseBytes []byte var err error - if (useIMDSTokenProxyEndPoint != "" && strings.Compare(strings.ToLower(useIMDSTokenProxyEndPoint), "true") == 0) { + if useIMDSTokenProxyEndPoint != "" && strings.Compare(strings.ToLower(useIMDSTokenProxyEndPoint), "true") == 0 { Log("Info Reading IMDS Access Token from IMDS Token proxy endpoint") mcsEndpoint := os.Getenv("MCS_ENDPOINT") msi_endpoint_string := fmt.Sprintf("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://%s/", mcsEndpoint) @@ -109,12 +151,12 @@ func getAccessTokenFromIMDS() (string, int64, error) { msi_endpoint, err := url.Parse(msi_endpoint_string) if err != nil { Log("getAccessTokenFromIMDS: Error creating IMDS endpoint URL: %s", err.Error()) - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } req, err := http.NewRequest("GET", msi_endpoint.String(), nil) if err != nil { Log("getAccessTokenFromIMDS: Error creating HTTP request: %s", err.Error()) - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } req.Header.Add("Metadata", "true") @@ -134,14 +176,14 @@ func getAccessTokenFromIMDS() (string, int64, error) { } if resp != nil && resp.Body != nil { - defer resp.Body.Close() + defer resp.Body.Close() } Log("getAccessTokenFromIMDS: IMDS Response Status: %d, retryCount: %d", resp.StatusCode, retryCount) - if IsRetriableError(resp.StatusCode) { + if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getAccessTokenFromIMDS: IMDS Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond + retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -156,44 +198,103 @@ func getAccessTokenFromIMDS() (string, int64, error) { message := fmt.Sprintf("getAccessTokenFromIMDS: IMDS Request failed with nonretryable error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) SendException(message) - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } IsSuccess = true break // call succeeded, don't retry any more } if !IsSuccess || resp == nil || resp.Body == nil { Log("getAccessTokenFromIMDS: IMDS Request ran out of retries") - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } // Pull out response body responseBytes, err = ioutil.ReadAll(resp.Body) if err != nil { Log("getAccessTokenFromIMDS: Error reading response body: %s", err.Error()) - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } } else { - Log("Info Reading IMDS Access Token from file : %s", IMDSTokenPathForWindows) - if _, err = os.Stat(IMDSTokenPathForWindows); os.IsNotExist(err) { - Log("getAccessTokenFromIMDS: IMDS token file doesnt exist: %s", err.Error()) - return imdsAccessToken, 0, err - } - //adding retries incase if we ended up reading the token file while the token file being written - for retryCount := 0; retryCount < MaxRetries; retryCount++ { - responseBytes, err = ioutil.ReadFile(IMDSTokenPathForWindows) - if err != nil { - Log("getAccessTokenFromIMDS: Could not read IMDS token from file: %s, retryCount: %d", err.Error(), retryCount) - time.Sleep(time.Duration((retryCount + 1) * 100) * time.Millisecond) - continue + resourceId := os.Getenv("AKS_RESOURCE_ID") + if resourceId != "" && strings.Contains(strings.ToLower(resourceId), strings.ToLower("Microsoft.ContainerService/managedClusters")) { + Log("Info Reading IMDS Access Token from file : %s", IMDSTokenPathForWindows) + if _, err = os.Stat(IMDSTokenPathForWindows); os.IsNotExist(err) { + Log("getAccessTokenFromIMDS: IMDS token file doesnt exist: %s", err.Error()) + return imdsAccessToken, expiration, err + } + //adding retries incase if we ended up reading the token file while the token file being written + for retryCount := 0; retryCount < MaxRetries; retryCount++ { + responseBytes, err = ioutil.ReadFile(IMDSTokenPathForWindows) + if err != nil { + Log("getAccessTokenFromIMDS: Could not read IMDS token from file: %s, retryCount: %d", err.Error(), retryCount) + time.Sleep(time.Duration((retryCount+1)*100) * time.Millisecond) + continue + } + break + } + } else { + Log("getAccessTokenFromIMDS: Info Getting MSI Access Token reference from CRD and token from secret for Azure Arc K8s cluster") + var crdResponseBytes []byte + var errorMessage string + for retryCount := 0; retryCount < MaxRetries; retryCount++ { + crd_request_endpoint := fmt.Sprintf("/apis/%s/namespaces/%s/azureclusteridentityrequests/%s", ArcK8sClusterConfigCRDAPIVersion, ArcK8sClusterIdentityResourceNameSpace, ArcK8sClusterIdentityResourceName) + crdResponseBytes, err = ClientSet.RESTClient().Get().AbsPath(crd_request_endpoint).DoRaw(context.TODO()) + if err != nil { + Log("getAccessTokenFromIMDS: Failed to get the CRD: %s in namespace: %s, retryCount: %d", ArcK8sClusterIdentityResourceName, ArcK8sClusterIdentityResourceNameSpace, err.Error(), retryCount) + time.Sleep(time.Duration((retryCount+1)*100) * time.Millisecond) + continue + } + break } - break - } - } + if crdResponseBytes != nil { + var ciCRDRequest ContainerInsightsIdentityRequest + err = json.Unmarshal(crdResponseBytes, &ciCRDRequest) + if err != nil { + errorMessage = fmt.Sprintf("getAccessTokenFromIMDS: Error unmarshalling the crdResponseBytes: %s", err.Error()) + Log(errorMessage) + return imdsAccessToken, expiration, errors.New(errorMessage) + } else { + status := ciCRDRequest.Status + tokenReference := status.TokenReference + dataFieldName := tokenReference.DataName + secretName := tokenReference.SecretName + expirationTime := status.ExpirationTime + if dataFieldName == "" || secretName == "" || expirationTime.IsZero() { + errorMessage = "getAccessTokenFromIMDS: Either dataName or SecretName or ExpirationTime values empty which indicates token not refreshed" + Log(errorMessage) + Log("getAccessTokenFromIMDS: dataName: %s, secretName: %s, expirationTime: %s", dataFieldName, secretName, expirationTime) + return imdsAccessToken, expiration, errors.New(errorMessage) + } else { + var secret *v1.Secret + for retryCount := 0; retryCount < MaxRetries; retryCount++ { + secret, err = ClientSet.CoreV1().Secrets(ArcK8sMSITokenSecretNameSpace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + Log("getAccessTokenFromIMDS: Failed to read the secret: %s in namespace: %s, error: %s, retryCount: %d", secretName, ArcK8sMSITokenSecretNameSpace, err.Error(), retryCount) + time.Sleep(time.Duration((retryCount+1)*100) * time.Millisecond) + continue + } + break + } + if secret == nil { + errorMessage = fmt.Sprintf("getAccessTokenFromIMDS: value of secret: %s in nil in namespace: %s", secretName, ArcK8sMSITokenSecretNameSpace) + return imdsAccessToken, expiration, errors.New(errorMessage) + } + imdsAccessToken = string(secret.Data[dataFieldName]) + expiration = expirationTime.Unix() + return imdsAccessToken, expiration, nil + } + } + } else { + errorMessage = fmt.Sprintf("getAccessTokenFromIMDS: faled to get the CRD: %s in namespace: %s", ArcK8sClusterIdentityResourceName, ArcK8sClusterIdentityResourceNameSpace) + return imdsAccessToken, expiration, errors.New(errorMessage) + } + } + } - if responseBytes == nil { + if responseBytes == nil { Log("getAccessTokenFromIMDS: Error responseBytes is nil") - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } // Unmarshall response body into struct @@ -201,14 +302,14 @@ func getAccessTokenFromIMDS() (string, int64, error) { err = json.Unmarshal(responseBytes, &imdsResponse) if err != nil { Log("getAccessTokenFromIMDS: Error unmarshalling the response: %s", err.Error()) - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } imdsAccessToken = imdsResponse.AccessToken - expiration, err := strconv.ParseInt(imdsResponse.ExpiresOn, 10, 64) + expiration, err = strconv.ParseInt(imdsResponse.ExpiresOn, 10, 64) if err != nil { Log("getAccessTokenFromIMDS: Error parsing ExpiresOn field from IMDS response: %s", err.Error()) - return imdsAccessToken, 0, err + return imdsAccessToken, expiration, err } Log("Info getAccessTokenFromIMDS: end") return imdsAccessToken, expiration, nil @@ -259,7 +360,7 @@ func getAgentConfiguration(imdsAccessToken string) (configurationId string, chan } if resp != nil && resp.Body != nil { defer resp.Body.Close() - } + } Log("getAgentConfiguration Response Status: %d", resp.StatusCode) if resp.StatusCode == 421 { // AMCS returns redirected endpoint incase of private link agentConfigEndpoint := resp.Header.Get("x-ms-agent-config-endpoint") @@ -283,7 +384,7 @@ func getAgentConfiguration(imdsAccessToken string) (configurationId string, chan if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getAgentConfiguration: Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond + retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -383,7 +484,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann req.Header.Add("Authorization", bearer) var resp *http.Response = nil - IsSuccess := false + IsSuccess := false for retryCount := 0; retryCount < MaxRetries; retryCount++ { // Call managed services for Azure resources token endpoint resp, err = HTTPClient.Do(req) @@ -397,7 +498,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann if resp != nil && resp.Body != nil { defer resp.Body.Close() - } + } Log("getIngestionAuthToken Response Status: %d", resp.StatusCode) if resp.StatusCode == 421 { // AMCS returns redirected endpoint incase of private link @@ -422,7 +523,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getIngestionAuthToken: Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond + retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -430,7 +531,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann retryDelay = time.Duration(after) * time.Second } } - } + } time.Sleep(retryDelay) continue } else if resp.StatusCode != 200 { @@ -505,7 +606,7 @@ func getTokenRefreshIntervalFromAmcsResponse(header http.Header) (refreshInterva func refreshIngestionAuthToken() { for ; true; <-IngestionAuthTokenRefreshTicker.C { - if IMDSToken == "" || IMDSTokenExpiration <= (time.Now().Unix() + 60 * 60) { // token valid 24 hrs and refresh token 1 hr before expiry + if IMDSToken == "" || IMDSTokenExpiration <= (time.Now().Unix()+60*60) { // token valid 24 hrs and refresh token 1 hr before expiry imdsToken, imdsTokenExpiry, err := getAccessTokenFromIMDS() if err != nil { message := fmt.Sprintf("refreshIngestionAuthToken: Error on getAccessTokenFromIMDS %s \n", err.Error()) @@ -533,7 +634,7 @@ func refreshIngestionAuthToken() { continue } } - if IMDSToken == "" || ConfigurationId == "" || ChannelId == "" { + if IMDSToken == "" || ConfigurationId == "" || ChannelId == "" { message := "refreshIngestionAuthToken: IMDSToken or ConfigurationId or ChannelId empty" Log(message) SendException(message) @@ -561,9 +662,9 @@ func refreshIngestionAuthToken() { func IsRetriableError(httpStatusCode int) bool { retryableStatusCodes := [5]int{408, 429, 502, 503, 504} for _, code := range retryableStatusCodes { - if code == httpStatusCode { - return true - } + if code == httpStatusCode { + return true + } } return false } diff --git a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb index 017bfb08d..49eab4558 100644 --- a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb +++ b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb @@ -34,7 +34,6 @@ class CAdvisorMetricsAPIClient @cAdvisorMetricsSecurePort = ENV["IS_SECURE_CADVISOR_PORT"] @containerLogsRoute = ENV["AZMON_CONTAINER_LOGS_ROUTE"] - @hmEnabled = ENV["AZMON_CLUSTER_ENABLE_HEALTH_MODEL"] @npmIntegrationBasic = ENV["TELEMETRY_NPM_INTEGRATION_METRICS_BASIC"] @npmIntegrationAdvanced = ENV["TELEMETRY_NPM_INTEGRATION_METRICS_ADVANCED"] @@ -277,10 +276,15 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met end #telemetry about containerlog Routing for daemonset telemetryProps["containerLogsRoute"] = @containerLogsRoute - - #telemetry about health model - if (!@hmEnabled.nil? && !@hmEnabled.empty?) - telemetryProps["hmEnabled"] = @hmEnabled + #telemetry for npm integration + if (!@npmIntegrationAdvanced.nil? && !@npmIntegrationAdvanced.empty?) + telemetryProps["int-npm-a"] = "1" + elsif (!@npmIntegrationBasic.nil? && !@npmIntegrationBasic.empty?) + telemetryProps["int-npm-b"] = "1" + end + #telemetry for Container log schema version clusterContainerLogSchemaVersion + if (!@clusterContainerLogSchemaVersion.nil? && !@clusterContainerLogSchemaVersion.empty?) + telemetryProps["containerLogVer"] = @clusterContainerLogSchemaVersion end #telemetry for npm integration if (!@npmIntegrationAdvanced.nil? && !@npmIntegrationAdvanced.empty?) diff --git a/source/plugins/ruby/MdmMetricsGenerator.rb b/source/plugins/ruby/MdmMetricsGenerator.rb index 0858990da..f4904697c 100644 --- a/source/plugins/ruby/MdmMetricsGenerator.rb +++ b/source/plugins/ruby/MdmMetricsGenerator.rb @@ -79,7 +79,7 @@ def populatePodReadyPercentageHash @pod_ready_hash.each { |dim_key, value| podsNotReady = @pod_not_ready_hash.key?(dim_key) ? @pod_not_ready_hash[dim_key] : 0 totalPods = value + podsNotReady - podsReadyPercentage = (value / totalPods) * 100 + podsReadyPercentage = value * 100.0 / totalPods @pod_ready_percentage_hash[dim_key] = podsReadyPercentage # Deleting this key value pair from not ready hash, # so that we can get those dimensions for which there are 100% of the pods in not ready state diff --git a/source/plugins/ruby/constants.rb b/source/plugins/ruby/constants.rb index b9516c2ce..542f342a6 100644 --- a/source/plugins/ruby/constants.rb +++ b/source/plugins/ruby/constants.rb @@ -117,7 +117,6 @@ class Constants KUBE_PV_INVENTORY_DATA_TYPE = "KUBE_PV_INVENTORY_BLOB" KUBE_EVENTS_DATA_TYPE = "KUBE_EVENTS_BLOB" KUBE_MON_AGENT_EVENTS_DATA_TYPE = "KUBE_MON_AGENT_EVENTS_BLOB" - KUBE_HEALTH_DATA_TYPE = "KUBE_HEALTH_BLOB" CONTAINERLOGV2_DATA_TYPE = "CONTAINERINSIGHTS_CONTAINERLOGV2" CONTAINERLOG_DATA_TYPE = "CONTAINER_LOG_BLOB" @@ -136,4 +135,7 @@ class Constants #This is for telemetry to track if any of the windows customer has any of the field size >= 64KB #To evaluate switching to Windows AMA 64KB impacts any existing customers MAX_RECORD_OR_FIELD_SIZE_FOR_TELEMETRY = 65536 + + # only used in windows in AAD MSI auth mode + IMDS_TOKEN_PATH_FOR_WINDOWS = "c:/etc/imds-access-token/token" end diff --git a/source/plugins/ruby/filter_cadvisor_health_container.rb b/source/plugins/ruby/filter_cadvisor_health_container.rb deleted file mode 100644 index ab64b6e61..000000000 --- a/source/plugins/ruby/filter_cadvisor_health_container.rb +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/local/bin/ruby -# frozen_string_literal: true - -require 'fluent/plugin/filter' - -module Fluent::Plugin - require 'logger' - require 'yajl/json_gem' - require_relative 'oms_common' - require_relative "ApplicationInsightsUtility" - Dir[File.join(__dir__, './health', '*.rb')].each { |file| require file } - - - class CAdvisor2ContainerHealthFilter < Filter - include HealthModel - Fluent::Plugin.register_filter('cadvisor_health_container', self) - - config_param :log_path, :string, :default => '/var/opt/microsoft/docker-cimprov/log/health_monitors.log' - config_param :metrics_to_collect, :string, :default => 'cpuUsageNanoCores,memoryRssBytes' - config_param :container_resource_refresh_interval_minutes, :integer, :default => 5 - - @@object_name_k8s_container = 'K8SContainer' - @@counter_name_cpu = 'cpuusagenanocores' - @@counter_name_memory_rss = 'memoryrssbytes' - @@cluster_health_model_enabled = HealthMonitorUtils.is_cluster_health_model_enabled - - def initialize - begin - super - @metrics_to_collect_hash = {} - @formatter = HealthContainerCpuMemoryRecordFormatter.new - rescue => e - @log.info "Error in filter_cadvisor_health_container initialize #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def configure(conf) - begin - super - @log = HealthMonitorUtils.get_log_handle - @log.debug {'Starting filter_cadvisor2health plugin'} - rescue => e - @log.info "Error in filter_cadvisor_health_container configure #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def start - begin - super - @metrics_to_collect_hash = HealthMonitorUtils.build_metrics_hash(@metrics_to_collect) - ApplicationInsightsUtility.sendCustomEvent("filter_cadvisor_health_container Plugin Start", {}) - rescue => e - @log.info "Error in filter_cadvisor_health_container start #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def filter_stream(tag, es) - if !@@cluster_health_model_enabled - @log.info "Cluster Health Model disabled in filter_cadvisor_health_container" - return Fluent::MultiEventStream.new - end - new_es = Fluent::MultiEventStream.new - records_count = 0 - es.each { |time, record| - begin - filtered_record = filter(tag, time, record) - if !filtered_record.nil? - new_es.add(time, filtered_record) - records_count += 1 - end - rescue => e - @log.info "Error in filter_cadvisor_health_container filter_stream #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - } - @log.debug "filter_cadvisor_health_container Records Count #{records_count}" - new_es - end - - def filter(tag, time, record) - begin - if record.key?("MonitorLabels") - return record - end - - object_name = record['ObjectName'] - counter_name = JSON.parse(record['json_Collections'])[0]['CounterName'].downcase - if @metrics_to_collect_hash.key?(counter_name) - if object_name == @@object_name_k8s_container - return @formatter.get_record_from_cadvisor_record(record) - end - end - return nil - rescue => e - @log.debug "Error in filter #{e}" - @log.debug "record #{record}" - @log.debug "backtrace #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - return nil - end - end - end -end diff --git a/source/plugins/ruby/filter_cadvisor_health_node.rb b/source/plugins/ruby/filter_cadvisor_health_node.rb deleted file mode 100644 index ddbb871e8..000000000 --- a/source/plugins/ruby/filter_cadvisor_health_node.rb +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/local/bin/ruby -# frozen_string_literal: true - -require 'fluent/plugin/filter' - -module Fluent::Plugin - require 'logger' - require 'yajl/json_gem' - require_relative 'oms_common' - require_relative "ApplicationInsightsUtility" - require_relative "KubernetesApiClient" - Dir[File.join(__dir__, './health', '*.rb')].each { |file| require file } - - class CAdvisor2NodeHealthFilter < Filter - include HealthModel - Fluent::Plugin.register_filter('cadvisor_health_node', self) - - attr_accessor :provider, :resources - - config_param :metrics_to_collect, :string, :default => 'cpuUsageNanoCores,memoryRssBytes' - config_param :container_resource_refresh_interval_minutes, :integer, :default => 5 - config_param :health_monitor_config_path, :default => '/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json' - - @@object_name_k8s_node = 'K8SNode' - @@object_name_k8s_container = 'K8SContainer' - - @@counter_name_cpu = 'cpuusagenanocores' - @@counter_name_memory_rss = 'memoryrssbytes' - - @@hm_log = HealthMonitorUtils.get_log_handle - @@hostName = (OMS::Common.get_hostname) - @@clusterName = KubernetesApiClient.getClusterName - @@clusterId = KubernetesApiClient.getClusterId - @@clusterRegion = KubernetesApiClient.getClusterRegion - @@cluster_health_model_enabled = HealthMonitorUtils.is_cluster_health_model_enabled - - def initialize - begin - super - @last_resource_refresh = DateTime.now.to_time.to_i - @metrics_to_collect_hash = {} - @resources = HealthKubernetesResources.instance # this doesnt require node and pod inventory. So no need to populate them - @provider = HealthMonitorProvider.new(@@clusterId, HealthMonitorUtils.get_cluster_labels, @resources, @health_monitor_config_path) - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def configure(conf) - begin - super - @log = HealthMonitorUtils.get_log_handle - @log.debug {'Starting filter_cadvisor2health plugin'} - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def start - begin - super - @cpu_capacity = 1.0 #avoid divide by zero error in case of network issues accessing kube-api - @memory_capacity = 1.0 - @metrics_to_collect_hash = HealthMonitorUtils.build_metrics_hash(@metrics_to_collect) - @log.debug "Calling ensure_cpu_memory_capacity_set cpu_capacity #{@cpu_capacity} memory_capacity #{@memory_capacity}" - node_capacity = HealthMonitorUtils.ensure_cpu_memory_capacity_set(@@hm_log, @cpu_capacity, @memory_capacity, @@hostName) - @cpu_capacity = node_capacity[0] - @memory_capacity = node_capacity[1] - @log.info "CPU Capacity #{@cpu_capacity} Memory Capacity #{@memory_capacity}" - #HealthMonitorUtils.refresh_kubernetes_api_data(@log, @@hostName) - ApplicationInsightsUtility.sendCustomEvent("filter_cadvisor_health Plugin Start", {}) - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def filter_stream(tag, es) - if !@@cluster_health_model_enabled - @log.info "Cluster Health Model disabled in filter_cadvisor_health_node" - return Fluent::MultiEventStream.new - end - begin - node_capacity = HealthMonitorUtils.ensure_cpu_memory_capacity_set(@@hm_log, @cpu_capacity, @memory_capacity, @@hostName) - @cpu_capacity = node_capacity[0] - @memory_capacity = node_capacity[1] - new_es = Fluent::MultiEventStream.new - records_count = 0 - es.each { |time, record| - filtered_record = filter(tag, time, record) - if !filtered_record.nil? - new_es.add(time, filtered_record) - records_count += 1 - end - } - @log.debug "Filter Records Count #{records_count}" - return new_es - rescue => e - @log.info "Error in filter_cadvisor_health_node filter_stream #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - return Fluent::MultiEventStream.new - end - end - - def filter(tag, time, record) - begin - if record.key?("MonitorLabels") - return record - end - - object_name = record['ObjectName'] - counter_name = JSON.parse(record['json_Collections'])[0]['CounterName'].downcase - if @metrics_to_collect_hash.key?(counter_name.downcase) - metric_value = JSON.parse(record['json_Collections'])[0]['Value'] - case object_name - when @@object_name_k8s_node - case counter_name.downcase - when @@counter_name_cpu - process_node_cpu_record(record, metric_value) - when @@counter_name_memory_rss - process_node_memory_record(record, metric_value) - end - end - end - rescue => e - @log.debug "Error in filter #{e}" - @log.debug "record #{record}" - @log.debug "backtrace #{e.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(e) - return nil - end - end - - def process_node_cpu_record(record, metric_value) - monitor_id = MonitorId::NODE_CPU_MONITOR_ID - #@log.debug "processing node cpu record" - if record.nil? - return nil - else - instance_name = record['InstanceName'] - #@log.info "CPU capacity #{@cpu_capacity}" - metric_value /= 1000000 - percent = (metric_value.to_f/@cpu_capacity*100).round(2) - #@log.debug "Percentage of CPU limit: #{percent}" - state = HealthMonitorUtils.compute_percentage_state(percent, @provider.get_config(MonitorId::NODE_CPU_MONITOR_ID)) - #@log.debug "Computed State : #{state}" - timestamp = record['Timestamp'] - health_monitor_record = {"timestamp" => timestamp, "state" => state, "details" => {"cpuUsageMillicores" => metric_value, "cpuUtilizationPercentage" => percent}} - - monitor_instance_id = HealthMonitorUtils.get_monitor_instance_id(monitor_id, [@@clusterId, @@hostName]) - # temp = record.nil? ? "Nil" : record["MonitorInstanceId"] - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::NODE_NAME] = @@hostName - @log.info "Processed Node CPU" - return health_record - end - return nil - end - - def process_node_memory_record(record, metric_value) - monitor_id = MonitorId::NODE_MEMORY_MONITOR_ID - #@log.debug "processing node memory record" - if record.nil? - return nil - else - instance_name = record['InstanceName'] - #@log.info "Memory capacity #{@memory_capacity}" - - percent = (metric_value.to_f/@memory_capacity*100).round(2) - #@log.debug "Percentage of Memory limit: #{percent}" - state = HealthMonitorUtils.compute_percentage_state(percent, @provider.get_config(MonitorId::NODE_MEMORY_MONITOR_ID)) - #@log.debug "Computed State : #{state}" - timestamp = record['Timestamp'] - health_monitor_record = {"timestamp" => timestamp, "state" => state, "details" => {"memoryRssBytes" => metric_value.to_f, "memoryUtilizationPercentage" => percent}} - #@log.info health_monitor_record - - monitor_instance_id = HealthMonitorUtils.get_monitor_instance_id(monitor_id, [@@clusterId, @@hostName]) - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::NODE_NAME] = @@hostName - @log.info "Processed Node Memory" - return health_record - end - return nil - end - end -end diff --git a/source/plugins/ruby/filter_health_model_builder.rb b/source/plugins/ruby/filter_health_model_builder.rb deleted file mode 100644 index 9decda881..000000000 --- a/source/plugins/ruby/filter_health_model_builder.rb +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. - -# frozen_string_literal: true - -require 'fluent/plugin/filter' - -module Fluent::Plugin - require_relative 'extension_utils' - require 'logger' - require 'yajl/json_gem' - Dir[File.join(__dir__, './health', '*.rb')].each { |file| require file } - - - class FilterHealthModelBuilder < Filter - include HealthModel - Fluent::Plugin.register_filter('health_model_builder', self) - - config_param :enable_log, :integer, :default => 0 - config_param :log_path, :string, :default => '/var/opt/microsoft/docker-cimprov/log/filter_health_model_builder.log' - config_param :model_definition_path, :default => '/etc/opt/microsoft/docker-cimprov/health/health_model_definition.json' - config_param :health_monitor_config_path, :default => '/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json' - config_param :health_state_serialized_path, :default => '/mnt/azure/health_model_state.json' - attr_reader :buffer, :model_builder, :health_model_definition, :monitor_factory, :state_finalizers, :monitor_set, :model_builder, :hierarchy_builder, :resources, :kube_api_down_handler, :provider, :reducer, :state, :generator, :telemetry - - - - @@cluster_id = KubernetesApiClient.getClusterId - @@token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" - @@cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - @@cluster_health_model_enabled = HealthMonitorUtils.is_cluster_health_model_enabled - - def initialize - begin - super - @rewrite_tag = 'oneagent.containerInsights.KUBE_HEALTH_BLOB' - @buffer = HealthModel::HealthModelBuffer.new - @cluster_health_state = ClusterHealthState.new(@@token_file_path, @@cert_file_path) - @health_model_definition = HealthModel::ParentMonitorProvider.new(HealthModel::HealthModelDefinitionParser.new(@model_definition_path).parse_file) - @monitor_factory = HealthModel::MonitorFactory.new - @hierarchy_builder = HealthHierarchyBuilder.new(@health_model_definition, @monitor_factory) - # TODO: Figure out if we need to add NodeMonitorHierarchyReducer to the list of finalizers. For now, dont compress/optimize, since it becomes impossible to construct the model on the UX side - @state_finalizers = [HealthModel::AggregateMonitorStateFinalizer.new] - @monitor_set = HealthModel::MonitorSet.new - @model_builder = HealthModel::HealthModelBuilder.new(@hierarchy_builder, @state_finalizers, @monitor_set) - @kube_api_down_handler = HealthKubeApiDownHandler.new - @resources = HealthKubernetesResources.instance - @reducer = HealthSignalReducer.new - @generator = HealthMissingSignalGenerator.new - @provider = HealthMonitorProvider.new(@@cluster_id, HealthMonitorUtils.get_cluster_labels, @resources, @health_monitor_config_path) - @cluster_old_state = 'none' - @cluster_new_state = 'none' - @container_cpu_memory_records = [] - @telemetry = HealthMonitorTelemetry.new - @state = HealthMonitorState.new - # move network calls to the end. This will ensure all the instance variables get initialized - if @@cluster_health_model_enabled - deserialized_state_info = @cluster_health_state.get_state - @state.initialize_state(deserialized_state_info) - end - - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def configure(conf) - begin - super - @log = nil - if @enable_log - @log = Logger.new(@log_path, 'weekly') - @log.info 'Starting filter_health_model_builder plugin' - end - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def start - super - end - - def shutdown - super - end - - def filter_stream(tag, es) - if !@@cluster_health_model_enabled - @log.info "Cluster Health Model disabled in filter_health_model_builder" - return Fluent::MultiEventStream.new - end - begin - new_es = Fluent::MultiEventStream.new - time = Time.now - if ExtensionUtils.isAADMSIAuthMode() - $log.info("filter_health_model_builder::enumerate: AAD AUTH MSI MODE") - if @rewrite_tag.nil? || !@rewrite_tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) - @rewrite_tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_HEALTH_DATA_TYPE) - end - $log.info("filter_health_model_builder::filter_stream: using tag -#{@rewrite_tag} @ #{Time.now.utc.iso8601}") - end - - if tag.start_with?("kubehealth.DaemonSet.Node") - node_records = [] - if !es.nil? - es.each{|time, record| - node_records.push(record) - } - @buffer.add_to_buffer(node_records) - end - return Fluent::MultiEventStream.new - elsif tag.start_with?("kubehealth.DaemonSet.Container") - container_records = [] - if !es.nil? - es.each{|time, record| - container_records.push(record) - } - end - container_records_aggregator = HealthContainerCpuMemoryAggregator.new(@resources, @provider) - if @container_cpu_memory_records.nil? - @log.info "@container_cpu_memory_records was not initialized" - @container_cpu_memory_records = [] #in some clusters, this is null, so initialize it again. - end - @container_cpu_memory_records.push(*container_records) # push the records for aggregation later - return Fluent::MultiEventStream.new - elsif tag.start_with?("kubehealth.ReplicaSet") - records = [] - es.each{|time, record| - records.push(record) - } - @buffer.add_to_buffer(records) # in_kube_health records - - aggregated_container_records = [] - if !@container_cpu_memory_records.nil? && !@container_cpu_memory_records.empty? - container_records_aggregator = HealthContainerCpuMemoryAggregator.new(@resources, @provider) - deduped_records = container_records_aggregator.dedupe_records(@container_cpu_memory_records) - container_records_aggregator.aggregate(deduped_records) - container_records_aggregator.compute_state - aggregated_container_records = container_records_aggregator.get_records - end - @buffer.add_to_buffer(aggregated_container_records) #container cpu/memory records - records_to_process = @buffer.get_buffer - @buffer.reset_buffer - @container_cpu_memory_records = [] - - health_monitor_records = [] - records_to_process.each do |record| - monitor_instance_id = record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] - monitor_id = record[HealthMonitorRecordFields::MONITOR_ID] - #HealthMonitorRecord - health_monitor_record = HealthMonitorRecord.new( - record[HealthMonitorRecordFields::MONITOR_ID], - record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], - record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], - record[HealthMonitorRecordFields::DETAILS]["state"], - @provider.get_labels(record), - @provider.get_config(monitor_id), - record[HealthMonitorRecordFields::DETAILS] - ) - health_monitor_records.push(health_monitor_record) - #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - end - - @log.info "health_monitor_records.size #{health_monitor_records.size}" - # Dedupe daemonset signals - # Remove unit monitor signals for “gone” objects - # update state for the reduced set of signals - reduced_records = @reducer.reduce_signals(health_monitor_records, @resources) - reduced_records.each{|record| - @state.update_state(record, - @provider.get_config(record.monitor_id), - false, - @telemetry - ) - # get the health state based on the monitor's operational state - # update state calls updates the state of the monitor based on configuration and history of the the monitor records - record.state = @state.get_state(record.monitor_instance_id).new_state - } - @log.info "after deduping and removing gone objects reduced_records.size #{reduced_records.size}" - - reduced_records = @kube_api_down_handler.handle_kube_api_down(reduced_records) - @log.info "after kube api down handler health_monitor_records.size #{health_monitor_records.size}" - - #get the list of 'none' and 'unknown' signals - missing_signals = @generator.get_missing_signals(@@cluster_id, reduced_records, @resources, @provider) - - @log.info "after getting missing signals missing_signals.size #{missing_signals.size}" - #update state for missing signals - missing_signals.each{|signal| - - @state.update_state(signal, @provider.get_config(signal.monitor_id), false, @telemetry) - @log.info "After Updating #{@state.get_state(signal.monitor_instance_id)} #{@state.get_state(signal.monitor_instance_id).new_state}" - # for unknown/none records, update the "monitor state" to be the latest state (new_state) of the monitor instance from the state - signal.state = @state.get_state(signal.monitor_instance_id).new_state - } - - @generator.update_last_received_records(reduced_records) - all_records = reduced_records.clone - all_records.push(*missing_signals) - - @log.info "after Adding missing signals all_records.size #{all_records.size}" - - HealthMonitorHelpers.add_agentpool_node_label_if_not_present(all_records) - - # build the health model - @model_builder.process_records(all_records) - all_monitors = @model_builder.finalize_model - - @log.info "after building health_model #{all_monitors.size}" - - # update the state for aggregate monitors (unit monitors are updated above) - all_monitors.each{|monitor_instance_id, monitor| - if monitor.is_aggregate_monitor - @state.update_state(monitor, - @provider.get_config(monitor.monitor_id), - true, - @telemetry - ) - end - - instance_state = @state.get_state(monitor_instance_id) - #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - should_send = instance_state.should_send - - # always send cluster monitor as a heartbeat - if !should_send && monitor_instance_id != MonitorId::CLUSTER - all_monitors.delete(monitor_instance_id) - end - } - - @log.info "after optimizing health signals all_monitors.size #{all_monitors.size}" - - - # for each key in monitor.keys, - # get the state from health_monitor_state - # generate the record to send - emit_time = Fluent::Engine.now - all_monitors.keys.each{|key| - record = @provider.get_record(all_monitors[key], state) - if record[HealthMonitorRecordFields::MONITOR_ID] == MonitorId::CLUSTER - if !record[HealthMonitorRecordFields::DETAILS].nil? - details = JSON.parse(record[HealthMonitorRecordFields::DETAILS]) - details[HealthMonitorRecordFields::HEALTH_MODEL_DEFINITION_VERSION] = "#{ENV['HEALTH_MODEL_DEFINITION_VERSION']}" - record[HealthMonitorRecordFields::DETAILS] = details.to_json - end - if all_monitors.size > 1 - old_state = record[HealthMonitorRecordFields::OLD_STATE] - new_state = record[HealthMonitorRecordFields::NEW_STATE] - if old_state != new_state && @cluster_old_state != old_state && @cluster_new_state != new_state - ApplicationInsightsUtility.sendCustomEvent("HealthModel_ClusterStateChanged",{"old_state" => old_state , "new_state" => new_state, "monitor_count" => all_monitors.size}) - @log.info "sent telemetry for cluster state change from #{record['OldState']} to #{record['NewState']}" - @cluster_old_state = old_state - @cluster_new_state = new_state - end - end - end - new_es.add(emit_time, record) - } - - #emit the stream - router.emit_stream(@rewrite_tag, new_es) - - #initialize monitor_set and model_builder - @monitor_set = HealthModel::MonitorSet.new - @model_builder = HealthModel::HealthModelBuilder.new(@hierarchy_builder, @state_finalizers, @monitor_set) - - #update cluster state custom resource - @cluster_health_state.update_state(@state.to_h) - @telemetry.send - # return an empty event stream, else the match will throw a NoMethodError - return Fluent::MultiEventStream.new - elsif tag.start_with?(@rewrite_tag) - # this filter also acts as a pass through as we are rewriting the tag and emitting to the fluent stream - es - else - raise "Invalid tag #{tag} received" - end - - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - @log.warn "Message: #{e.message} Backtrace: #{e.backtrace}" - return nil - end - end - end -end diff --git a/source/plugins/ruby/health/agg_monitor_id_labels.rb b/source/plugins/ruby/health/agg_monitor_id_labels.rb deleted file mode 100644 index 03680d054..000000000 --- a/source/plugins/ruby/health/agg_monitor_id_labels.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' - -module HealthModel - class AggregateMonitorInstanceIdLabels - @@id_labels_mapping = { - MonitorId::SYSTEM_WORKLOAD => [HealthMonitorLabels::NAMESPACE, HealthMonitorLabels::WORKLOAD_NAME], - MonitorId::USER_WORKLOAD => [HealthMonitorLabels::NAMESPACE, HealthMonitorLabels::WORKLOAD_NAME], - MonitorId::NODE => [HealthMonitorLabels::AGENTPOOL, HealthMonitorLabels::ROLE, HealthMonitorLabels::HOSTNAME], - MonitorId::NAMESPACE => [HealthMonitorLabels::NAMESPACE], - MonitorId::AGENT_NODE_POOL => [HealthMonitorLabels::AGENTPOOL], - MonitorId::CONTAINER => [HealthMonitorLabels::NAMESPACE, HealthMonitorLabels::WORKLOAD_NAME, HealthMonitorLabels::CONTAINER], - MonitorId::CONTAINER_CPU_MONITOR_ID => [HealthMonitorLabels::NAMESPACE, HealthMonitorLabels::WORKLOAD_NAME], - MonitorId::CONTAINER_MEMORY_MONITOR_ID => [HealthMonitorLabels::NAMESPACE, HealthMonitorLabels::WORKLOAD_NAME], - } - - def self.get_labels_for(monitor_id) - if @@id_labels_mapping.key?(monitor_id) - return @@id_labels_mapping[monitor_id] - else - return [] - end - - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/aggregate_monitor.rb b/source/plugins/ruby/health/aggregate_monitor.rb deleted file mode 100644 index a774478e7..000000000 --- a/source/plugins/ruby/health/aggregate_monitor.rb +++ /dev/null @@ -1,227 +0,0 @@ -# frozen_string_literal: true - -require_relative 'health_model_constants' -require 'yajl/json_gem' - -# Require only when running inside container. -# otherwise unit tests will fail due to ApplicationInsightsUtility dependency on base omsagent ruby files. If you have your dev machine starting with omsagent-rs, then GOOD LUCK! -if Socket.gethostname.start_with?('omsagent-rs') - require_relative '../ApplicationInsightsUtility' -end - -module HealthModel - class AggregateMonitor - attr_accessor :monitor_id, :monitor_instance_id, :state, :transition_date_time, :aggregation_algorithm, :aggregation_algorithm_params, :labels, :is_aggregate_monitor, :details - attr_reader :member_monitors, :member_state_counts - - @@sort_key_order = { - MonitorState::UNKNOWN => 1, - MonitorState::CRITICAL => 2, - MonitorState::WARNING => 3, - MonitorState::HEALTHY => 4, - MonitorState::NONE => 5 - } - - @@telemetry_sent_hash = {} - - # constructor - def initialize( - monitor_id, - monitor_instance_id, - state, - transition_date_time, - aggregation_algorithm, - aggregation_algorithm_params, - labels - ) - @monitor_id = monitor_id - @monitor_instance_id = monitor_instance_id - @state = state - @transition_date_time = transition_date_time - @aggregation_algorithm = aggregation_algorithm || AggregationAlgorithm::WORSTOF - @aggregation_algorithm_params = aggregation_algorithm_params - @labels = labels - @member_monitors = {} - @member_state_counts = {} - @is_aggregate_monitor = true - end - - # adds a member monitor as a child - def add_member_monitor(member_monitor_instance_id) - unless @member_monitors.key?(member_monitor_instance_id) - @member_monitors[member_monitor_instance_id] = true - end - end - - #removes a member monitor - def remove_member_monitor(member_monitor_instance_id) - if @member_monitors.key?(member_monitor_instance_id) - @member_monitors.delete(member_monitor_instance_id) - end - end - - # return the member monitors as an array - def get_member_monitors - @member_monitors.map(&:first) - end - - # calculates the state of the aggregate monitor based on aggregation algorithm and child monitor states - def calculate_state(monitor_set) - case @aggregation_algorithm - when AggregationAlgorithm::WORSTOF - @state = calculate_worst_of_state(monitor_set) - when AggregationAlgorithm::PERCENTAGE - @state = calculate_percentage_state(monitor_set) - else - raise 'No aggregation algorithm specified' - end - end - - def calculate_details(monitor_set) - @details = {} - @details['details'] = {} - @details['state'] = state - @details['timestamp'] = transition_date_time - ids = [] - member_monitor_instance_ids = get_member_monitors - member_monitor_instance_ids.each{|member_monitor_id| - member_monitor = monitor_set.get_monitor(member_monitor_id) - member_state = member_monitor.state - if @details['details'].key?(member_state) - ids = @details['details'][member_state] - if !ids.include?(member_monitor.monitor_instance_id) - ids.push(member_monitor.monitor_instance_id) - end - @details['details'][member_state] = ids - else - @details['details'][member_state] = [member_monitor.monitor_instance_id] - end - } - end - - # calculates the worst of state, given the member monitors - def calculate_worst_of_state(monitor_set) - - @member_state_counts = map_member_monitor_states(monitor_set) - - if member_state_counts.length === 0 - return MonitorState::NONE - end - - if member_state_counts.key?(MonitorState::CRITICAL) && member_state_counts[MonitorState::CRITICAL] > 0 - return MonitorState::CRITICAL - end - if member_state_counts.key?(MonitorState::ERROR) && member_state_counts[MonitorState::ERROR] > 0 - return MonitorState::ERROR - end - if member_state_counts.key?(MonitorState::WARNING) && member_state_counts[MonitorState::WARNING] > 0 - return MonitorState::WARNING - end - - if member_state_counts.key?(MonitorState::UNKNOWN) && member_state_counts[MonitorState::UNKNOWN] > 0 - return MonitorState::UNKNOWN - end - - if member_state_counts.key?(MonitorState::HEALTHY) && member_state_counts[MonitorState::HEALTHY] > 0 - return MonitorState::HEALTHY #healthy should win over none in aggregation - end - - return MonitorState::NONE - - end - - # calculates a percentage state, given the aggregation algorithm parameters - def calculate_percentage_state(monitor_set) - - #sort - #TODO: What if sorted_filtered is empty? is that even possible? - log = HealthMonitorHelpers.get_log_handle - sorted_filtered = sort_filter_member_monitors(monitor_set) - - state_threshold = @aggregation_algorithm_params['state_threshold'].to_f - - if sorted_filtered.nil? - size = 0 - else - size = sorted_filtered.size - end - - if size == 1 - @state = sorted_filtered[0].state - else - count = ((state_threshold*size)/100).ceil - index = size - count - if sorted_filtered.nil? || sorted_filtered[index].nil? - @state = HealthMonitorStates::UNKNOWN - if !@@telemetry_sent_hash.key?(@monitor_instance_id) - log.debug "Adding to telemetry sent hash #{@monitor_instance_id}" - @@telemetry_sent_hash[@monitor_instance_id] = true - log.info "Index: #{index} size: #{size} Count: #{count}" - custom_error_event_map = {} - custom_error_event_map["count"] = count - custom_error_event_map["index"] = index - custom_error_event_map["size"] = size - if !sorted_filtered.nil? - sorted_filtered.each_index{|i| - custom_error_event_map[i] = sorted_filtered[i].state - } - end - ApplicationInsightsUtility.sendCustomEvent("PercentageStateCalculationErrorEvent", custom_error_event_map) - end - else - @state = sorted_filtered[index].state - end - @state - end - end - - # maps states of member monitors to counts - def map_member_monitor_states(monitor_set) - member_monitor_instance_ids = get_member_monitors - if member_monitor_instance_ids.nil? || member_monitor_instance_ids.size == 0 - return {} - end - - state_counts = {} - - member_monitor_instance_ids.each {|monitor_instance_id| - - member_monitor = monitor_set.get_monitor(monitor_instance_id) - monitor_state = member_monitor.state - - if !state_counts.key?(monitor_state) - state_counts[monitor_state] = 1 - else - count = state_counts[monitor_state] - state_counts[monitor_state] = count+1 - end - } - - return state_counts; - end - - # Sort the member monitors in the following order -=begin - 1. Error - 2. Unknown - 3. Critical - 4. Warning - 5. Healthy - Remove 'none' state monitors -=end - def sort_filter_member_monitors(monitor_set) - member_monitor_instance_ids = get_member_monitors - member_monitors = [] - - member_monitor_instance_ids.each {|monitor_instance_id| - member_monitor = monitor_set.get_monitor(monitor_instance_id) - member_monitors.push(member_monitor) - } - - filtered = member_monitors.keep_if{|monitor| monitor.state != MonitorState::NONE} - sorted = filtered.sort_by{ |monitor| [@@sort_key_order[monitor.state]] } - - return sorted - end - end -end diff --git a/source/plugins/ruby/health/aggregate_monitor_state_finalizer.rb b/source/plugins/ruby/health/aggregate_monitor_state_finalizer.rb deleted file mode 100644 index dd69c9c4d..000000000 --- a/source/plugins/ruby/health/aggregate_monitor_state_finalizer.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module HealthModel - class AggregateMonitorStateFinalizer - - def finalize(monitor_set) - top_level_monitor = monitor_set.get_monitor(MonitorId::CLUSTER) - if !top_level_monitor.nil? - calculate_subtree_state(top_level_monitor, monitor_set) - end - monitor_set.get_map.each{|k,v| - if v.is_aggregate_monitor - v.calculate_details(monitor_set) - end - } - end - - private - def calculate_subtree_state(monitor, monitor_set) - if monitor.nil? || !monitor.is_aggregate_monitor - raise 'AggregateMonitorStateFinalizer:calculateSubtreeState Parameter monitor must be non-null AggregateMonitor' - end - - member_monitor_instance_ids = monitor.get_member_monitors # monitor_instance_ids - member_monitor_instance_ids.each{|member_monitor_instance_id| - member_monitor = monitor_set.get_monitor(member_monitor_instance_id) - - if !member_monitor.nil? && member_monitor.is_aggregate_monitor - calculate_subtree_state(member_monitor, monitor_set) - end - } - monitor.calculate_state(monitor_set) - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/cluster_health_state.rb b/source/plugins/ruby/health/cluster_health_state.rb deleted file mode 100644 index e46d0bf5f..000000000 --- a/source/plugins/ruby/health/cluster_health_state.rb +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -require "net/http" -require "net/https" -require "uri" -require 'yajl/json_gem' - -module HealthModel - class ClusterHealthState - - attr_reader :token_file_path, :cert_file_path, :log, :http_client, :uri, :token - @@resource_uri_template = "%{kube_api_server_url}/apis/azmon.container.insights/v1/namespaces/kube-system/healthstates/cluster-health-state" - - def initialize(token_file_path, cert_file_path) - @token_file_path = token_file_path - @cert_file_path = cert_file_path - @log = HealthMonitorHelpers.get_log_handle - @http_client = get_http_client - @token = get_token - end - - def update_state(state) #state = hash of monitor_instance_id to HealthMonitorInstanceState struct - get_request = Net::HTTP::Get.new(@uri.request_uri) - monitor_states_hash = {} - state.each {|monitor_instance_id, health_monitor_instance_state| - monitor_states_hash[monitor_instance_id] = health_monitor_instance_state.to_h - } - - get_request["Authorization"] = "Bearer #{@token}" - @log.info "Making GET request to #{@uri.request_uri} @ #{Time.now.utc.iso8601}" - get_response = @http_client.request(get_request) - @log.info "Got response of #{get_response.code} for #{@uri.request_uri} @ #{Time.now.utc.iso8601}" - - if get_response.code.to_i == 404 # NOT found - #POST - update_request = Net::HTTP::Post.new(@uri.request_uri) - update_request["Content-Type"] = "application/json" - - elsif get_response.code.to_i == 200 # Update == Patch - #PATCH - update_request = Net::HTTP::Patch.new(@uri.request_uri) - update_request["Content-Type"] = "application/merge-patch+json" - end - update_request["Authorization"] = "Bearer #{@token}" - - update_request_body = get_update_request_body - update_request_body["state"] = monitor_states_hash.to_json - update_request.body = update_request_body.to_json - - update_response = @http_client.request(update_request) - @log.info "Got a response of #{update_response.code} for #{update_request.method}" - end - - def get_state - get_request = Net::HTTP::Get.new(@uri.request_uri) - get_request["Authorization"] = "Bearer #{@token}" - @log.info "Making GET request to #{@uri.request_uri} @ #{Time.now.utc.iso8601}" - get_response = @http_client.request(get_request) - @log.info "Got response of #{get_response.code} for #{@uri.request_uri} @ #{Time.now.utc.iso8601}" - - if get_response.code.to_i == 200 - return JSON.parse(JSON.parse(get_response.body)["state"]) - else - return {} - end - end - - private - def get_token() - begin - if File.exist?(@token_file_path) && File.readable?(@token_file_path) - token_str = File.read(@token_file_path).strip - return token_str - else - @log.info ("Unable to read token string from #{@token_file_path}") - return nil - end - end - end - - def get_http_client() - kube_api_server_url = get_kube_api_server_url - resource_uri = @@resource_uri_template % { - kube_api_server_url: kube_api_server_url - } - @uri = URI.parse(resource_uri) - http = Net::HTTP.new(@uri.host, @uri.port) - http.use_ssl = true - if !File.exist?(@cert_file_path) - raise "#{@cert_file_path} doesnt exist" - else - http.ca_file = @cert_file_path - end - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - return http - end - - def get_kube_api_server_url - if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] - return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" - else - @log.warn ("Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri") - if Gem.win_platform? #unit testing on windows dev machine - value = %x( kubectl -n default get endpoints kubernetes --no-headers) - url = "https://#{value.split(' ')[1]}" - return "https://localhost:8080" # This is NEVER used. this is just to return SOME value - end - return nil - end - end - - def get_update_request_body - body = {} - body["apiVersion"] = "azmon.container.insights/v1" - body["kind"] = "HealthState" - body["metadata"] = {} - body["metadata"]["name"] = "cluster-health-state" - body["metadata"]["namespace"] = "kube-system" - return body - end - end -end diff --git a/source/plugins/ruby/health/health_container_cpu_memory_aggregator.rb b/source/plugins/ruby/health/health_container_cpu_memory_aggregator.rb deleted file mode 100644 index e93c66c14..000000000 --- a/source/plugins/ruby/health/health_container_cpu_memory_aggregator.rb +++ /dev/null @@ -1,386 +0,0 @@ -# frozen_string_literal: true - -require_relative 'health_model_constants' - -# Require only when running inside container. -# otherwise unit tests will fail due to ApplicationInsightsUtility dependency on base omsagent ruby files. If you have your dev machine starting with omsagent-rs, then GOOD LUCK! -if Socket.gethostname.start_with?('omsagent-rs') - require_relative '../ApplicationInsightsUtility' -end -=begin - @cpu_records/@memory_records - [ - { - "namespace_workload_container_name" : { - "limit" : limit, #number - "limit_set" : limit_set, #bool - "record_count" : record_count, #number - "workload_name": workload_name, - "workload_kind": workload_kind, - "namespace" : namespace, - "container": container, - records:[ - { - "counter_value": counter_value, - "pod_name": pod_name, - "container": container, - "state" : state - }, - { - "counter_value": counter_value, - "pod_name": pod_name, - "container": container, - "state" : state - } - ] - } - } - ] -=end -module HealthModel - # this class aggregates the records at the container level - class HealthContainerCpuMemoryAggregator - - attr_reader :pod_uid_lookup, :workload_container_count, :cpu_records, :memory_records, :provider - - @@memory_counter_name = 'memoryRssBytes' - @@cpu_counter_name = 'cpuUsageNanoCores' - @@workload_container_count_empty_event_sent = {} - @@limit_is_array_event_sent = {} - @@WORKLOAD_CONTAINER_COUNT_EMPTY_EVENT = "WorkloadContainerCountEmptyEvent" - @@LIMIT_IS_ARRAY_EVENT = "ResourceLimitIsAnArrayEvent" - @@cpu_last_sent_monitors = {} - @@memory_last_sent_monitors = {} - - def initialize(resources, provider) - @pod_uid_lookup = resources.get_pod_uid_lookup - @workload_container_count = resources.get_workload_container_count - @cpu_records = {} - @memory_records = {} - @log = HealthMonitorHelpers.get_log_handle - @provider = provider - end - - def dedupe_records(container_records) - cpu_deduped_instances = {} - memory_deduped_instances = {} - container_records = container_records.keep_if{|record| record['CounterName'] == @@memory_counter_name || record['CounterName'] == @@cpu_counter_name} - - container_records.each do |record| - begin - instance_name = record["InstanceName"] - counter_name = record["CounterName"] - case counter_name - when @@memory_counter_name - resource_instances = memory_deduped_instances - when @@cpu_counter_name - resource_instances = cpu_deduped_instances - else - @log.info "Unexpected Counter Name #{counter_name}" - next - end - if !resource_instances.key?(instance_name) - resource_instances[instance_name] = record - else - r = resource_instances[instance_name] - if record["Timestamp"] > r["Timestamp"] - @log.info "Dropping older record for instance #{instance_name} new: #{record["Timestamp"]} old: #{r["Timestamp"]}" - resource_instances[instance_name] = record - end - end - rescue => e - @log.info "Exception when deduping record #{record}" - next - end - end - return cpu_deduped_instances.values.concat(memory_deduped_instances.values) - end - - def aggregate(container_records) - #filter and select only cpuUsageNanoCores and memoryRssBytes - container_records = container_records.keep_if{|record| record['CounterName'] == @@memory_counter_name || record['CounterName'] == @@cpu_counter_name} - # poduid lookup has poduid/cname --> workload_name, namespace, cpu_limit, memory limit mapping - # from the container records, extract the poduid/cname, get the values from poduid_lookup, and aggregate based on namespace_workload_cname - container_records.each do |record| - begin - instance_name = record["InstanceName"] - lookup_key = instance_name.split('/').last(2).join('/') - if !@pod_uid_lookup.key?(lookup_key) - next - end - namespace = @pod_uid_lookup[lookup_key]['namespace'] - workload_name = @pod_uid_lookup[lookup_key]['workload_name'] - cname = lookup_key.split('/')[1] - counter_name = record["CounterName"] - case counter_name - when @@memory_counter_name - resource_hash = @memory_records - resource_type = 'memory' - when @@cpu_counter_name - resource_hash = @cpu_records - resource_type = 'cpu' - else - @log.info "Unexpected Counter Name #{counter_name}" - next - end - - # this is used as a look up from the pod_uid_lookup in kubernetes_health_resources object - resource_hash_key = "#{namespace}_#{workload_name.split('~~')[1]}_#{cname}" - - # if the resource map doesnt contain the key, add limit, count and records - if !resource_hash.key?(resource_hash_key) - resource_hash[resource_hash_key] = {} - resource_hash[resource_hash_key]["limit"] = @pod_uid_lookup[lookup_key]["#{resource_type}_limit"] - resource_hash[resource_hash_key]["limit_set"] = @pod_uid_lookup[lookup_key]["#{resource_type}_limit_set"] - resource_hash[resource_hash_key]["record_count"] = @workload_container_count[resource_hash_key] - resource_hash[resource_hash_key]["workload_name"] = @pod_uid_lookup[lookup_key]["workload_name"] - resource_hash[resource_hash_key]["workload_kind"] = @pod_uid_lookup[lookup_key]["workload_kind"] - resource_hash[resource_hash_key]["namespace"] = @pod_uid_lookup[lookup_key]["namespace"] - resource_hash[resource_hash_key]["container"] = @pod_uid_lookup[lookup_key]["container"] - resource_hash[resource_hash_key]["records"] = [] - end - - container_instance_record = {} - pod_name = @pod_uid_lookup[lookup_key]["pod_name"] - #append the record to the hash - # append only if the record is not a duplicate record - container_instance_record["pod_name"] = pod_name - container_instance_record["counter_value"] = record["CounterValue"] - container_instance_record["container"] = @pod_uid_lookup[lookup_key]["container"] - container_instance_record["state"] = calculate_container_instance_state( - container_instance_record["counter_value"], - resource_hash[resource_hash_key]["limit"], - @provider.get_config(MonitorId::CONTAINER_MEMORY_MONITOR_ID)) - resource_hash[resource_hash_key]["records"].push(container_instance_record) - rescue => e - @log.info "Error in HealthContainerCpuMemoryAggregator aggregate #{e.backtrace} #{e.message} #{record}" - end - end - end - - def compute_state() - # if missing records, set state to unknown - # if limits not set, set state to warning - # if all records present, sort in descending order of metric, compute index based on StateThresholdPercentage, get the state (pass/fail/warn) based on monitor state (Using [Fail/Warn]ThresholdPercentage, and set the state) - @memory_records.each{|k,v| - @@memory_last_sent_monitors.delete(k) #remove from last sent list if the record is present in the current set of signals - calculate_monitor_state(v, @provider.get_config(MonitorId::CONTAINER_MEMORY_MONITOR_ID)) - } - - @cpu_records.each{|k,v| - @@cpu_last_sent_monitors.delete(k) #remove from last sent list if the record is present in the current set of signals - calculate_monitor_state(v, @provider.get_config(MonitorId::CONTAINER_CPU_MONITOR_ID)) - } - @log.info "Finished computing state" - end - - def get_records - time_now = Time.now.utc.iso8601 - container_cpu_memory_records = [] - - @cpu_records.each{|resource_key, record| - cpu_limit_mc = 1.0 - if record["limit"].is_a?(Numeric) - cpu_limit_mc = record["limit"]/1000000.to_f - else - @log.info "CPU Limit is not a number #{record['limit']}" - if !@@limit_is_array_event_sent.key?(resource_key) - custom_properties = {} - custom_properties['limit'] = record['limit'] - if record['limit'].is_a?(Array) - record['limit'].each_index{|i| - custom_properties[i] = record['limit'][i] - } - end - @@limit_is_array_event_sent[resource_key] = true - #send once per resource key - ApplicationInsightsUtility.sendCustomEvent(@@LIMIT_IS_ARRAY_EVENT, custom_properties) - end - end - health_monitor_record = { - "timestamp" => time_now, - "state" => record["state"], - "details" => { - "cpu_limit_millicores" => cpu_limit_mc, - "cpu_usage_instances" => record["records"].map{|r| r.each {|k,v| - k == "counter_value" ? r[k] = r[k] / 1000000.to_f : r[k] - }}, - "workload_name" => record["workload_name"], - "workload_kind" => record["workload_kind"], - "namespace" => record["namespace"], - "container" => record["container"], - "limit_set" => record["limit_set"] - } - } - - monitor_instance_id = HealthMonitorHelpers.get_monitor_instance_id(MonitorId::CONTAINER_CPU_MONITOR_ID, resource_key.split('_')) #container_cpu_utilization-namespace-workload-container - - health_record = {} - health_record[HealthMonitorRecordFields::MONITOR_ID] = MonitorId::CONTAINER_CPU_MONITOR_ID - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - container_cpu_memory_records.push(health_record) - } - - # If all records that were sent previously are present in current set, this will not be executed - if @@cpu_last_sent_monitors.keys.size != 0 - @@cpu_last_sent_monitors.keys.each{|key| - begin - @log.info "Container CPU monitor #{key} not present in current set. Sending none state transition" - tokens = key.split('_') - namespace = tokens[0] - workload_name = "#{tokens[0]}~~#{tokens[1]}" - container = tokens[2] - health_monitor_record = { - "timestamp" => time_now, - "state" => HealthMonitorStates::NONE, - "details" => { - "reason" => "No record received for workload #{workload_name}", - "workload_name" => workload_name, - "namespace" => namespace, - "container" => container - } - } - - monitor_instance_id = HealthMonitorHelpers.get_monitor_instance_id(MonitorId::CONTAINER_CPU_MONITOR_ID, key.split('_')) #container_cpu_utilization-namespace-workload-container - - health_record = {} - health_record[HealthMonitorRecordFields::MONITOR_ID] = MonitorId::CONTAINER_CPU_MONITOR_ID - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - container_cpu_memory_records.push(health_record) - rescue => e - @log.info "Error when trying to create NONE State transition signal for #{key} for monitor #{monitor_instance_id} #{e.message}" - next - end - } - end - - @memory_records.each{|resource_key, record| - health_monitor_record = { - "timestamp" => time_now, - "state" => record["state"], - "details" => { - "memory_limit_bytes" => record["limit"], - "memory_usage_instances" => record["records"], - "workload_name" => record["workload_name"], - "workload_kind" => record["workload_kind"], - "namespace" => record["namespace"], - "container" => record["container"] - } - } - - monitor_instance_id = HealthMonitorHelpers.get_monitor_instance_id(MonitorId::CONTAINER_MEMORY_MONITOR_ID, resource_key.split('_')) #container_cpu_utilization-namespace-workload-container - - health_record = {} - health_record[HealthMonitorRecordFields::MONITOR_ID] = MonitorId::CONTAINER_MEMORY_MONITOR_ID - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - container_cpu_memory_records.push(health_record) - } - - # If all records that were sent previously are present in current set, this will not be executed - if @@memory_last_sent_monitors.keys.size != 0 - @@memory_last_sent_monitors.keys.each{|key| - begin - @log.info "Container Memory monitor #{key} not present in current set. Sending none state transition" - tokens = key.split('_') - namespace = tokens[0] - workload_name = "#{tokens[0]}~~#{tokens[1]}" - container = tokens[2] - health_monitor_record = { - "timestamp" => time_now, - "state" => HealthMonitorStates::NONE, - "details" => { - "reason" => "No record received for workload #{workload_name}", - "workload_name" => workload_name, - "namespace" => namespace, - "container" => container - } - } - monitor_instance_id = HealthMonitorHelpers.get_monitor_instance_id(MonitorId::CONTAINER_MEMORY_MONITOR_ID, key.split('_')) #container_cpu_utilization-namespace-workload-container - health_record = {} - health_record[HealthMonitorRecordFields::MONITOR_ID] = MonitorId::CONTAINER_MEMORY_MONITOR_ID - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - container_cpu_memory_records.push(health_record) - rescue => e - @log.info "Error when trying to create NONE State transition signal for #{key} for monitor #{monitor_instance_id} #{e.message}" - next - end - } - end - - #reset the last sent monitors list - @@memory_last_sent_monitors = {} - @@cpu_last_sent_monitors = {} - - # add the current set of signals for comparison in next iteration - @cpu_records.keys.each{|k| - @@cpu_last_sent_monitors[k] = true - } - @memory_records.keys.each{|k| - @@memory_last_sent_monitors[k] = true - } - return container_cpu_memory_records - end - - private - def calculate_monitor_state(v, config) - # sort records by descending order of metric - v["records"] = v["records"].sort_by{|record| record["counter_value"]}.reverse - size = v["records"].size - if !v["record_count"].nil? - if size < v["record_count"] - unknown_count = v["record_count"] - size - for i in unknown_count.downto(1) - # it requires a lot of computation to figure out which actual pod is not sending the signal - v["records"].insert(0, {"counter_value" => -1, "container" => v["container"], "pod_name" => "???", "state" => HealthMonitorStates::UNKNOWN }) #insert -1 for unknown records - end - end - else - v["state"] = HealthMonitorStates::UNKNOWN - container_key = "#{v['workload_name']}~~#{v['container']}" - @log.info "ContainerKey: #{container_key} Records Size: #{size} Records: #{v['records']} Record Count: #{v['record_count']} #{@workload_container_count}" - - if !@@workload_container_count_empty_event_sent.key?(container_key) - custom_properties = {} - custom_properties = custom_properties.merge(v) - custom_properties = custom_properties.merge(@workload_container_count) - @log.info "Custom Properties : #{custom_properties}" - @@workload_container_count_empty_event_sent[container_key] = true - ApplicationInsightsUtility.sendCustomEvent(@@WORKLOAD_CONTAINER_COUNT_EMPTY_EVENT, custom_properties) - end - return #simply return the state as unknown here - end - - if size == 1 - state_index = 0 - else - state_threshold = config['StateThresholdPercentage'].to_f - count = ((state_threshold*size)/100).ceil - state_index = size - count - end - v["state"] = v["records"][state_index]["state"] - end - - def calculate_container_instance_state(counter_value, limit, config) - percent_value = counter_value * 100 / limit - if percent_value > config['FailIfGreaterThanPercentage'] - return HealthMonitorStates::FAIL - elsif percent_value > config['WarnIfGreaterThanPercentage'] - return HealthMonitorStates::WARNING - else - return HealthMonitorStates::PASS - end - end - end -end diff --git a/source/plugins/ruby/health/health_container_cpu_memory_record_formatter.rb b/source/plugins/ruby/health/health_container_cpu_memory_record_formatter.rb deleted file mode 100644 index ebf3abd7e..000000000 --- a/source/plugins/ruby/health/health_container_cpu_memory_record_formatter.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'yajl/json_gem' - -module HealthModel - class HealthContainerCpuMemoryRecordFormatter - - @@health_container_cpu_memory_record_template = '{ - "InstanceName": "%{instance_name}", - "CounterName" : "%{counter_name}", - "CounterValue" : %{metric_value}, - "Timestamp" : "%{timestamp}" - }' - def initialize - @log = HealthMonitorHelpers.get_log_handle - end - - def get_record_from_cadvisor_record(cadvisor_record) - begin - instance_name = cadvisor_record['InstanceName'] - counter_name = JSON.parse(cadvisor_record['json_Collections'])[0]['CounterName'] - metric_value = JSON.parse(cadvisor_record['json_Collections'])[0]['Value'] - timestamp = cadvisor_record['Timestamp'] - - health_container_cpu_memory_record = @@health_container_cpu_memory_record_template % { - instance_name: instance_name, - counter_name: counter_name, - metric_value: metric_value, - timestamp: timestamp - } - return JSON.parse(health_container_cpu_memory_record) - rescue => e - @log.info "Error in get_record_from_cadvisor_record #{e.message} #{e.backtrace}" - return nil - end - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_hierarchy_builder.rb b/source/plugins/ruby/health/health_hierarchy_builder.rb deleted file mode 100644 index a59020996..000000000 --- a/source/plugins/ruby/health/health_hierarchy_builder.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true -require 'yajl/json_gem' - -module HealthModel - class HealthHierarchyBuilder - - attr_accessor :health_model_definition, :monitor_factory - - def initialize(health_model_definition, monitor_factory) - - if !health_model_definition.is_a?(ParentMonitorProvider) - raise "Invalid Type Expected: ParentMonitorProvider Actual: #{@health_model_definition.class.name}" - end - @health_model_definition = health_model_definition - - if !monitor_factory.is_a?(MonitorFactory) - raise "Invalid Type Expected: MonitorFactory Actual: #{@monitor_factory.class.name}" - end - @monitor_factory = monitor_factory - end - - def process_record(health_monitor_record, monitor_set) - if !health_monitor_record.is_a?(HealthMonitorRecord) - raise "Unexpected Type #{health_monitor_record.class}" - end - - # monitor state transition will always be on a unit monitor - child_monitor = @monitor_factory.create_unit_monitor(health_monitor_record) - monitor_set.add_or_update(child_monitor) - parent_monitor_id = @health_model_definition.get_parent_monitor_id(child_monitor) - monitor_labels = child_monitor.labels - monitor_id = child_monitor.monitor_id - - # to construct the parent monitor, - # 1. Child's labels - # 2. Parent monitor's config to determine what labels to copy - # 3. Parent Monitor Id - # 4. Monitor Id --> Labels to hash Mapping to generate the monitor instance id for aggregate monitors - - while !parent_monitor_id.nil? - #puts "Parent Monitor Id #{parent_monitor_id}" - # get the set of labels to copy to parent monitor - parent_monitor_labels = @health_model_definition.get_parent_monitor_labels(monitor_id, monitor_labels, parent_monitor_id) - # get the parent monitor configuration - parent_monitor_configuration = @health_model_definition.get_parent_monitor_config(parent_monitor_id) - #get monitor instance id for parent monitor. Does this belong in ParentMonitorProvider? - parent_monitor_instance_id = @health_model_definition.get_parent_monitor_instance_id(child_monitor.monitor_instance_id, parent_monitor_id, parent_monitor_labels) - # check if monitor set has the parent monitor id - # if not present, add - # if present, update the state based on the aggregation algorithm - parent_monitor = nil - if !monitor_set.contains?(parent_monitor_instance_id) - parent_monitor = @monitor_factory.create_aggregate_monitor(parent_monitor_id, parent_monitor_instance_id, parent_monitor_labels, parent_monitor_configuration['aggregation_algorithm'], parent_monitor_configuration['aggregation_algorithm_params'], child_monitor) - parent_monitor.add_member_monitor(child_monitor.monitor_instance_id) - else - parent_monitor = monitor_set.get_monitor(parent_monitor_instance_id) - # required to calculate the rollup state - parent_monitor.add_member_monitor(child_monitor.monitor_instance_id) - # update to the earliest of the transition times of child monitors - if child_monitor.transition_date_time < parent_monitor.transition_date_time - parent_monitor.transition_date_time = child_monitor.transition_date_time - end - end - - if parent_monitor.nil? - raise 'Parent_monitor should not be nil for #{monitor_id}' - end - - monitor_set.add_or_update(parent_monitor) - - child_monitor = parent_monitor - parent_monitor_id = @health_model_definition.get_parent_monitor_id(child_monitor) - monitor_labels = child_monitor.labels - monitor_id = child_monitor.monitor_id - end - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_kube_api_down_handler.rb b/source/plugins/ruby/health/health_kube_api_down_handler.rb deleted file mode 100644 index bb91f2e3b..000000000 --- a/source/plugins/ruby/health/health_kube_api_down_handler.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require_relative 'health_model_constants' -module HealthModel - class HealthKubeApiDownHandler - def initialize - @@monitors_to_change = [MonitorId::WORKLOAD_CPU_OVERSUBSCRIBED_MONITOR_ID, - MonitorId::WORKLOAD_MEMORY_OVERSUBSCRIBED_MONITOR_ID, - MonitorId::NODE_CONDITION_MONITOR_ID, - MonitorId::USER_WORKLOAD_PODS_READY_MONITOR_ID, - MonitorId::SYSTEM_WORKLOAD_PODS_READY_MONITOR_ID] - end - - # update kube-api dependent monitors to be 'unknown' if kube-api is down or monitor is unavailable - def handle_kube_api_down(health_monitor_records) - health_monitor_records_map = {} - - health_monitor_records.map{|record| health_monitor_records_map[record.monitor_instance_id] = record} - if !health_monitor_records_map.key?(MonitorId::KUBE_API_STATUS) || (health_monitor_records_map.key?(MonitorId::KUBE_API_STATUS) && health_monitor_records_map[MonitorId::KUBE_API_STATUS].state != 'pass') - #iterate over the map and set the state to unknown for related monitors - health_monitor_records.each{|health_monitor_record| - if @@monitors_to_change.include?(health_monitor_record.monitor_id) - health_monitor_record.state = HealthMonitorStates::UNKNOWN - end - } - end - return health_monitor_records - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_kubernetes_resources.rb b/source/plugins/ruby/health/health_kubernetes_resources.rb deleted file mode 100644 index 743dd8b94..000000000 --- a/source/plugins/ruby/health/health_kubernetes_resources.rb +++ /dev/null @@ -1,288 +0,0 @@ -# frozen_string_literal: true - -require 'singleton' -require_relative 'health_model_constants' - -module HealthModel - class HealthKubernetesResources - - include Singleton - attr_accessor :node_inventory, :pod_inventory, :replicaset_inventory, :pod_uid_lookup, :workload_container_count - attr_reader :nodes, :pods, :workloads, :deployment_lookup - - def initialize - @node_inventory = {} - @pod_inventory = {} - @replicaset_inventory = {} - @nodes = [] - @pods = [] - @workloads = [] - @log = HealthMonitorHelpers.get_log_handle - @pod_uid_lookup = {} - @workload_container_count = {} - @workload_name_cache = {} - end - - def get_node_inventory - return @node_inventory - end - - def get_nodes - @nodes = [] - @node_inventory['items'].each {|node| - if !@nodes.include?(node['metadata']['name']) - @nodes.push(node['metadata']['name']) - end - - } - return @nodes - end - - def set_replicaset_inventory(replicasets) - @replicaset_inventory = replicasets - end - - def get_workload_names - workload_names = {} - @pod_inventory['items'].each do |pod| - workload_name = get_workload_name(pod) - workload_names[workload_name] = true if workload_name - end - return workload_names.keys - end - - def build_pod_uid_lookup - if @pod_inventory.nil? || @pod_inventory['items'].nil? || @pod_inventory['items'].empty? || @pod_inventory['items'].size == 0 - @log.info "Not Clearing pod_uid_lookup and workload_container_count since pod inventory is nil" - return - end - @workload_container_count = {} - @pod_uid_lookup = {} - @pod_inventory['items'].each do |pod| - begin - namespace = pod['metadata']['namespace'] - poduid = pod['metadata']['uid'] - pod_name = pod['metadata']['name'] - workload_name = get_workload_name(pod) - workload_kind = get_workload_kind(pod) - # we don't show jobs in container health - if workload_kind.casecmp('job') == 0 - next - end - pod['spec']['containers'].each do |container| - cname = container['name'] - key = "#{poduid}/#{cname}" - cpu_limit_set = true - memory_limit_set = true - begin - cpu_limit = get_numeric_value('cpu', container['resources']['limits']['cpu']) - rescue => exception - #@log.info "Exception getting container cpu limit #{container['resources']}" - cpu_limit = get_node_capacity(pod['spec']['nodeName'], 'cpu') - cpu_limit_set = false - end - begin - memory_limit = get_numeric_value('memory', container['resources']['limits']['memory']) - rescue => exception - #@log.info "Exception getting container memory limit #{container['resources']}" - memory_limit = get_node_capacity(pod['spec']['nodeName'], 'memory') - memory_limit_set = false - end - @pod_uid_lookup[key] = {"workload_kind" => workload_kind, "workload_name" => workload_name, "namespace" => namespace, "cpu_limit" => cpu_limit, "memory_limit" => memory_limit, "cpu_limit_set" => cpu_limit_set, "memory_limit_set" => memory_limit_set, "container" => cname, "pod_name" => pod_name} - container_count_key = "#{namespace}_#{workload_name.split('~~')[1]}_#{cname}" - if !@workload_container_count.key?(container_count_key) - @workload_container_count[container_count_key] = 1 - else - count = @workload_container_count[container_count_key] - @workload_container_count[container_count_key] = count + 1 - end - end - rescue => e - @log.info "Error in build_pod_uid_lookup for POD: #{pod_name} #{e.message} #{e.backtrace}" - end - end - end - - def get_pod_uid_lookup - return @pod_uid_lookup - end - - def get_workload_container_count - return @workload_container_count - end - - def get_workload_name(pod) - begin - has_owner = !pod['metadata']['ownerReferences'].nil? - owner_kind = '' - if has_owner - owner_kind = pod['metadata']['ownerReferences'][0]['kind'] - controller_name = pod['metadata']['ownerReferences'][0]['name'] - else - owner_kind = pod['kind'] - controller_name = pod['metadata']['name'] - end - namespace = pod['metadata']['namespace'] - workload_name = '' - if owner_kind.nil? - owner_kind = 'Pod' - end - case owner_kind.downcase - when 'job' - # we are excluding jobs - return nil - when 'replicaset' - #TODO: - workload_name = get_replica_set_owner_ref(controller_name) - workload_name = "#{namespace}~~#{workload_name}" - when 'daemonset' - workload_name = "#{namespace}~~#{controller_name}" - else - workload_name = "#{namespace}~~#{controller_name}" - end - return workload_name - rescue => e - @log.info "Error in get_workload_name(pod) #{e.message} #{e.backtrace}" - return nil - end - end - - def get_workload_kind(pod) - begin - has_owner = !pod['metadata']['ownerReferences'].nil? - owner_kind = '' - if has_owner - owner_kind = pod['metadata']['ownerReferences'][0]['kind'] - else - owner_kind = pod['kind'] - end - - if owner_kind.nil? - owner_kind = 'Pod' - end - return owner_kind - rescue => e - @log.info "Error in get_workload_kind(pod) #{e.message}" - return nil - end - end - - private - def get_replica_set_owner_ref(controller_name) - if @workload_name_cache.key?(controller_name) - return @workload_name_cache[controller_name] - end - begin - owner_ref = controller_name - @replicaset_inventory['items'].each{|rs| - rs_name = rs['metadata']['name'] - if controller_name.casecmp(rs_name) == 0 - if !rs['metadata']['ownerReferences'].nil? - owner_ref = rs['metadata']['ownerReferences'][0]['name'] if rs['metadata']['ownerReferences'][0]['name'] - end - break - end - } - @workload_name_cache[controller_name] = owner_ref - return owner_ref - rescue => e - @log.info "Error in get_replica_set_owner_ref(controller_name) #{e.message}" - return controller_name - end - end - - def get_node_capacity(node_name, type) - if node_name.nil? #unscheduled pods will not have a node name - return -1 - end - begin - @node_inventory["items"].each do |node| - if (!node["status"]["capacity"].nil?) && node["metadata"]["name"].casecmp(node_name.downcase) == 0 - return get_numeric_value(type, node["status"]["capacity"][type]) - end - end - rescue => e - @log.info "Error in get_node_capacity(pod, #{type}) #{e.backtrace} #{e.message}" - return -1 - end - end - - #Cannot reuse the code from KubernetesApiClient, for unit testing reasons. KubernetesApiClient has a dependency on oms_common.rb etc. - def get_numeric_value(metricName, metricVal) - metricValue = metricVal.downcase - begin - case metricName - when "memory" #convert to bytes for memory - #https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/ - if (metricValue.end_with?("ki")) - metricValue.chomp!("ki") - metricValue = Float(metricValue) * 1024.0 ** 1 - elsif (metricValue.end_with?("mi")) - metricValue.chomp!("mi") - metricValue = Float(metricValue) * 1024.0 ** 2 - elsif (metricValue.end_with?("gi")) - metricValue.chomp!("gi") - metricValue = Float(metricValue) * 1024.0 ** 3 - elsif (metricValue.end_with?("ti")) - metricValue.chomp!("ti") - metricValue = Float(metricValue) * 1024.0 ** 4 - elsif (metricValue.end_with?("pi")) - metricValue.chomp!("pi") - metricValue = Float(metricValue) * 1024.0 ** 5 - elsif (metricValue.end_with?("ei")) - metricValue.chomp!("ei") - metricValue = Float(metricValue) * 1024.0 ** 6 - elsif (metricValue.end_with?("zi")) - metricValue.chomp!("zi") - metricValue = Float(metricValue) * 1024.0 ** 7 - elsif (metricValue.end_with?("yi")) - metricValue.chomp!("yi") - metricValue = Float(metricValue) * 1024.0 ** 8 - elsif (metricValue.end_with?("k")) - metricValue.chomp!("k") - metricValue = Float(metricValue) * 1000.0 ** 1 - elsif (metricValue.end_with?("m")) - metricValue.chomp!("m") - metricValue = Float(metricValue) * 1000.0 ** 2 - elsif (metricValue.end_with?("g")) - metricValue.chomp!("g") - metricValue = Float(metricValue) * 1000.0 ** 3 - elsif (metricValue.end_with?("t")) - metricValue.chomp!("t") - metricValue = Float(metricValue) * 1000.0 ** 4 - elsif (metricValue.end_with?("p")) - metricValue.chomp!("p") - metricValue = Float(metricValue) * 1000.0 ** 5 - elsif (metricValue.end_with?("e")) - metricValue.chomp!("e") - metricValue = Float(metricValue) * 1000.0 ** 6 - elsif (metricValue.end_with?("z")) - metricValue.chomp!("z") - metricValue = Float(metricValue) * 1000.0 ** 7 - elsif (metricValue.end_with?("y")) - metricValue.chomp!("y") - metricValue = Float(metricValue) * 1000.0 ** 8 - else #assuming there are no units specified, it is bytes (the below conversion will fail for other unsupported 'units') - metricValue = Float(metricValue) - end - when "cpu" #convert to nanocores for cpu - #https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/ - if (metricValue.end_with?("m")) - metricValue.chomp!("m") - metricValue = Float(metricValue) * 1000.0 ** 2 - else #assuming no units specified, it is cores that we are converting to nanocores (the below conversion will fail for other unsupported 'units') - metricValue = Float(metricValue) * 1000.0 ** 3 - end - else - @Log.warn("getMetricNumericValue: Unsupported metric #{metricName}. Returning 0 for metric value") - metricValue = 0 - end #case statement - rescue => error - @Log.warn("getMetricNumericValue failed: #{error} for metric #{metricName} with value #{metricVal}. Returning 0 formetric value") - return 0 - end - return metricValue - end - - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_missing_signal_generator.rb b/source/plugins/ruby/health/health_missing_signal_generator.rb deleted file mode 100644 index 84af81ea7..000000000 --- a/source/plugins/ruby/health/health_missing_signal_generator.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -require_relative 'health_model_constants' -require_relative 'health_monitor_record' - -module HealthModel - class HealthMissingSignalGenerator - attr_accessor :last_received_records, :current_received_records - attr_reader :missing_signals, :unknown_signals_hash - - def initialize() - @last_received_records = {} - @unknown_signals_hash = {} - end - - def get_missing_signals(cluster_id, health_monitor_records, health_k8s_inventory, provider) - missing_monitor_ids = [] - nodes = health_k8s_inventory.get_nodes - workload_names = health_k8s_inventory.get_workload_names - missing_signals_map = {} - missing_signals = [] - health_monitor_records_map = {} - health_monitor_records.map{ - |monitor| health_monitor_records_map[monitor.monitor_instance_id] = monitor - } - - node_signals_hash = {} - nodes.each{|node| - node_signals_hash[node] = [MonitorId::NODE_MEMORY_MONITOR_ID, MonitorId::NODE_CPU_MONITOR_ID, MonitorId::NODE_CONDITION_MONITOR_ID] - } - log = HealthMonitorHelpers.get_log_handle - log.info "last_received_records #{@last_received_records.size} nodes #{nodes}" - @last_received_records.each{|monitor_instance_id, monitor| - if !health_monitor_records_map.key?(monitor_instance_id) - if HealthMonitorHelpers.is_node_monitor(monitor.monitor_id) - node_name = monitor.labels[HealthMonitorLabels::HOSTNAME] - new_monitor = HealthMonitorRecord.new( - monitor.monitor_id, - monitor.monitor_instance_id, - Time.now.utc.iso8601, - monitor.state, - monitor.labels, - monitor.config, - {"timestamp" => Time.now.utc.iso8601, "state" => HealthMonitorStates::UNKNOWN, "details" => ""} - ) - if !node_name.nil? && nodes.include?(node_name) - new_monitor.state = HealthMonitorStates::UNKNOWN - new_monitor.details["state"] = HealthMonitorStates::UNKNOWN - new_monitor.details["details"] = "Node present in inventory but no signal for #{monitor.monitor_id} from node #{node_name}" - @unknown_signals_hash[monitor_instance_id] = new_monitor - elsif !node_name.nil? && !nodes.include?(node_name) - new_monitor.state = HealthMonitorStates::NONE - new_monitor.details["state"] = HealthMonitorStates::NONE - new_monitor.details["details"] = "Node NOT present in inventory. node: #{node_name}" - end - missing_signals_map[monitor_instance_id] = new_monitor - log.info "Added missing signal #{new_monitor.monitor_instance_id} #{new_monitor.state}" - elsif HealthMonitorHelpers.is_pods_ready_monitor(monitor.monitor_id) - lookup = "#{monitor.labels[HealthMonitorLabels::NAMESPACE]}~~#{monitor.labels[HealthMonitorLabels::WORKLOAD_NAME]}" - new_monitor = HealthMonitorRecord.new( - monitor.monitor_id, - monitor.monitor_instance_id, - Time.now.utc.iso8601, - monitor.state, - monitor.labels, - monitor.config, - {"timestamp" => Time.now.utc.iso8601, "state" => HealthMonitorStates::UNKNOWN, "details" => ""} - ) - if !lookup.nil? && workload_names.include?(lookup) - new_monitor.state = HealthMonitorStates::UNKNOWN - new_monitor.details["state"] = HealthMonitorStates::UNKNOWN - new_monitor.details["details"] = "Workload present in inventory. But no signal for #{lookup}" - @unknown_signals_hash[monitor_instance_id] = new_monitor - elsif !lookup.nil? && !workload_names.include?(lookup) - new_monitor.state = HealthMonitorStates::NONE - new_monitor.details["state"] = HealthMonitorStates::NONE - new_monitor.details["details"] = "Workload #{lookup} NOT present in inventory" - end - missing_signals_map[monitor_instance_id] = new_monitor - end - end - } - - - health_monitor_records.each{|health_monitor_record| - # remove signals from the list of expected signals if we see them in the list of current signals - if HealthMonitorHelpers.is_node_monitor(health_monitor_record.monitor_id) - node_name = health_monitor_record.labels[HealthMonitorLabels::HOSTNAME] - if node_signals_hash.key?(node_name) - signals = node_signals_hash[node_name] - signals.delete(health_monitor_record.monitor_id) - if signals.size == 0 - node_signals_hash.delete(node_name) - end - end - end - } - - # if the hash is not empty, means we have missing signals - if node_signals_hash.size > 0 - # these signals were not sent previously - # these signals need to be assigned an unknown state - node_signals_hash.each{|node, monitor_ids| - monitor_ids.each{|monitor_id| - monitor_instance_id = HealthMonitorHelpers.get_monitor_instance_id(monitor_id, [cluster_id, node]) - new_monitor = HealthMonitorRecord.new( - monitor_id, - monitor_instance_id, - Time.now.utc.iso8601, - HealthMonitorStates::UNKNOWN, - provider.get_node_labels(node), - {}, - {"timestamp" => Time.now.utc.iso8601, "state" => HealthMonitorStates::UNKNOWN, "details" => "no signal received from node #{node}"} - ) - missing_signals_map[monitor_instance_id] = new_monitor - log.info "Added missing signal when node_signals_hash was not empty #{new_monitor.monitor_instance_id} #{new_monitor.state} #{new_monitor.labels.keys}" - } - } - end - - missing_signals_map.each{|k,v| - missing_signals.push(v) - } - - # if an unknown signal is present neither in missing signals or the incoming signals, change its state to none, and remove from unknown_signals - # in update_state of HealthMonitorState, send if latest_record_state is none - @unknown_signals_hash.each{|k,v| - if !missing_signals_map.key?(k) && !health_monitor_records_map.key?(k) - monitor_record = @unknown_signals_hash[k] - monitor_record.details["state"] = HealthMonitorStates::NONE # used for calculating the old and new states in update_state - monitor_record.state = HealthMonitorStates::NONE #used for calculating the aggregate monitor state - missing_signals.push(monitor_record) - @unknown_signals_hash.delete(k) - log.info "Updating state from unknown to none for #{k}" - end - } - return missing_signals - end - - def update_last_received_records(last_received_records) - last_received_records_map = {} - last_received_records.map {|record| last_received_records_map[record.monitor_instance_id] = record } - @last_received_records = last_received_records_map - end - end - -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_model_buffer.rb b/source/plugins/ruby/health/health_model_buffer.rb deleted file mode 100644 index 1c3ec3332..000000000 --- a/source/plugins/ruby/health/health_model_buffer.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module HealthModel - -=begin - Class that is used to create a buffer for collecting the health records -=end - class HealthModelBuffer - - attr_reader :records_buffer, :log - - def initialize - @records_buffer = [] - end - - # Returns the current buffer - def get_buffer - return @records_buffer - end - - # adds records to the buffer - def add_to_buffer(records) - @records_buffer.push(*records) - end - - # clears/resets the buffer - def reset_buffer - @records_buffer = [] - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_model_builder.rb b/source/plugins/ruby/health/health_model_builder.rb deleted file mode 100644 index 43ed30d05..000000000 --- a/source/plugins/ruby/health/health_model_builder.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true -require 'time' - -module HealthModel - class HealthModelBuilder - attr_accessor :hierarchy_builder, :state_finalizers, :monitor_set - - def initialize(hierarchy_builder, state_finalizers, monitor_set) - @hierarchy_builder = hierarchy_builder - @state_finalizers = state_finalizers - @monitor_set = monitor_set - end - - def process_records(health_records) - health_records.each{|health_record| - @hierarchy_builder.process_record(health_record, @monitor_set) - } - end - - def finalize_model - if !@state_finalizers.is_a?(Array) - raise 'state finalizers should be an array' - end - - if @state_finalizers.length == 0 - raise '@state_finalizers length should not be zero or empty' - end - - @state_finalizers.each{|finalizer| - finalizer.finalize(@monitor_set) - } - - return @monitor_set.get_map - end - - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_model_constants.rb b/source/plugins/ruby/health/health_model_constants.rb deleted file mode 100644 index c74f86f4d..000000000 --- a/source/plugins/ruby/health/health_model_constants.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true -module HealthModel - class MonitorState - CRITICAL = "fail" - ERROR = "err" - HEALTHY = "pass" - NONE = "none" - UNKNOWN = "unknown" - WARNING = "warn" - end - - class AggregationAlgorithm - PERCENTAGE = "percentage" - WORSTOF = "worstOf" - end - - class MonitorId - AGENT_NODE_POOL = 'agent_node_pool' - ALL_AGENT_NODE_POOLS = 'all_agent_node_pools' - ALL_NODE_POOLS = 'all_node_pools' - ALL_NODES = 'all_nodes' - CAPACITY = 'capacity' - CLUSTER = 'cluster' - CONTAINER = 'container' - CONTAINER_CPU_MONITOR_ID = "container_cpu_utilization" - CONTAINER_MEMORY_MONITOR_ID = "container_memory_utilization" - K8S_INFRASTRUCTURE = 'k8s_infrastructure' - KUBE_API_STATUS = "kube_api_status" - MASTER_NODE_POOL = 'master_node_pool' - NAMESPACE = 'namespace'; - NODE = 'node'; - NODE_CONDITION_MONITOR_ID = "node_condition" - NODE_CPU_MONITOR_ID = "node_cpu_utilization" - NODE_MEMORY_MONITOR_ID = "node_memory_utilization" - SYSTEM_WORKLOAD = 'system_workload' - SYSTEM_WORKLOAD_PODS_READY_MONITOR_ID = "system_workload_pods_ready" - USER_WORKLOAD = 'user_workload'; - USER_WORKLOAD_PODS_READY_MONITOR_ID = "user_workload_pods_ready" - WORKLOAD = 'all_workloads'; - WORKLOAD_CONTAINER_CPU_PERCENTAGE_MONITOR_ID = "container_cpu_utilization" - WORKLOAD_CONTAINER_MEMORY_PERCENTAGE_MONITOR_ID = "container_memory_utilization" - WORKLOAD_CPU_OVERSUBSCRIBED_MONITOR_ID = "subscribed_capacity_cpu" - WORKLOAD_MEMORY_OVERSUBSCRIBED_MONITOR_ID = "subscribed_capacity_memory" - end - - class HealthMonitorRecordFields - CLUSTER_ID = "ClusterId" - DETAILS = "Details" - HEALTH_MODEL_DEFINITION_VERSION = "HealthModelDefinitionVersion" - MONITOR_CONFIG = "MonitorConfig" - MONITOR_ID = "MonitorTypeId" - MONITOR_INSTANCE_ID = "MonitorInstanceId" - MONITOR_LABELS = "MonitorLabels" - NEW_STATE = "NewState" - NODE_NAME = "NodeName" - OLD_STATE = "OldState" - PARENT_MONITOR_INSTANCE_ID = "ParentMonitorInstanceId" - TIME_FIRST_OBSERVED = "TimeFirstObserved" - TIME_GENERATED = "TimeGenerated" - end - - class HealthMonitorStates - FAIL = "fail" - NONE = "none" - PASS = "pass" - UNKNOWN = "unknown" - WARNING = "warn" - end - - class HealthMonitorLabels - AGENTPOOL = "agentpool" - CONTAINER = "container.azm.ms/container" - HOSTNAME = "kubernetes.io/hostname" - NAMESPACE = "container.azm.ms/namespace" - ROLE = "kubernetes.io/role" - WORKLOAD_KIND = "container.azm.ms/workload-kind" - WORKLOAD_NAME = "container.azm.ms/workload-name" - MASTERROLE = "node-role.kubernetes.io/master" - COMPUTEROLE = "node-role.kubernetes.io/compute" - INFRAROLE = "node-role.kubernetes.io/infra" - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_model_definition_parser.rb b/source/plugins/ruby/health/health_model_definition_parser.rb deleted file mode 100644 index c185e5389..000000000 --- a/source/plugins/ruby/health/health_model_definition_parser.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true -=begin - Class to parse the health model definition. The definition expresses the relationship between monitors, how to roll up to an aggregate monitor, - and what labels to "pass on" to the parent monitor -=end -require 'yajl/json_gem' - -module HealthModel - class HealthModelDefinitionParser - attr_accessor :health_model_definition_path, :health_model_definition - - # Constructor - def initialize(path) - @health_model_definition = {} - @health_model_definition_path = path - end - - # Parse the health model definition file and build the model roll-up hierarchy - def parse_file - if (!File.exist?(@health_model_definition_path)) - raise "File does not exist in the specified path" - end - - file = File.read(@health_model_definition_path) - temp_model = JSON.parse(file) - temp_model.each { |entry| - monitor_id = entry['monitor_id'] - parent_monitor_id = entry['parent_monitor_id'] - labels = entry['labels'] if entry['labels'] - aggregation_algorithm = entry['aggregation_algorithm'] if entry['aggregation_algorithm'] - aggregation_algorithm_params = entry['aggregation_algorithm_params'] if entry['aggregation_algorithm_params'] - default_parent_monitor_id = entry['default_parent_monitor_id'] if entry['default_parent_monitor_id'] - if parent_monitor_id.is_a?(Array) - conditions = [] - parent_monitor_id.each{|condition| - key = condition['label'] - operator = condition['operator'] - value = condition['value'] - parent_id = condition['id'] - conditions.push({"key" => key, "operator" => operator, "value" => value, "parent_id" => parent_id}) - } - @health_model_definition[monitor_id] = {"conditions" => conditions, "labels" => labels, "aggregation_algorithm" => aggregation_algorithm, "aggregation_algorithm_params" =>aggregation_algorithm_params, "default_parent_monitor_id" => default_parent_monitor_id} - elsif parent_monitor_id.is_a?(String) - @health_model_definition[monitor_id] = {"parent_monitor_id" => parent_monitor_id, "labels" => labels, "aggregation_algorithm" => aggregation_algorithm, "aggregation_algorithm_params" =>aggregation_algorithm_params} - elsif parent_monitor_id.nil? - @health_model_definition[monitor_id] = {"parent_monitor_id" => nil, "labels" => labels, "aggregation_algorithm" => aggregation_algorithm, "aggregation_algorithm_params" =>aggregation_algorithm_params} - end - } - @health_model_definition - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_monitor_helpers.rb b/source/plugins/ruby/health/health_monitor_helpers.rb deleted file mode 100644 index 74aa35af0..000000000 --- a/source/plugins/ruby/health/health_monitor_helpers.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true -require 'logger' -require 'digest' -require_relative 'health_model_constants' - -module HealthModel - # static class that provides a bunch of utility methods - class HealthMonitorHelpers - - @log_path = "/var/opt/microsoft/docker-cimprov/log/health_monitors.log" - - if Gem.win_platform? #unit testing on windows dev machine - @log_path = "C:\Temp\health_monitors.log" - end - - @log = Logger.new(@log_path, 2, 10 * 1048576) #keep last 2 files, max log file size = 10M - - class << self - def is_node_monitor(monitor_id) - return (monitor_id == MonitorId::NODE_CPU_MONITOR_ID || monitor_id == MonitorId::NODE_MEMORY_MONITOR_ID || monitor_id == MonitorId::NODE_CONDITION_MONITOR_ID) - end - - def is_pods_ready_monitor(monitor_id) - return (monitor_id == MonitorId::USER_WORKLOAD_PODS_READY_MONITOR_ID || monitor_id == MonitorId::SYSTEM_WORKLOAD_PODS_READY_MONITOR_ID) - end - - def get_log_handle - return @log - end - - def get_monitor_instance_id(monitor_id, args = []) - string_to_hash = args.join("/") - return "#{monitor_id}-#{Digest::MD5.hexdigest(string_to_hash)}" - end - - def add_agentpool_node_label_if_not_present(records) - records.each{|record| - # continue if it is not a node monitor - if !is_node_monitor(record.monitor_id) - #@log.info "#{record.monitor_id} is not a NODE MONITOR" - next - end - labels_keys = record.labels.keys - - if labels_keys.include?(HealthMonitorLabels::AGENTPOOL) - @log.info "#{record.monitor_id} includes agentpool label. Value = #{record.labels[HealthMonitorLabels::AGENTPOOL]}" - next - else - #@log.info "#{record} does not include agentpool label." - role_name = 'unknown' - if record.labels.include?(HealthMonitorLabels::ROLE) - role_name = record.labels[HealthMonitorLabels::ROLE] - elsif record.labels.include?(HealthMonitorLabels::MASTERROLE) - if !record.labels[HealthMonitorLabels::MASTERROLE].empty? - role_name = 'master' - end - elsif record.labels.include?(HealthMonitorLabels::COMPUTEROLE) - if !record.labels[HealthMonitorLabels::COMPUTEROLE].empty? - role_name = 'compute' - end - elsif record.labels.include?(HealthMonitorLabels::INFRAROLE) - if !record.labels[HealthMonitorLabels::INFRAROLE].empty? - role_name = 'infra' - end - end - @log.info "Adding agentpool label #{role_name}_node_pool for #{record.monitor_id}" - record.labels[HealthMonitorLabels::AGENTPOOL] = "#{role_name}_node_pool" - end - } - end - end - - end -end diff --git a/source/plugins/ruby/health/health_monitor_optimizer.rb b/source/plugins/ruby/health/health_monitor_optimizer.rb deleted file mode 100644 index d87540941..000000000 --- a/source/plugins/ruby/health/health_monitor_optimizer.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true -require 'yajl/json_gem' -module HealthModel - class HealthMonitorOptimizer - #ctor - def initialize - @@health_signal_timeout = 240 - @@first_record_sent = {} - end - - def should_send(monitor_instance_id, health_monitor_state, health_monitor_config) - - health_monitor_instance_state = health_monitor_state.get_state(monitor_instance_id) - health_monitor_records = health_monitor_instance_state.prev_records - health_monitor_config['ConsecutiveSamplesForStateTransition'].nil? ? samples_to_check = 1 : samples_to_check = health_monitor_config['ConsecutiveSamplesForStateTransition'].to_i - - latest_record = health_monitor_records[health_monitor_records.size-1] #since we push new records to the end, and remove oldest records from the beginning - latest_record_state = latest_record["state"] - latest_record_time = latest_record["timestamp"] #string representation of time - - new_state = health_monitor_instance_state.new_state - prev_sent_time = health_monitor_instance_state.prev_sent_record_time - time_first_observed = health_monitor_instance_state.state_change_time - - if latest_record_state.downcase == new_state.downcase - time_elapsed = (Time.parse(latest_record_time) - Time.parse(prev_sent_time)) / 60 - if time_elapsed > @@health_signal_timeout # minutes - return true - elsif !@@first_record_sent.key?(monitor_instance_id) - @@first_record_sent[monitor_instance_id] = true - return true - else - return false - end - else - if samples_to_check == 1 - return true - elsif health_monitor_instance_state.prev_records.size == 1 && samples_to_check > 1 - return true - elsif health_monitor_instance_state.prev_records.size < samples_to_check - return false - else - # state change from previous sent state to latest record state - #check state of last n records to see if they are all in the same state - if (health_monitor_instance_state.is_state_change_consistent) - return true - else - return false - end - end - end - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_monitor_provider.rb b/source/plugins/ruby/health/health_monitor_provider.rb deleted file mode 100644 index 8e1d11143..000000000 --- a/source/plugins/ruby/health/health_monitor_provider.rb +++ /dev/null @@ -1,139 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' -require 'yajl/json_gem' - -module HealthModel - class HealthMonitorProvider - - attr_accessor :cluster_labels, :health_kubernetes_resources, :monitor_configuration_path, :cluster_id - attr_reader :monitor_configuration - - def initialize(cluster_id, cluster_labels, health_kubernetes_resources, monitor_configuration_path) - @cluster_labels = Hash.new - cluster_labels.each{|k,v| @cluster_labels[k] = v} - @cluster_id = cluster_id - @health_kubernetes_resources = health_kubernetes_resources - @monitor_configuration_path = monitor_configuration_path - begin - @monitor_configuration = {} - file = File.open(@monitor_configuration_path, "r") - if !file.nil? - fileContents = file.read - @monitor_configuration = JSON.parse(fileContents) - file.close - end - rescue => e - @log.info "Error when opening health config file #{e}" - end - end - - def get_record(health_monitor_record, health_monitor_state) - - labels = Hash.new - @cluster_labels.each{|k,v| labels[k] = v} - monitor_id = health_monitor_record.monitor_id - monitor_instance_id = health_monitor_record.monitor_instance_id - health_monitor_instance_state = health_monitor_state.get_state(monitor_instance_id) - - - monitor_labels = health_monitor_record.labels - if !monitor_labels.empty? - monitor_labels.keys.each do |key| - labels[key] = monitor_labels[key] - end - end - - prev_records = health_monitor_instance_state.prev_records - time_first_observed = health_monitor_instance_state.state_change_time # the oldest collection time - new_state = health_monitor_instance_state.new_state # this is updated before formatRecord is called - old_state = health_monitor_instance_state.old_state - - config = get_config(monitor_id) - - if prev_records.size == 1 - details = prev_records[0] - else - details = prev_records - end - - time_observed = Time.now.utc.iso8601 - - monitor_record = {} - - monitor_record[HealthMonitorRecordFields::CLUSTER_ID] = @cluster_id - monitor_record[HealthMonitorRecordFields::MONITOR_LABELS] = labels.to_json - monitor_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - monitor_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - monitor_record[HealthMonitorRecordFields::NEW_STATE] = new_state - monitor_record[HealthMonitorRecordFields::OLD_STATE] = old_state - monitor_record[HealthMonitorRecordFields::DETAILS] = details.to_json - monitor_record[HealthMonitorRecordFields::MONITOR_CONFIG] = config.to_json - monitor_record[HealthMonitorRecordFields::TIME_GENERATED] = Time.now.utc.iso8601 - monitor_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_first_observed - monitor_record[HealthMonitorRecordFields::PARENT_MONITOR_INSTANCE_ID] = '' - - return monitor_record - end - - def get_config(monitor_id) - if @monitor_configuration.key?(monitor_id) - return @monitor_configuration[monitor_id] - else - return {} - end - end - - def get_labels(health_monitor_record) - monitor_labels = Hash.new - @cluster_labels.keys.each{|key| - monitor_labels[key] = @cluster_labels[key] - } - monitor_id = health_monitor_record[HealthMonitorRecordFields::MONITOR_ID] - case monitor_id - when MonitorId::CONTAINER_CPU_MONITOR_ID, MonitorId::CONTAINER_MEMORY_MONITOR_ID, MonitorId::USER_WORKLOAD_PODS_READY_MONITOR_ID, MonitorId::SYSTEM_WORKLOAD_PODS_READY_MONITOR_ID - - namespace = health_monitor_record[HealthMonitorRecordFields::DETAILS]['details']['namespace'] - workload_name = health_monitor_record[HealthMonitorRecordFields::DETAILS]['details']['workload_name'] - workload_kind = health_monitor_record[HealthMonitorRecordFields::DETAILS]['details']['workload_kind'] - - monitor_labels[HealthMonitorLabels::WORKLOAD_NAME] = workload_name.split('~~')[1] - monitor_labels[HealthMonitorLabels::WORKLOAD_KIND] = workload_kind - monitor_labels[HealthMonitorLabels::NAMESPACE] = namespace - - # add the container name for container memory/cpu - if monitor_id == MonitorId::CONTAINER_CPU_MONITOR_ID || monitor_id == MonitorId::CONTAINER_MEMORY_MONITOR_ID - container = health_monitor_record[HealthMonitorRecordFields::DETAILS]['details']['container'] - monitor_labels[HealthMonitorLabels::CONTAINER] = container - end - - #TODO: This doesn't belong here. Move this elsewhere - health_monitor_record[HealthMonitorRecordFields::DETAILS]['details'].delete('namespace') - health_monitor_record[HealthMonitorRecordFields::DETAILS]['details'].delete('workload_name') - health_monitor_record[HealthMonitorRecordFields::DETAILS]['details'].delete('workload_kind') - - when MonitorId::NODE_CPU_MONITOR_ID, MonitorId::NODE_MEMORY_MONITOR_ID, MonitorId::NODE_CONDITION_MONITOR_ID - node_name = health_monitor_record[HealthMonitorRecordFields::NODE_NAME] - @health_kubernetes_resources.get_node_inventory['items'].each do |node| - if !node_name.nil? && !node['metadata']['name'].nil? && node_name == node['metadata']['name'] - if !node["metadata"].nil? && !node["metadata"]["labels"].nil? - monitor_labels = monitor_labels.merge(node["metadata"]["labels"]) - end - end - end - end - return monitor_labels - end - - def get_node_labels(node_name) - monitor_labels = {} - @health_kubernetes_resources.get_node_inventory['items'].each do |node| - if !node_name.nil? && !node['metadata']['name'].nil? && node_name == node['metadata']['name'] - if !node["metadata"].nil? && !node["metadata"]["labels"].nil? - monitor_labels = node["metadata"]["labels"] - end - end - end - return monitor_labels - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_monitor_record.rb b/source/plugins/ruby/health/health_monitor_record.rb deleted file mode 100644 index 7df84ff53..000000000 --- a/source/plugins/ruby/health/health_monitor_record.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true -HealthMonitorRecord = Struct.new( - :monitor_id, - :monitor_instance_id, - :transition_date_time, - :state, - :labels, - :config, - :details - ) do -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_monitor_state.rb b/source/plugins/ruby/health/health_monitor_state.rb deleted file mode 100644 index 110793eeb..000000000 --- a/source/plugins/ruby/health/health_monitor_state.rb +++ /dev/null @@ -1,266 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' -require 'yajl/json_gem' - -module HealthModel - - HealthMonitorInstanceState = Struct.new(:prev_sent_record_time, :old_state, :new_state, :state_change_time, :prev_records, :is_state_change_consistent, :should_send) do - end - - # Class that is used to store the last sent state and latest monitors - # provides services like - # get_state -- returns the current state and details - # update_instance -- updates the state of the health monitor history records - # set_state -- sets the last health monitor state - class HealthMonitorState - - def initialize - @@monitor_states = {} - @@first_record_sent = {} - @@health_signal_timeout = 240 - - end - - def get_state(monitor_instance_id) - if @@monitor_states.key?(monitor_instance_id) - return @@monitor_states[monitor_instance_id] - end - end - - def set_state(monitor_instance_id, health_monitor_instance_state) - @@monitor_states[monitor_instance_id] = health_monitor_instance_state - end - - def to_h - return @@monitor_states - end - - def initialize_state(deserialized_state) - @@monitor_states = {} - deserialized_state.each{|k,v| - health_monitor_instance_state_hash = v - state = HealthMonitorInstanceState.new(*health_monitor_instance_state_hash.values_at(*HealthMonitorInstanceState.members)) - state.prev_sent_record_time = health_monitor_instance_state_hash["prev_sent_record_time"] - state.old_state = health_monitor_instance_state_hash["old_state"] - state.new_state = health_monitor_instance_state_hash["new_state"] - state.state_change_time = health_monitor_instance_state_hash["state_change_time"] - state.prev_records = health_monitor_instance_state_hash["prev_records"] - state.is_state_change_consistent = health_monitor_instance_state_hash["is_state_change_consistent"] || false - state.should_send = health_monitor_instance_state_hash["should_send"] - @@monitor_states[k] = state - @@first_record_sent[k] = true - } - end - -=begin -when do u send? ---------------- -1. if the signal hasnt been sent before -2. if there is a "consistent" state change for monitors -3. if the signal is stale (> 4hrs) -4. If the latest state is none -5. If an aggregate monitor has a change in its details, but no change in state -=end - def update_state(monitor, #UnitMonitor/AggregateMonitor - monitor_config, #Hash - is_aggregate_monitor = false, - telemetry = nil - ) - samples_to_keep = 1 - monitor_id = monitor.monitor_id - monitor_instance_id = monitor.monitor_instance_id - log = HealthMonitorHelpers.get_log_handle - current_time = Time.now.utc.iso8601 - health_monitor_instance_state = get_state(monitor_instance_id) - if !health_monitor_instance_state.nil? - health_monitor_instance_state.is_state_change_consistent = false - health_monitor_instance_state.should_send = false - set_state(monitor_instance_id, health_monitor_instance_state) # reset is_state_change_consistent - end - - if !monitor_config.nil? && !monitor_config['ConsecutiveSamplesForStateTransition'].nil? - samples_to_keep = monitor_config['ConsecutiveSamplesForStateTransition'].to_i - end - - deleted_record = {} - if @@monitor_states.key?(monitor_instance_id) - health_monitor_instance_state = @@monitor_states[monitor_instance_id] - health_monitor_records = health_monitor_instance_state.prev_records #This should be an array - - if health_monitor_records.size == samples_to_keep - deleted_record = health_monitor_records.delete_at(0) - end - health_monitor_records.push(monitor.details) - health_monitor_instance_state.prev_records = health_monitor_records - @@monitor_states[monitor_instance_id] = health_monitor_instance_state - else - # if samples_to_keep == 1, then set new state to be the health_monitor_record state, else set it as none - - old_state = HealthMonitorStates::NONE - new_state = HealthMonitorStates::NONE - if samples_to_keep == 1 - new_state = monitor.state - end - - health_monitor_instance_state = HealthMonitorInstanceState.new( - monitor.transition_date_time, - old_state, - new_state, - monitor.transition_date_time, - [monitor.details]) - - health_monitor_instance_state.should_send = true - @@monitor_states[monitor_instance_id] = health_monitor_instance_state - end - - # update old and new state based on the history and latest record. - # TODO: this is a little hairy. Simplify - - health_monitor_records = health_monitor_instance_state.prev_records - if monitor_config['ConsecutiveSamplesForStateTransition'].nil? - samples_to_check = 1 - else - samples_to_check = monitor_config['ConsecutiveSamplesForStateTransition'].to_i - end - - latest_record = health_monitor_records[health_monitor_records.size-1] #since we push new records to the end, and remove oldest records from the beginning - latest_record_state = latest_record["state"] - latest_record_time = latest_record["timestamp"] #string representation of time - - new_state = health_monitor_instance_state.new_state - prev_sent_time = health_monitor_instance_state.prev_sent_record_time - - # if the last sent state (new_state is different from latest monitor state) - if latest_record_state.downcase == new_state.downcase - time_elapsed = (Time.parse(latest_record_time) - Time.parse(prev_sent_time)) / 60 - # check if health signal has "timed out" - if time_elapsed > @@health_signal_timeout # minutes - # update record for last sent record time - health_monitor_instance_state.old_state = health_monitor_instance_state.new_state - health_monitor_instance_state.new_state = latest_record_state - health_monitor_instance_state.prev_sent_record_time = current_time - health_monitor_instance_state.should_send = true - #log.debug "After Updating Monitor State #{health_monitor_instance_state}" - set_state(monitor_instance_id, health_monitor_instance_state) - log.debug "#{monitor_instance_id} condition: signal timeout should_send #{health_monitor_instance_state.should_send} #{health_monitor_instance_state.old_state} --> #{health_monitor_instance_state.new_state}" - # check if the first record has been sent - elsif !@@first_record_sent.key?(monitor_instance_id) - @@first_record_sent[monitor_instance_id] = true - health_monitor_instance_state.should_send = true - set_state(monitor_instance_id, health_monitor_instance_state) - elsif agg_monitor_details_changed?(is_aggregate_monitor, deleted_record, health_monitor_instance_state.prev_records[0]) - health_monitor_instance_state.should_send = true - set_state(monitor_instance_id, health_monitor_instance_state) - log.debug "#{monitor_instance_id} condition: agg monitor details changed should_send #{health_monitor_instance_state.should_send}" - end - # latest state is different that last sent state - else - #if latest_record_state is none, send - if latest_record_state.downcase == HealthMonitorStates::NONE - health_monitor_instance_state.old_state = health_monitor_instance_state.new_state #initially old = new, so when state change occurs, assign old to be new, and set new to be the latest record state - health_monitor_instance_state.new_state = latest_record_state - health_monitor_instance_state.state_change_time = current_time - health_monitor_instance_state.prev_sent_record_time = current_time - health_monitor_instance_state.should_send = true - if !is_aggregate_monitor - if !telemetry.nil? - telemetry.add_monitor_to_telemetry(monitor_id, health_monitor_instance_state.old_state, health_monitor_instance_state.new_state) - end - end - if !@@first_record_sent.key?(monitor_instance_id) - @@first_record_sent[monitor_instance_id] = true - end - set_state(monitor_instance_id, health_monitor_instance_state) - log.debug "#{monitor_instance_id} condition: NONE state should_send #{health_monitor_instance_state.should_send} #{health_monitor_instance_state.old_state} --> #{health_monitor_instance_state.new_state}" - # if it is a monitor that needs to instantly notify on state change, update the state - # mark the monitor to be sent - elsif samples_to_check == 1 - health_monitor_instance_state.old_state = health_monitor_instance_state.new_state #initially old = new, so when state change occurs, assign old to be new, and set new to be the latest record state - health_monitor_instance_state.new_state = latest_record_state - health_monitor_instance_state.state_change_time = current_time - health_monitor_instance_state.prev_sent_record_time = current_time - health_monitor_instance_state.should_send = true - if !is_aggregate_monitor - if !telemetry.nil? - telemetry.add_monitor_to_telemetry(monitor_id, health_monitor_instance_state.old_state, health_monitor_instance_state.new_state) - end - end - if !@@first_record_sent.key?(monitor_instance_id) - @@first_record_sent[monitor_instance_id] = true - end - set_state(monitor_instance_id, health_monitor_instance_state) - log.debug "#{monitor_instance_id} condition: state change, samples_to_check = #{samples_to_check} should_send #{health_monitor_instance_state.should_send} #{health_monitor_instance_state.old_state} --> #{health_monitor_instance_state.new_state}" - else - # state change from previous sent state to latest record state - #check state of last n records to see if they are all in the same state - if (is_state_change_consistent(health_monitor_records, samples_to_keep)) - first_record = health_monitor_records[0] - latest_record = health_monitor_records[health_monitor_records.size-1] #since we push new records to the end, and remove oldest records from the beginning - latest_record_state = latest_record["state"] - latest_record_time = latest_record["timestamp"] #string representation of time - - health_monitor_instance_state.old_state = health_monitor_instance_state.new_state - health_monitor_instance_state.is_state_change_consistent = true # This way it wont be recomputed in the optimizer. - health_monitor_instance_state.should_send = true - health_monitor_instance_state.new_state = latest_record_state - health_monitor_instance_state.prev_sent_record_time = current_time - health_monitor_instance_state.state_change_time = current_time - if !is_aggregate_monitor - if !telemetry.nil? - telemetry.add_monitor_to_telemetry(monitor_id, health_monitor_instance_state.old_state, health_monitor_instance_state.new_state) - end - end - - set_state(monitor_instance_id, health_monitor_instance_state) - - if !@@first_record_sent.key?(monitor_instance_id) - @@first_record_sent[monitor_instance_id] = true - end - log.debug "#{monitor_instance_id} condition: consistent state change, samples_to_check = #{samples_to_check} should_send #{health_monitor_instance_state.should_send} #{health_monitor_instance_state.old_state} --> #{health_monitor_instance_state.new_state}" - end - end - end - end - - private - def is_state_change_consistent(health_monitor_records, samples_to_check) - if health_monitor_records.nil? || health_monitor_records.size == 0 || health_monitor_records.size < samples_to_check - return false - end - i = 0 - while i < health_monitor_records.size - 1 - #log.debug "Prev: #{health_monitor_records[i].state} Current: #{health_monitor_records[i + 1].state}" - if health_monitor_records[i]["state"] != health_monitor_records[i + 1]["state"] - return false - end - i += 1 - end - return true - end - - def agg_monitor_details_changed?(is_aggregate_monitor, last_sent_details, latest_details) - log = HealthMonitorHelpers.get_log_handle - if !is_aggregate_monitor - return false - end - - # Do a deep comparison of the keys under details, since a shallow comparison is hit or miss. - # Actual bug was the array inside the keys were in random order and the previous equality comparison was failing - latest_details['details'].keys.each{|k| - if !last_sent_details['details'].key?(k) - return true - end - if latest_details['details'][k].size != last_sent_details['details'][k].size - return true - end - } - # Explanation: a = [1,2] b = [2,1] a & b = [1,2] , c = [2,3] d = [2] c & d = [2] c.size != (c&d).size - latest_details['details'].keys.each{|k| - if !(latest_details['details'][k].size == (last_sent_details['details'][k] & latest_details['details'][k]).size) - return true - end - } - return false - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_monitor_telemetry.rb b/source/plugins/ruby/health/health_monitor_telemetry.rb deleted file mode 100644 index 1227e1f83..000000000 --- a/source/plugins/ruby/health/health_monitor_telemetry.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' -require 'socket' -if Socket.gethostname.start_with?('omsagent-rs') - require_relative '../ApplicationInsightsUtility' -end - - -module HealthModel - class HealthMonitorTelemetry - - attr_reader :monitor_records, :last_sent_time - @@TELEMETRY_SEND_INTERVAL = 60 - - def initialize - @last_sent_time = Time.now - @monitor_records = {} - end - - def send - if Time.now > @last_sent_time + @@TELEMETRY_SEND_INTERVAL * 60 - log = HealthMonitorHelpers.get_log_handle - log.info "Sending #{@monitor_records.size} state change events" - if @monitor_records.size > 0 - hash_to_send = {} - @monitor_records.each{|k,v| - v.each{|k1,v1| - hash_to_send["#{k}-#{k1}"] = v1 - } - } - ApplicationInsightsUtility.sendCustomEvent("HealthMonitorStateChangeEvent", hash_to_send) - end - @monitor_records = {} - @last_sent_time = Time.now - end - end - - def add_monitor_to_telemetry(monitor_id, old_state, new_state) - if @monitor_records.nil? || @monitor_records.empty? - @monitor_records = {} - end - if @monitor_records.key?(monitor_id) - monitor_hash = @monitor_records[monitor_id] - if monitor_hash.key?("#{old_state}-#{new_state}") - count = monitor_hash["#{old_state}-#{new_state}"] - count = count + 1 - monitor_hash["#{old_state}-#{new_state}"] = count - else - monitor_hash["#{old_state}-#{new_state}"] = 1 - end - @monitor_records[monitor_id] = monitor_hash - else - monitor_hash = {} - monitor_hash["#{old_state}-#{new_state}"] = 1 - @monitor_records[monitor_id] = monitor_hash - end - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_monitor_utils.rb b/source/plugins/ruby/health/health_monitor_utils.rb deleted file mode 100644 index 58f2ecc36..000000000 --- a/source/plugins/ruby/health/health_monitor_utils.rb +++ /dev/null @@ -1,323 +0,0 @@ -# frozen_string_literal: true -require 'logger' -require 'digest' -require_relative 'health_model_constants' -require 'yajl/json_gem' -require_relative '../kubelet_utils' - -module HealthModel - # static class that provides a bunch of utility methods - class HealthMonitorUtils - - begin - if !Gem.win_platform? - require_relative '../KubernetesApiClient' - end - rescue => e - $log.info "Error loading KubernetesApiClient #{e.message}" - end - - @@nodeInventory = {} - - @log_path = "/var/opt/microsoft/docker-cimprov/log/health_monitors.log" - - if Gem.win_platform? #unit testing on windows dev machine - @log_path = "C:\Temp\health_monitors.log" - end - - @log = Logger.new(@log_path, 2, 10 * 1048576) #keep last 2 files, max log file size = 10M - @@last_refresh_time = '2019-01-01T00:00:00Z' - - class << self - # compute the percentage state given a value and a monitor configuration - #TODO : Add Unit Tests for this method - def compute_percentage_state(value, config) - if config.nil? || ( config['WarnIfGreaterThanPercentage'].nil? && config['WarnIfLesserThanPercentage'].nil? ) - warn_percentage = nil - else - warn_percentage = !config['WarnIfGreaterThanPercentage'].nil? ? config['WarnIfGreaterThanPercentage'].to_f : config['WarnIfLesserThanPercentage'].to_f - end - fail_percentage = !config['FailIfGreaterThanPercentage'].nil? ? config['FailIfGreaterThanPercentage'].to_f : config['FailIfLesserThanPercentage'].to_f - is_less_than_comparer = config['FailIfGreaterThanPercentage'].nil? ? true : false # Fail percentage config always present for percentage computation monitors - - if !config.nil? && is_less_than_comparer - if value < fail_percentage - return HealthMonitorStates::FAIL - elsif !warn_percentage.nil? && value < warn_percentage - return HealthMonitorStates::WARNING - else - return HealthMonitorStates::PASS - end - else - if value > fail_percentage - return HealthMonitorStates::FAIL - elsif !warn_percentage.nil? && value > warn_percentage - return HealthMonitorStates::WARNING - else - return HealthMonitorStates::PASS - end - end - end - - def is_node_monitor(monitor_id) - return (monitor_id == MonitorId::NODE_CPU_MONITOR_ID || monitor_id == MonitorId::NODE_MEMORY_MONITOR_ID || monitor_id == MonitorId::NODE_CONDITION_MONITOR_ID) - end - - def is_pods_ready_monitor(monitor_id) - return (monitor_id == MonitorId::USER_WORKLOAD_PODS_READY_MONITOR_ID || monitor_id == MonitorId::SYSTEM_WORKLOAD_PODS_READY_MONITOR_ID) - end - - def is_cluster_health_model_enabled - enabled = ENV["AZMON_CLUSTER_ENABLE_HEALTH_MODEL"] - if !enabled.nil? && enabled.casecmp("true") == 0 - return true - else - return false - end - end - - def get_pods_ready_hash(resources) - pods_ready_percentage_hash = {} - resources.pod_inventory['items'].each do |pod| - begin - workload_name = resources.get_workload_name(pod) - namespace = pod['metadata']['namespace'] - status = pod['status']['phase'] - owner_kind = resources.get_workload_kind(pod) - if owner_kind.casecmp('job') == 0 - next - end - if pods_ready_percentage_hash.key?(workload_name) - total_pods = pods_ready_percentage_hash[workload_name]['totalPods'] - pods_ready = pods_ready_percentage_hash[workload_name]['podsReady'] - else - total_pods = 0 - pods_ready = 0 - end - - total_pods += 1 - if status == 'Running' - pods_ready += 1 - end - - pods_ready_percentage_hash[workload_name] = {'totalPods' => total_pods, 'podsReady' => pods_ready, 'namespace' => namespace, 'workload_name' => workload_name, 'kind' => owner_kind} - rescue => e - @log.info "Error when processing pod #{pod['metadata']['name']} #{e.message}" - end - end - return pods_ready_percentage_hash - end - - def get_node_state_from_node_conditions(monitor_config, node_conditions) - pass = false - warn = false - fail = false - failtypes = ['outofdisk', 'networkunavailable'].to_set #default fail types - if !monitor_config.nil? && !monitor_config["NodeConditionTypesForFailedState"].nil? - failtypes = monitor_config["NodeConditionTypesForFailedState"] - if !failtypes.nil? - failtypes = failtypes.split(',').map{|x| x.downcase}.map{|x| x.gsub(" ","")}.to_set - end - end - log = get_log_handle - #log.info "Fail Types #{failtypes.inspect}" - node_conditions.each do |condition| - type = condition['type'] - status = condition['status'] - - #for each condition in the configuration, check if the type is not false. If yes, update state to fail - if (failtypes.include?(type.downcase) && (status == 'True' || status == 'Unknown')) - fail = true - elsif ((type == "DiskPressure" || type == "MemoryPressure" || type == "PIDPressure") && (status == 'True' || status == 'Unknown')) - warn = true - elsif type == "Ready" && status == 'True' - pass = true - end - end - - if fail - return HealthMonitorStates::FAIL - elsif warn - return HealthMonitorStates::WARNING - else - return HealthMonitorStates::PASS - end - end - - def get_resource_subscription(pod_inventory, metric_name, metric_capacity) - subscription = 0.0 - if !pod_inventory.empty? - pod_inventory['items'].each do |pod| - pod['spec']['containers'].each do |container| - if !container['resources']['requests'].nil? && !container['resources']['requests'][metric_name].nil? - subscription += KubernetesApiClient.getMetricNumericValue(metric_name, container['resources']['requests'][metric_name]) - end - end - end - end - #log.debug "#{metric_name} Subscription #{subscription}" - return subscription - end - - def get_cluster_cpu_memory_capacity(log, node_inventory: nil) - begin - if node_inventory.nil? - resourceUri = KubernetesApiClient.getNodesResourceUri("nodes") - node_inventory = JSON.parse(KubernetesApiClient.getKubeResourceInfo(resourceUri).body) - end - cluster_cpu_capacity = 0.0 - cluster_memory_capacity = 0.0 - if !node_inventory.empty? - cpu_capacity_json = KubernetesApiClient.parseNodeLimits(node_inventory, "capacity", "cpu", "cpuCapacityNanoCores") - if !cpu_capacity_json.nil? - cpu_capacity_json.each do |cpu_capacity_node| - metricVal = JSON.parse(cpu_capacity_node['json_Collections'])[0]['Value'] - if !metricVal.to_s.nil? - cluster_cpu_capacity += metricVal - end - end - else - log.info "Error getting cpu_capacity" - end - memory_capacity_json = KubernetesApiClient.parseNodeLimits(node_inventory, "capacity", "memory", "memoryCapacityBytes") - if !memory_capacity_json.nil? - memory_capacity_json.each do |memory_capacity_node| - metricVal = JSON.parse(memory_capacity_node['json_Collections'])[0]['Value'] - if !metricVal.to_s.nil? - cluster_memory_capacity += metricVal - end - end - else - log.info "Error getting memory_capacity" - end - else - log.info "Unable to get cpu and memory capacity" - return [0.0, 0.0] - end - return [cluster_cpu_capacity, cluster_memory_capacity] - rescue => e - log.info e - end - end - - def refresh_kubernetes_api_data(log, hostName, force: false) - #log.debug "refresh_kubernetes_api_data" - if ( ((Time.now.utc - Time.parse(@@last_refresh_time)) / 60 ) < 5.0 && !force) - log.debug "Less than 5 minutes since last refresh at #{@@last_refresh_time}" - return - end - if force - log.debug "Force Refresh" - end - - begin - resourceUri = KubernetesApiClient.getNodesResourceUri("nodes") - @@nodeInventory = JSON.parse(KubernetesApiClient.getKubeResourceInfo(resourceUri).body) - if !hostName.nil? - podInventory = JSON.parse(KubernetesApiClient.getKubeResourceInfo("pods?fieldSelector=spec.nodeName%3D#{hostName}").body) - else - podInventory = JSON.parse(KubernetesApiClient.getKubeResourceInfo("pods").body) - end - podInventory['items'].each do |pod| - has_owner = !pod['metadata']['ownerReferences'].nil? - if !has_owner - workload_name = pod['metadata']['name'] - else - workload_name = pod['metadata']['ownerReferences'][0]['name'] - end - namespace = pod['metadata']['namespace'] - #TODO: Figure this out for container cpu/memory - #@@controllerMapping[workload_name] = namespace - #log.debug "workload_name #{workload_name} namespace #{namespace}" - pod['spec']['containers'].each do |container| - key = [pod['metadata']['uid'], container['name']].join('/') - - if !container['resources'].empty? && !container['resources']['limits'].nil? && !container['resources']['limits']['cpu'].nil? - cpu_limit_value = KubernetesApiClient.getMetricNumericValue('cpu', container['resources']['limits']['cpu']) - else - log.info "CPU limit not set for container : #{container['name']}. Using Node Capacity" - #TODO: Send warning health event #bestpractices - cpu_limit_value = @cpu_capacity - end - - if !container['resources'].empty? && !container['resources']['limits'].nil? && !container['resources']['limits']['memory'].nil? - #log.info "Raw Memory Value #{container['resources']['limits']['memory']}" - memory_limit_value = KubernetesApiClient.getMetricNumericValue('memory', container['resources']['limits']['memory']) - else - log.info "Memory limit not set for container : #{container['name']}. Using Node Capacity" - memory_limit_value = @memory_capacity - end - - #TODO: Figure this out for container cpu/memory - #@@containerMetadata[key] = {"cpuLimit" => cpu_limit_value, "memoryLimit" => memory_limit_value, "controllerName" => workload_name, "namespace" => namespace} - end - end - rescue => e - log.info "Error Refreshing Container Resource Limits #{e.backtrace}" - end - # log.info "Controller Mapping #{@@controllerMapping}" - # log.info "Node Inventory #{@@nodeInventory}" - # log.info "Container Metadata #{@@containerMetadata}" - # log.info "------------------------------------" - @@last_refresh_time = Time.now.utc.iso8601 - end - - def get_monitor_instance_id(monitor_id, args = []) - string_to_hash = args.join("/") - return "#{monitor_id}-#{Digest::MD5.hexdigest(string_to_hash)}" - end - - def ensure_cpu_memory_capacity_set(log, cpu_capacity, memory_capacity, hostname) - log.info "ensure_cpu_memory_capacity_set cpu_capacity #{cpu_capacity} memory_capacity #{memory_capacity}" - if cpu_capacity != 1.0 && memory_capacity != 1.0 - log.info "CPU And Memory Capacity are already set" - return [cpu_capacity, memory_capacity] - end - log.info "CPU and Memory Capacity Not set" - return KubeletUtils.get_node_capacity - end - - def build_metrics_hash(metrics_to_collect) - metrics_to_collect_arr = metrics_to_collect.split(',').map(&:strip) - metrics_hash = metrics_to_collect_arr.map {|x| [x.downcase,true]}.to_h - return metrics_hash - end - - def get_health_monitor_config - health_monitor_config = {} - begin - file = File.open('/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json', "r") - if !file.nil? - fileContents = file.read - health_monitor_config = JSON.parse(fileContents) - file.close - end - rescue => e - log.info "Error when opening health config file #{e}" - end - return health_monitor_config - end - - def get_cluster_labels - labels = {} - cluster_id = KubernetesApiClient.getClusterId - region = KubernetesApiClient.getClusterRegion - labels['container.azm.ms/cluster-region'] = region - if !cluster_id.nil? - cluster_id_elements = cluster_id.split('/') - azure_sub_id = cluster_id_elements[2] - resource_group = cluster_id_elements[4] - cluster_name = cluster_id_elements[8] - labels['container.azm.ms/cluster-subscription-id'] = azure_sub_id - labels['container.azm.ms/cluster-resource-group'] = resource_group - labels['container.azm.ms/cluster-name'] = cluster_name - end - return labels - end - - def get_log_handle - return @log - end - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/health_signal_reducer.rb b/source/plugins/ruby/health/health_signal_reducer.rb deleted file mode 100644 index 4708c4ee5..000000000 --- a/source/plugins/ruby/health/health_signal_reducer.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' - -module HealthModel - # this class - # 1. dedupes daemonset signals and takes only the latest - # 2. removes signals for objects that are no longer in the inventory e.g. node might have sent signal before being scaled down - class HealthSignalReducer - def initialize - - end - - def reduce_signals(health_monitor_records, health_k8s_inventory) - nodes = health_k8s_inventory.get_nodes - workload_names = health_k8s_inventory.get_workload_names - reduced_signals_map = {} - reduced_signals = [] - health_monitor_records.each{|health_monitor_record| - monitor_instance_id = health_monitor_record.monitor_instance_id - monitor_id = health_monitor_record.monitor_id - if reduced_signals_map.key?(monitor_instance_id) - record = reduced_signals_map[monitor_instance_id] - if health_monitor_record.transition_date_time > record.transition_date_time # always take the latest record for a monitor instance id - reduced_signals_map[monitor_instance_id] = health_monitor_record - end - elsif HealthMonitorHelpers.is_node_monitor(monitor_id) - node_name = health_monitor_record.labels['kubernetes.io/hostname'] - if (node_name.nil? || !nodes.include?(node_name)) # only add daemon set records if node is present in the inventory - next - end - reduced_signals_map[monitor_instance_id] = health_monitor_record - elsif HealthMonitorHelpers.is_pods_ready_monitor(monitor_id) - workload_name = health_monitor_record.labels[HealthMonitorLabels::WORKLOAD_NAME] - namespace = health_monitor_record.labels[HealthMonitorLabels::NAMESPACE] - lookup = "#{namespace}~~#{workload_name}" - if (workload_name.nil? || !workload_names.include?(lookup)) #only add pod record if present in the inventory - next - end - reduced_signals_map[monitor_instance_id] = health_monitor_record - else - reduced_signals_map[monitor_instance_id] = health_monitor_record - end - } - - reduced_signals_map.each{|k,v| - reduced_signals.push(v) - } - - return reduced_signals - end - - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/monitor_factory.rb b/source/plugins/ruby/health/monitor_factory.rb deleted file mode 100644 index 1e4f6f5b8..000000000 --- a/source/plugins/ruby/health/monitor_factory.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true -require_relative 'aggregate_monitor' -require_relative 'unit_monitor' - -module HealthModel - class MonitorFactory - - def initialize - - end - - def create_unit_monitor(monitor_record) - return UnitMonitor.new(monitor_record.monitor_id, - monitor_record.monitor_instance_id, - monitor_record.state, - monitor_record.transition_date_time, - monitor_record.labels, - monitor_record.config, - monitor_record.details) - end - - def create_aggregate_monitor(monitor_id, monitor_instance_id, labels, aggregation_algorithm, aggregation_algorithm_params, child_monitor) - return AggregateMonitor.new(monitor_id, - monitor_instance_id, - child_monitor.state, - child_monitor.transition_date_time, - aggregation_algorithm, - aggregation_algorithm_params, - labels) - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/monitor_set.rb b/source/plugins/ruby/health/monitor_set.rb deleted file mode 100644 index 8d5994419..000000000 --- a/source/plugins/ruby/health/monitor_set.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module HealthModel - class MonitorSet - attr_accessor :monitors - - #constructor - def initialize - @monitors = {} - end - - # checks if the monitor is present in the set - def contains?(monitor_instance_id) - @monitors.key?(monitor_instance_id) - end - - # adds or updates the monitor - def add_or_update(monitor) - @monitors[monitor.monitor_instance_id] = monitor - end - - # gets the monitor given the monitor instance id - def get_monitor(monitor_instance_id) - @monitors[monitor_instance_id] if @monitors.key?(monitor_instance_id) - end - - # deletes a monitor from the set - def delete(monitor_instance_id) - if @monitors.key?(monitor_instance_id) - @monitors.delete(monitor_instance_id) - end - end - - # gets the size of the monitor set - def get_size - @monitors.length - end - - # gets the map of monitor instance id to monitors - def get_map - @monitors - end - end -end diff --git a/source/plugins/ruby/health/node_monitor_hierarchy_reducer.rb b/source/plugins/ruby/health/node_monitor_hierarchy_reducer.rb deleted file mode 100644 index 0bad4517e..000000000 --- a/source/plugins/ruby/health/node_monitor_hierarchy_reducer.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' - -module HealthModel - class NodeMonitorHierarchyReducer - def initialize - end - - # Finalizes the Node Hierarchy. This removes node pools and node pool set from the hierarchy if they are not present. - def finalize(monitor_set) - monitors_to_reduce = [MonitorId::ALL_AGENT_NODE_POOLS, MonitorId::ALL_NODES] - # for the above monitors, which are constant per cluster, the monitor_id and monitor_instance_id are the same - monitors_to_reduce.each do |monitor_to_reduce| - monitor = monitor_set.get_monitor(monitor_to_reduce) - if !monitor.nil? - if monitor.is_aggregate_monitor && monitor.get_member_monitors.size == 1 - #copy the children of member monitor as children of parent - member_monitor_instance_id = monitor.get_member_monitors[0] #gets the only member monitor instance id - member_monitor = monitor_set.get_monitor(member_monitor_instance_id) - #reduce only if the aggregation algorithms are the same - if !member_monitor.aggregation_algorithm.nil? && member_monitor.aggregation_algorithm == AggregationAlgorithm::WORSTOF && monitor.aggregation_algorithm == member_monitor.aggregation_algorithm - member_monitor.get_member_monitors.each{|grandchild_monitor| - monitor.add_member_monitor(grandchild_monitor) - } - monitor.remove_member_monitor(member_monitor_instance_id) - # delete the member monitor from the monitor_set - monitor_set.delete(member_monitor_instance_id) - end - end - end - end - end - end -end diff --git a/source/plugins/ruby/health/parent_monitor_provider.rb b/source/plugins/ruby/health/parent_monitor_provider.rb deleted file mode 100644 index e5766ea1b..000000000 --- a/source/plugins/ruby/health/parent_monitor_provider.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' -module HealthModel - class ParentMonitorProvider - - attr_reader :health_model_definition, :parent_monitor_mapping, :parent_monitor_instance_mapping - - def initialize(definition) - @health_model_definition = definition - @parent_monitor_mapping = {} #monitorId --> parent_monitor_id mapping - @parent_monitor_instance_mapping = {} #child monitor id -- > parent monitor instance mapping. Used in instances when the node no longer exists and impossible to compute from kube api results - @log = HealthMonitorHelpers.get_log_handle - end - - # gets the parent monitor id given the state transition. It requires the monitor id and labels to determine the parent id - def get_parent_monitor_id(monitor) - monitor_id = monitor.monitor_id - - # cache the parent monitor id so it is not recomputed every time - if @parent_monitor_mapping.key?(monitor.monitor_instance_id) - return @parent_monitor_mapping[monitor.monitor_instance_id] - end - - if @health_model_definition.key?(monitor_id) - parent_monitor_id = @health_model_definition[monitor_id]['parent_monitor_id'] - # check parent_monitor_id is an array, then evaluate the conditions, else return the parent_monitor_id - if parent_monitor_id.is_a?(String) - @parent_monitor_mapping[monitor.monitor_instance_id] = parent_monitor_id - return parent_monitor_id - end - if parent_monitor_id.nil? - conditions = @health_model_definition[monitor_id]['conditions'] - if !conditions.nil? && conditions.is_a?(Array) - labels = monitor.labels - conditions.each{|condition| - left = "#{labels[condition['key']]}" - op = "#{condition['operator']}" - right = "#{condition['value']}" - cond = left.send(op.to_sym, right) - if cond - @parent_monitor_mapping[monitor.monitor_instance_id] = condition['parent_id'] - return condition['parent_id'] - end - } - end - return @health_model_definition[monitor_id]['default_parent_monitor_id'] - end - else - raise "Invalid Monitor Id #{monitor_id} in get_parent_monitor_id" - end - end - - def get_parent_monitor_labels(monitor_id, monitor_labels, parent_monitor_id) - labels_to_copy = @health_model_definition[monitor_id]['labels'] - if labels_to_copy.nil? - return {} - end - parent_monitor_labels = {} - labels_to_copy.each{|label| - parent_monitor_labels[label] = monitor_labels[label] - } - return parent_monitor_labels - end - - def get_parent_monitor_config(parent_monitor_id) - return @health_model_definition[parent_monitor_id] - end - - def get_parent_monitor_instance_id(monitor_instance_id, parent_monitor_id, parent_monitor_labels) - if @parent_monitor_instance_mapping.key?(monitor_instance_id) - return @parent_monitor_instance_mapping[monitor_instance_id] - end - - labels = AggregateMonitorInstanceIdLabels.get_labels_for(parent_monitor_id) - if !labels.is_a?(Array) - raise "Expected #{labels} to be an Array for #{parent_monitor_id}" - end - values = labels.map{|label| parent_monitor_labels[label]} - if values.nil? || values.empty? || values.size == 0 - @parent_monitor_instance_mapping[monitor_instance_id] = parent_monitor_id - return parent_monitor_id - end - parent_monitor_instance_id = "#{parent_monitor_id}-#{values.join('-')}" - @parent_monitor_instance_mapping[monitor_instance_id] = parent_monitor_instance_id - @log.info "parent_monitor_instance_id for #{monitor_instance_id} => #{parent_monitor_instance_id}" - return parent_monitor_instance_id - end - end -end \ No newline at end of file diff --git a/source/plugins/ruby/health/unit_monitor.rb b/source/plugins/ruby/health/unit_monitor.rb deleted file mode 100644 index 8e2de210b..000000000 --- a/source/plugins/ruby/health/unit_monitor.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true -require_relative 'health_model_constants' -require 'yajl/json_gem' - -module HealthModel - class UnitMonitor - - attr_accessor :monitor_id, :monitor_instance_id, :state, :transition_date_time, :labels, :config, :details, :is_aggregate_monitor - - # constructor - def initialize(monitor_id, monitor_instance_id, state, transition_date_time, labels, config, details) - @monitor_id = monitor_id - @monitor_instance_id = monitor_instance_id - @transition_date_time = transition_date_time - @state = state - @labels = labels - @config = config - @details = details - @is_aggregate_monitor = false - end - - def get_member_monitors - return nil - end - - end -end \ No newline at end of file diff --git a/source/plugins/ruby/in_cadvisor_perf.rb b/source/plugins/ruby/in_cadvisor_perf.rb index aba24ecc2..901ecefab 100644 --- a/source/plugins/ruby/in_cadvisor_perf.rb +++ b/source/plugins/ruby/in_cadvisor_perf.rb @@ -27,8 +27,6 @@ def initialize config_param :run_interval, :time, :default => 60 config_param :tag, :string, :default => "oneagent.containerInsights.LINUX_PERF_BLOB" config_param :mdmtag, :string, :default => "mdm.cadvisorperf" - config_param :nodehealthtag, :string, :default => "kubehealth.DaemonSet.Node" - config_param :containerhealthtag, :string, :default => "kubehealth.DaemonSet.Container" config_param :insightsmetricstag, :string, :default => "oneagent.containerInsights.INSIGHTS_METRICS_BLOB" def configure(conf) @@ -82,8 +80,6 @@ def enumerate() end router.emit_stream(@tag, eventStream) if eventStream router.emit_stream(@mdmtag, eventStream) if eventStream - router.emit_stream(@containerhealthtag, eventStream) if eventStream - router.emit_stream(@nodehealthtag, eventStream) if eventStream if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0 && eventStream.count > 0) $log.info("cAdvisorPerfEmitStreamSuccess @ #{Time.now.utc.iso8601}") diff --git a/source/plugins/ruby/in_kube_health.rb b/source/plugins/ruby/in_kube_health.rb deleted file mode 100644 index db981c53e..000000000 --- a/source/plugins/ruby/in_kube_health.rb +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/local/bin/ruby -# frozen_string_literal: true - -require 'fluent/plugin/input' - -require_relative "KubernetesApiClient" -require_relative "oms_common" -require_relative "omslog" -require_relative "ApplicationInsightsUtility" - -module Fluent::Plugin - Dir[File.join(__dir__, "./health", "*.rb")].each { |file| require file } - - class KubeHealthInput < Input - include HealthModel - Fluent::Plugin.register_input("kube_health", self) - - config_param :health_monitor_config_path, :default => "/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json" - - @@clusterCpuCapacity = 0.0 - @@clusterMemoryCapacity = 0.0 - @@cluster_health_model_enabled = HealthMonitorUtils.is_cluster_health_model_enabled - - def initialize - begin - super - require "yaml" - require "yajl/json_gem" - require "yajl" - require "time" - - @@cluster_id = KubernetesApiClient.getClusterId - @resources = HealthKubernetesResources.instance - @provider = HealthMonitorProvider.new(@@cluster_id, HealthMonitorUtils.get_cluster_labels, @resources, @health_monitor_config_path) - @@ApiGroupApps = "apps" - @@KubeInfraNamespace = "kube-system" - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - config_param :run_interval, :time, :default => 60 - config_param :tag, :string, :default => "kubehealth.ReplicaSet" - - def configure(conf) - super - end - - def start - begin - super - if @run_interval - @finished = false - @condition = ConditionVariable.new - @mutex = Mutex.new - @thread = Thread.new(&method(:run_periodic)) - - @@hmlog = HealthMonitorUtils.get_log_handle - @@clusterName = KubernetesApiClient.getClusterName - @@clusterRegion = KubernetesApiClient.getClusterRegion - cluster_capacity = HealthMonitorUtils.get_cluster_cpu_memory_capacity(@@hmlog) - @@clusterCpuCapacity = cluster_capacity[0] - @@clusterMemoryCapacity = cluster_capacity[1] - @@hmlog.info "Cluster CPU Capacity: #{@@clusterCpuCapacity} Memory Capacity: #{@@clusterMemoryCapacity}" - initialize_inventory - if @@cluster_health_model_enabled - ApplicationInsightsUtility.sendCustomEvent("in_kube_health Plugin Start", {}) - end - end - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) - end - end - - def shutdown - if @run_interval - @mutex.synchronize { - @finished = true - @condition.signal - } - @thread.join - super # This super must be at the end of shutdown method - end - end - - def enumerate - if !@@cluster_health_model_enabled - @@hmlog.info "Cluster Health Model disabled in in_kube_health" - return Fluent::MultiEventStream.new - end - begin - currentTime = Time.now - emitTime = Fluent::Engine.now - batchTime = currentTime.utc.iso8601 - health_monitor_records = [] - eventStream = Fluent::MultiEventStream.new - - #HealthMonitorUtils.refresh_kubernetes_api_data(@@hmlog, nil) - # we do this so that if the call fails, we get a response code/header etc. - resourceUri = KubernetesApiClient.getNodesResourceUri("nodes") - node_inventory_response = KubernetesApiClient.getKubeResourceInfo(resourceUri) - if !node_inventory_response.nil? && !node_inventory_response.body.nil? - node_inventory = Yajl::Parser.parse(StringIO.new(node_inventory_response.body)) - @resources.node_inventory = node_inventory - end - - pod_inventory_response = KubernetesApiClient.getKubeResourceInfo("pods?fieldSelector=metadata.namespace%3D#{@@KubeInfraNamespace}") - if !pod_inventory_response.nil? && !pod_inventory_response.body.nil? - pod_inventory = Yajl::Parser.parse(StringIO.new(pod_inventory_response.body)) - @resources.pod_inventory = pod_inventory - @resources.build_pod_uid_lookup - end - - replicaset_inventory_response = KubernetesApiClient.getKubeResourceInfo("replicasets?fieldSelector=metadata.namespace%3D#{@@KubeInfraNamespace}", api_group: @@ApiGroupApps) - if !replicaset_inventory_response.nil? && !replicaset_inventory_response.body.nil? - replicaset_inventory = Yajl::Parser.parse(StringIO.new(replicaset_inventory_response.body)) - @resources.set_replicaset_inventory(replicaset_inventory) - end - - if !node_inventory_response.nil? && !node_inventory_response.code.nil? - if node_inventory_response.code.to_i != 200 - record = process_kube_api_up_monitor("fail", node_inventory_response) - health_monitor_records.push(record) if record - else - record = process_kube_api_up_monitor("pass", node_inventory_response) - health_monitor_records.push(record) if record - end - end - - if !pod_inventory.nil? - record = process_cpu_oversubscribed_monitor(pod_inventory, node_inventory) - health_monitor_records.push(record) if record - record = process_memory_oversubscribed_monitor(pod_inventory, node_inventory) - health_monitor_records.push(record) if record - pods_ready_hash = HealthMonitorUtils.get_pods_ready_hash(@resources) - - system_pods = pods_ready_hash.keep_if { |k, v| v["namespace"] == @@KubeInfraNamespace } - workload_pods = Hash.new # pods_ready_hash.select{ |k, v| v["namespace"] != @@KubeInfraNamespace } - - system_pods_ready_percentage_records = process_pods_ready_percentage(system_pods, MonitorId::SYSTEM_WORKLOAD_PODS_READY_MONITOR_ID) - system_pods_ready_percentage_records.each do |record| - health_monitor_records.push(record) if record - end - - workload_pods_ready_percentage_records = process_pods_ready_percentage(workload_pods, MonitorId::USER_WORKLOAD_PODS_READY_MONITOR_ID) - workload_pods_ready_percentage_records.each do |record| - health_monitor_records.push(record) if record - end - else - @@hmlog.info "POD INVENTORY IS NIL" - end - - if !node_inventory.nil? - node_condition_records = process_node_condition_monitor(node_inventory) - node_condition_records.each do |record| - health_monitor_records.push(record) if record - end - else - @@hmlog.info "NODE INVENTORY IS NIL" - end - - health_monitor_records.each do |record| - eventStream.add(emitTime, record) - end - router.emit_stream(@tag, eventStream) if eventStream - rescue => errorStr - @@hmlog.warn("error in_kube_health: #{errorStr.to_s}") - @@hmlog.debug "backtrace Input #{errorStr.backtrace}" - ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) - end - end - - def process_cpu_oversubscribed_monitor(pod_inventory, node_inventory) - timestamp = Time.now.utc.iso8601 - @@clusterCpuCapacity = HealthMonitorUtils.get_cluster_cpu_memory_capacity(@@hmlog, node_inventory: node_inventory)[0] - subscription = HealthMonitorUtils.get_resource_subscription(pod_inventory, "cpu", @@clusterCpuCapacity) - @@hmlog.info "Refreshed Cluster CPU Capacity #{@@clusterCpuCapacity}" - state = subscription > @@clusterCpuCapacity ? "fail" : "pass" - - #CPU - monitor_id = MonitorId::WORKLOAD_CPU_OVERSUBSCRIBED_MONITOR_ID - health_monitor_record = {"timestamp" => timestamp, "state" => state, "details" => {"clusterCpuCapacity" => @@clusterCpuCapacity / 1000000.to_f, "clusterCpuRequests" => subscription / 1000000.to_f}} - # @@hmlog.info health_monitor_record - - monitor_instance_id = HealthMonitorUtils.get_monitor_instance_id(monitor_id, [@@cluster_id]) - #hmlog.info "Monitor Instance Id: #{monitor_instance_id}" - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::CLUSTER_ID] = @@cluster_id - #@@hmlog.info "Successfully processed process_cpu_oversubscribed_monitor" - return health_record - end - - def process_memory_oversubscribed_monitor(pod_inventory, node_inventory) - timestamp = Time.now.utc.iso8601 - @@clusterMemoryCapacity = HealthMonitorUtils.get_cluster_cpu_memory_capacity(@@hmlog, node_inventory: node_inventory)[1] - @@hmlog.info "Refreshed Cluster Memory Capacity #{@@clusterMemoryCapacity}" - subscription = HealthMonitorUtils.get_resource_subscription(pod_inventory, "memory", @@clusterMemoryCapacity) - state = subscription > @@clusterMemoryCapacity ? "fail" : "pass" - #@@hmlog.debug "Memory Oversubscribed Monitor State : #{state}" - - #CPU - monitor_id = MonitorId::WORKLOAD_MEMORY_OVERSUBSCRIBED_MONITOR_ID - health_monitor_record = {"timestamp" => timestamp, "state" => state, "details" => {"clusterMemoryCapacity" => @@clusterMemoryCapacity.to_f, "clusterMemoryRequests" => subscription.to_f}} - hmlog = HealthMonitorUtils.get_log_handle - - monitor_instance_id = HealthMonitorUtils.get_monitor_instance_id(monitor_id, [@@cluster_id]) - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::CLUSTER_ID] = @@cluster_id - #@@hmlog.info "Successfully processed process_memory_oversubscribed_monitor" - return health_record - end - - def process_kube_api_up_monitor(state, response) - timestamp = Time.now.utc.iso8601 - - monitor_id = MonitorId::KUBE_API_STATUS - details = response.each_header.to_h - details["ResponseCode"] = response.code - health_monitor_record = {"timestamp" => timestamp, "state" => state, "details" => details} - hmlog = HealthMonitorUtils.get_log_handle - #hmlog.info health_monitor_record - - monitor_instance_id = MonitorId::KUBE_API_STATUS - #hmlog.info "Monitor Instance Id: #{monitor_instance_id}" - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::CLUSTER_ID] = @@cluster_id - #@@hmlog.info "Successfully processed process_kube_api_up_monitor" - return health_record - end - - def process_pods_ready_percentage(pods_hash, config_monitor_id) - monitor_config = @provider.get_config(config_monitor_id) - hmlog = HealthMonitorUtils.get_log_handle - - records = [] - pods_hash.keys.each do |key| - workload_name = key - total_pods = pods_hash[workload_name]["totalPods"] - pods_ready = pods_hash[workload_name]["podsReady"] - namespace = pods_hash[workload_name]["namespace"] - workload_kind = pods_hash[workload_name]["kind"] - percent = pods_ready / total_pods * 100 - timestamp = Time.now.utc.iso8601 - - state = HealthMonitorUtils.compute_percentage_state(percent, monitor_config) - health_monitor_record = {"timestamp" => timestamp, "state" => state, "details" => {"totalPods" => total_pods, "podsReady" => pods_ready, "workload_name" => workload_name, "namespace" => namespace, "workload_kind" => workload_kind}} - monitor_instance_id = HealthMonitorUtils.get_monitor_instance_id(config_monitor_id, [@@cluster_id, namespace, workload_name]) - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = config_monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::CLUSTER_ID] = @@cluster_id - records.push(health_record) - end - #@@hmlog.info "Successfully processed pods_ready_percentage for #{config_monitor_id} #{records.size}" - return records - end - - def process_node_condition_monitor(node_inventory) - monitor_id = MonitorId::NODE_CONDITION_MONITOR_ID - timestamp = Time.now.utc.iso8601 - monitor_config = @provider.get_config(monitor_id) - node_condition_monitor_records = [] - if !node_inventory.nil? - node_inventory["items"].each do |node| - node_name = node["metadata"]["name"] - conditions = node["status"]["conditions"] - node_state = HealthMonitorUtils.get_node_state_from_node_conditions(monitor_config, conditions) - details = {} - conditions.each do |condition| - condition_state = HealthMonitorStates::PASS - if condition["type"].downcase != "ready" - if (condition["status"].downcase == "true" || condition["status"].downcase == "unknown") - condition_state = HealthMonitorStates::FAIL - end - else #Condition == READY - if condition["status"].downcase != "true" - condition_state = HealthMonitorStates::FAIL - end - end - details[condition["type"]] = {"Reason" => condition["reason"], "Message" => condition["message"], "State" => condition_state} - end - health_monitor_record = {"timestamp" => timestamp, "state" => node_state, "details" => details} - monitor_instance_id = HealthMonitorUtils.get_monitor_instance_id(monitor_id, [@@cluster_id, node_name]) - health_record = {} - time_now = Time.now.utc.iso8601 - health_record[HealthMonitorRecordFields::MONITOR_ID] = monitor_id - health_record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] = monitor_instance_id - health_record[HealthMonitorRecordFields::DETAILS] = health_monitor_record - health_record[HealthMonitorRecordFields::TIME_GENERATED] = time_now - health_record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED] = time_now - health_record[HealthMonitorRecordFields::CLUSTER_ID] = @@cluster_id - health_record[HealthMonitorRecordFields::NODE_NAME] = node_name - node_condition_monitor_records.push(health_record) - end - end - #@@hmlog.info "Successfully processed process_node_condition_monitor #{node_condition_monitor_records.size}" - return node_condition_monitor_records - end - - def initialize_inventory - #this is required because there are other components, like the container cpu memory aggregator, that depends on the mapping being initialized - resourceUri = KubernetesApiClient.getNodesResourceUri("nodes") - node_inventory_response = KubernetesApiClient.getKubeResourceInfo(resourceUri) - node_inventory = Yajl::Parser.parse(StringIO.new(node_inventory_response.body)) - pod_inventory_response = KubernetesApiClient.getKubeResourceInfo("pods?fieldSelector=metadata.namespace%3D#{@@KubeInfraNamespace}") - pod_inventory = Yajl::Parser.parse(StringIO.new(pod_inventory_response.body)) - replicaset_inventory_response = KubernetesApiClient.getKubeResourceInfo("replicasets?fieldSelector=metadata.namespace%3D#{@@KubeInfraNamespace}", api_group: @@ApiGroupApps) - replicaset_inventory = Yajl::Parser.parse(StringIO.new(replicaset_inventory_response.body)) - - @resources.node_inventory = node_inventory - @resources.pod_inventory = pod_inventory - @resources.set_replicaset_inventory(replicaset_inventory) - @resources.build_pod_uid_lookup - end - - def run_periodic - @mutex.lock - done = @finished - @nextTimeToRun = Time.now - @waitTimeout = @run_interval - until done - @nextTimeToRun = @nextTimeToRun + @run_interval - @now = Time.now - if @nextTimeToRun <= @now - @waitTimeout = 1 - @nextTimeToRun = @now - else - @waitTimeout = @nextTimeToRun - @now - end - @condition.wait(@mutex, @waitTimeout) - done = @finished - @mutex.unlock - if !done - begin - @@hmlog.info("in_kube_health::run_periodic.enumerate.start @ #{Time.now.utc.iso8601}") - enumerate - @@hmlog.info("in_kube_health::run_periodic.enumerate.end @ #{Time.now.utc.iso8601}") - rescue => errorStr - @@hmlog.warn "in_kube_health::run_periodic: enumerate Failed for kubeapi sourced data health: #{errorStr}" - ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) - end - end - @mutex.lock - end - @mutex.unlock - end - end -end diff --git a/source/plugins/ruby/out_health_forward.rb b/source/plugins/ruby/out_health_forward.rb deleted file mode 100644 index 59eed97da..000000000 --- a/source/plugins/ruby/out_health_forward.rb +++ /dev/null @@ -1,838 +0,0 @@ -# frozen_string_literal: true -# -# Fluentd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'fluent/output' -require 'fluent/config/error' -require 'fluent/clock' -require 'fluent/tls' -require 'base64' -require 'forwardable' - -require 'fluent/compat/socket_util' -require 'fluent/plugin/out_forward/handshake_protocol' -require 'fluent/plugin/out_forward/load_balancer' -require 'fluent/plugin/out_forward/socket_cache' -require 'fluent/plugin/out_forward/failure_detector' -require 'fluent/plugin/out_forward/error' -require 'fluent/plugin/out_forward/connection_manager' -require 'fluent/plugin/out_forward/ack_handler' - -module Fluent::Plugin - class HealthForwardOutput < Output - Fluent::Plugin.register_output('health_forward', self) - - helpers :socket, :server, :timer, :thread, :compat_parameters, :service_discovery - - LISTEN_PORT = 25227 - - desc 'The transport protocol.' - config_param :transport, :enum, list: [:tcp, :tls], default: :tcp - # TODO: TLS session cache/tickets - - desc 'The timeout time when sending event logs.' - config_param :send_timeout, :time, default: 60 - desc 'The timeout time for socket connect' - config_param :connect_timeout, :time, default: nil - # TODO: add linger_timeout, recv_timeout - - desc 'The protocol to use for heartbeats (default is the same with "transport").' - config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport - desc 'The interval of the heartbeat packer.' - config_param :heartbeat_interval, :time, default: 1 - desc 'The wait time before accepting a server fault recovery.' - config_param :recover_wait, :time, default: 10 - desc 'The hard timeout used to detect server failure.' - config_param :hard_timeout, :time, default: 60 - desc 'The threshold parameter used to detect server faults.' - config_param :phi_threshold, :integer, default: 16 - desc 'Use the "Phi accrual failure detector" to detect server failure.' - config_param :phi_failure_detector, :bool, default: true - - desc 'Change the protocol to at-least-once.' - config_param :require_ack_response, :bool, default: false # require in_forward to respond with ack - - ## The reason of default value of :ack_response_timeout: - # Linux default tcp_syn_retries is 5 (in many environment) - # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec) - desc 'This option is used when require_ack_response is true.' - config_param :ack_response_timeout, :time, default: 190 - - desc 'The interval while reading data from server' - config_param :read_interval_msec, :integer, default: 50 # 50ms - desc 'Reading data size from server' - config_param :read_length, :size, default: 512 # 512bytes - - desc 'Set TTL to expire DNS cache in seconds.' - config_param :expire_dns_cache, :time, default: nil # 0 means disable cache - desc 'Enable client-side DNS round robin.' - config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this - - desc 'Ignore DNS resolution and errors at startup time.' - config_param :ignore_network_errors_at_startup, :bool, default: false - - desc 'Verify that a connection can be made with one of out_forward nodes at the time of startup.' - config_param :verify_connection_at_startup, :bool, default: false - - desc 'Compress buffered data.' - config_param :compress, :enum, list: [:text, :gzip], default: :text - - desc 'The default version of TLS transport.' - config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION - desc 'The cipher configuration of TLS transport.' - config_param :tls_ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT - desc 'Skip all verification of certificates or not.' - config_param :tls_insecure_mode, :bool, default: false - desc 'Allow self signed certificates or not.' - config_param :tls_allow_self_signed_cert, :bool, default: false - desc 'Verify hostname of servers and certificates or not in TLS transport.' - config_param :tls_verify_hostname, :bool, default: true - desc 'The additional CA certificate path for TLS.' - config_param :tls_ca_cert_path, :array, value_type: :string, default: nil - desc 'The additional certificate path for TLS.' - config_param :tls_cert_path, :array, value_type: :string, default: nil - desc 'The client certificate path for TLS.' - config_param :tls_client_cert_path, :string, default: nil - desc 'The client private key path for TLS.' - config_param :tls_client_private_key_path, :string, default: nil - desc 'The client private key passphrase for TLS.' - config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true - desc 'The certificate thumbprint for searching from Windows system certstore.' - config_param :tls_cert_thumbprint, :string, default: nil, secret: true - desc 'The certificate logical store name on Windows system certstore.' - config_param :tls_cert_logical_store_name, :string, default: nil - desc 'Enable to use certificate enterprise store on Windows system certstore.' - config_param :tls_cert_use_enterprise_store, :bool, default: true - desc "Enable keepalive connection." - config_param :keepalive, :bool, default: false - desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible" - config_param :keepalive_timeout, :time, default: nil - - config_section :security, required: false, multi: false do - desc 'The hostname' - config_param :self_hostname, :string - desc 'Shared key for authentication' - config_param :shared_key, :string, secret: true - end - - config_section :server, param_name: :servers do - desc "The IP address or host name of the server." - config_param :host, :string - desc "The name of the server. Used for logging and certificate verification in TLS transport (when host is address)." - config_param :name, :string, default: nil - desc "The port number of the host." - config_param :port, :integer, default: LISTEN_PORT - desc "The shared key per server." - config_param :shared_key, :string, default: nil, secret: true - desc "The username for authentication." - config_param :username, :string, default: '' - desc "The password for authentication." - config_param :password, :string, default: '', secret: true - desc "Marks a node as the standby node for an Active-Standby model between Fluentd nodes." - config_param :standby, :bool, default: false - desc "The load balancing weight." - config_param :weight, :integer, default: 60 - end - - attr_reader :nodes - - config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User section instead." - config_param :host, :string, default: nil, obsoleted: "Use section instead." - - config_section :buffer do - config_set_default :chunk_keys, ["tag"] - end - - attr_reader :read_interval, :recover_sample_size - - def initialize - super - - @nodes = [] #=> [Node] - @loop = nil - @thread = nil - - @usock = nil - @keep_alive_watcher_interval = 5 # TODO - @suspend_flush = false - end - - def configure(conf) - compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag') - - super - - unless @chunk_key_tag - raise Fluent::ConfigError, "buffer chunk key must include 'tag' for forward output" - end - - @read_interval = @read_interval_msec / 1000.0 - @recover_sample_size = @recover_wait / @heartbeat_interval - - if @heartbeat_type == :tcp - log.warn "'heartbeat_type tcp' is deprecated. use 'transport' instead." - @heartbeat_type = :transport - end - - if @dns_round_robin && @heartbeat_type == :udp - raise Fluent::ConfigError, "forward output heartbeat type must be 'transport' or 'none' to use dns_round_robin option" - end - - if @transport == :tls - # socket helper adds CA cert or signed certificate to same cert store internally so unify it in this place. - if @tls_cert_path && !@tls_cert_path.empty? - @tls_ca_cert_path = @tls_cert_path - end - if @tls_ca_cert_path && !@tls_ca_cert_path.empty? - @tls_ca_cert_path.each do |path| - raise Fluent::ConfigError, "specified cert path does not exist:#{path}" unless File.exist?(path) - raise Fluent::ConfigError, "specified cert path is not readable:#{path}" unless File.readable?(path) - end - end - - if @tls_insecure_mode - log.warn "TLS transport is configured in insecure way" - @tls_verify_hostname = false - @tls_allow_self_signed_cert = true - end - - if Fluent.windows? - if (@tls_cert_path || @tls_ca_cert_path) && @tls_cert_logical_store_name - raise Fluent::ConfigError, "specified both cert path and tls_cert_logical_store_name is not permitted" - end - else - raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_logical_store_name - raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_thumbprint - end - end - - @ack_handler = @require_ack_response ? AckHandler.new(timeout: @ack_response_timeout, log: @log, read_length: @read_length) : nil - socket_cache = @keepalive ? SocketCache.new(@keepalive_timeout, @log) : nil - @connection_manager = Fluent::Plugin::ForwardOutput::ConnectionManager.new( - log: @log, - secure: !!@security, - connection_factory: method(:create_transfer_socket), - socket_cache: socket_cache, - ) - - configs = [] - - # rewrite for using server as sd_static - conf.elements(name: 'server').each do |s| - s.name = 'service' - end - - unless conf.elements(name: 'service').empty? - # To copy `services` element only - new_elem = Fluent::Config::Element.new('static_service_discovery', {}, {}, conf.elements(name: 'service')) - configs << { type: :static, conf: new_elem } - end - - conf.elements(name: 'service_discovery').each_with_index do |c, i| - configs << { type: @service_discovery[i][:@type], conf: c } - end - - service_discovery_create_manager( - :out_forward_service_discovery_watcher, - configurations: configs, - load_balancer: Fluent::Plugin::ForwardOutput::LoadBalancer.new(log), - custom_build_method: method(:build_node), - ) - - discovery_manager.services.each do |server| - # it's only for test - @nodes << server - unless @heartbeat_type == :none - begin - server.validate_host_resolution! - rescue => e - raise unless @ignore_network_errors_at_startup - log.warn "failed to resolve node name when configured", server: (server.name || server.host), error: e - server.disable! - end - end - end - - unless @as_secondary - if @compress == :gzip && @buffer.compress == :text - @buffer.compress = :gzip - elsif @compress == :text && @buffer.compress == :gzip - log.info "buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in " - end - end - - if discovery_manager.services.empty? - raise Fluent::ConfigError, "forward output plugin requires at least one node is required. Add or " - end - - if !@keepalive && @keepalive_timeout - log.warn('The value of keepalive_timeout is ignored. if you want to use keepalive, please add `keepalive true` to your conf.') - end - - raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1 - end - - def multi_workers_ready? - true - end - - def prefer_delayed_commit - @require_ack_response - end - - def overwrite_delayed_commit_timeout - # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout - # But it should be overwritten by ack_response_timeout to rollback chunks after timeout - if @delayed_commit_timeout != @ack_response_timeout - log.info "delayed_commit_timeout is overwritten by ack_response_timeout" - @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s - end - end - - def start - super - - unless @heartbeat_type == :none - if @heartbeat_type == :udp - @usock = socket_create_udp(discovery_manager.services.first.host, discovery_manager.services.first.port, nonblock: true) - server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv)) - end - timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer)) - end - - if @require_ack_response - overwrite_delayed_commit_timeout - thread_create(:out_forward_receiving_ack, &method(:ack_reader)) - end - - if @verify_connection_at_startup - discovery_manager.services.each do |node| - begin - node.verify_connection - rescue StandardError => e - log.fatal "forward's connection setting error: #{e.message}" - raise Fluent::UnrecoverableError, e.message - end - end - end - - if @keepalive - timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks)) - end - end - - def close - if @usock - # close socket and ignore errors: this socket will not be used anyway. - @usock.close rescue nil - end - - super - end - - def stop - super - - if @keepalive - @connection_manager.stop - end - end - - def before_shutdown - super - @suspend_flush = true - end - - def after_shutdown - last_ack if @require_ack_response - super - end - - def try_flush - return if @require_ack_response && @suspend_flush - super - end - - def last_ack - overwrite_delayed_commit_timeout - ack_check(ack_select_interval) - end - - def write(chunk) - return if chunk.empty? - tag = chunk.metadata.tag - - discovery_manager.select_service { |node| node.send_data(tag, chunk) } - end - - def try_write(chunk) - log.trace "writing a chunk to destination", chunk_id: dump_unique_id_hex(chunk.unique_id) - if chunk.empty? - commit_write(chunk.unique_id) - return - end - tag = chunk.metadata.tag - discovery_manager.select_service { |node| node.send_data(tag, chunk) } - last_ack if @require_ack_response && @suspend_flush - end - - def create_transfer_socket(host, port, hostname, &block) - case @transport - when :tls - socket_create_tls( - host, port, - version: @tls_version, - ciphers: @tls_ciphers, - insecure: @tls_insecure_mode, - verify_fqdn: @tls_verify_hostname, - fqdn: hostname, - allow_self_signed_cert: @tls_allow_self_signed_cert, - cert_paths: @tls_ca_cert_path, - cert_path: @tls_client_cert_path, - private_key_path: @tls_client_private_key_path, - private_key_passphrase: @tls_client_private_key_passphrase, - cert_thumbprint: @tls_cert_thumbprint, - cert_logical_store_name: @tls_cert_logical_store_name, - cert_use_enterprise_store: @tls_cert_use_enterprise_store, - - # Enabling SO_LINGER causes tcp port exhaustion on Windows. - # This is because dynamic ports are only 16384 (from 49152 to 65535) and - # expiring SO_LINGER enabled ports should wait 4 minutes - # where set by TcpTimeDelay. Its default value is 4 minutes. - # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports. - linger_timeout: Fluent.windows? ? nil : @send_timeout, - send_timeout: @send_timeout, - recv_timeout: @ack_response_timeout, - connect_timeout: @connect_timeout, - &block - ) - when :tcp - socket_create_tcp( - host, port, - linger_timeout: @send_timeout, - send_timeout: @send_timeout, - recv_timeout: @ack_response_timeout, - connect_timeout: @connect_timeout, - &block - ) - else - raise "BUG: unknown transport protocol #{@transport}" - end - end - - def statistics - stats = super - services = discovery_manager.services - healthy_nodes_count = 0 - registed_nodes_count = services.size - services.each do |s| - if s.available? - healthy_nodes_count += 1 - end - end - - stats.merge( - 'healthy_nodes_count' => healthy_nodes_count, - 'registered_nodes_count' => registed_nodes_count, - ) - end - - # MessagePack FixArray length is 3 - FORWARD_HEADER = [0x93].pack('C').freeze - def forward_header - FORWARD_HEADER - end - - private - - def build_node(server) - name = server.name || "#{server.host}:#{server.port}" - log.info "adding forwarding server '#{name}'", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id - - failure = Fluent::Plugin::ForwardOutput::FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f) - if @heartbeat_type == :none - NoneHeartbeatNode.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler) - else - Node.new(self, server, failure: failure, connection_manager: @connection_manager, ack_handler: @ack_handler) - end - end - - def on_heartbeat_timer - need_rebuild = false - discovery_manager.services.each do |n| - begin - log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type - n.usock = @usock if @usock - need_rebuild = n.send_heartbeat || need_rebuild - rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e - log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e - rescue => e - log.debug "unexpected error happen during heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e - end - - need_rebuild = n.tick || need_rebuild - end - - if need_rebuild - discovery_manager.rebalance - end - end - - def on_udp_heatbeat_response_recv(data, sock) - sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host) - if node = discovery_manager.services.find { |n| n.sockaddr == sockaddr } - # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port - if node.heartbeat - discovery_manager.rebalance - end - else - log.warn("Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}. It may service out") - end - end - - def on_purge_obsolete_socks - @connection_manager.purge_obsolete_socks - end - - def ack_select_interval - if @delayed_commit_timeout > 3 - 1 - else - @delayed_commit_timeout / 3.0 - end - end - - def ack_reader - select_interval = ack_select_interval - - while thread_current_running? - ack_check(select_interval) - end - end - - def ack_check(select_interval) - @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result| - @connection_manager.close(sock) - - case result - when AckHandler::Result::SUCCESS - commit_write(chunk_id) - when AckHandler::Result::FAILED - node.disable! - rollback_write(chunk_id, update_retry: false) - when AckHandler::Result::CHUNKID_UNMATCHED - rollback_write(chunk_id, update_retry: false) - else - log.warn("BUG: invalid status #{result} #{chunk_id}") - - if chunk_id - rollback_write(chunk_id, update_retry: false) - end - end - end - end - - class Node - extend Forwardable - def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby - - # @param connection_manager [Fluent::Plugin::ForwardOutput::ConnectionManager] - # @param ack_handler [Fluent::Plugin::ForwardOutput::AckHandler] - def initialize(sender, server, failure:, connection_manager:, ack_handler:) - @sender = sender - @log = sender.log - @compress = sender.compress - @server = server - - @name = server.name - @host = server.host - @port = server.port - @weight = server.weight - @standby = server.standby - @failure = failure - @available = true - - # @hostname is used for certificate verification & TLS SNI - host_is_hostname = !(IPAddr.new(@host) rescue false) - @hostname = case - when host_is_hostname then @host - when @name then @name - else nil - end - - @usock = nil - - @handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new( - log: @log, - hostname: sender.security && sender.security.self_hostname, - shared_key: server.shared_key || (sender.security && sender.security.shared_key) || '', - password: server.password || '', - username: server.username || '', - ) - - @unpacker = Fluent::MessagePackFactory.msgpack_unpacker - - @resolved_host = nil - @resolved_time = 0 - @resolved_once = false - - @connection_manager = connection_manager - @ack_handler = ack_handler - end - - attr_accessor :usock - - attr_reader :state - attr_reader :sockaddr # used by on_udp_heatbeat_response_recv - attr_reader :failure # for test - - def validate_host_resolution! - resolved_host - end - - def available? - @available - end - - def disable! - @available = false - end - - def standby? - @standby - end - - def verify_connection - connect do |sock, ri| - ensure_established_connection(sock, ri) - end - end - - def establish_connection(sock, ri) - while ri.state != :established - begin - # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly. - # We need rewrite around here using new socket/server plugin helper. - buf = sock.read_nonblock(@sender.read_length) - if buf.empty? - sleep @sender.read_interval - next - end - @unpacker.feed_each(buf) do |data| - if @handshake.invoke(sock, ri, data) == :established - @log.debug "connection established", host: @host, port: @port - end - end - rescue IO::WaitReadable - # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable. - # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock. - # https//docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock - sleep @sender.read_interval unless ri.state == :established - rescue SystemCallError => e - @log.warn "disconnected by error", host: @host, port: @port, error: e - disable! - break - rescue EOFError - @log.warn "disconnected", host: @host, port: @port - disable! - break - rescue HeloError => e - @log.warn "received invalid helo message from #{@name}" - disable! - break - rescue PingpongError => e - @log.warn "connection refused to #{@name || @host}: #{e.message}" - disable! - break - end - end - end - - def send_data_actual(sock, tag, chunk) - option = { 'size' => chunk.size, 'compressed' => @compress } - option['chunk'] = Base64.encode64(chunk.unique_id) if @ack_handler - - # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#packedforward-mode - # out_forward always uses str32 type for entries. - # str16 can store only 64kbytes, and it should be much smaller than buffer chunk size. - - tag = tag.dup.force_encoding(Encoding::UTF_8) - - sock.write @sender.forward_header # array, size=3 - sock.write tag.to_msgpack # 1. tag: String (str) - chunk.open(compressed: @compress) do |chunk_io| - entries = [0xdb, chunk_io.size].pack('CN') - sock.write entries.force_encoding(Encoding::UTF_8) # 2. entries: String (str32) - IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es) - end - sock.write option.to_msgpack # 3. option: Hash(map) - - # TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported - end - - def send_data(tag, chunk) - ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self) - connect(nil, ack: ack) do |sock, ri| - ensure_established_connection(sock, ri) - send_data_actual(sock, tag, chunk) - end - - heartbeat(false) - nil - end - - # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack - # - # @return [Boolean] return true if it needs to rebuild nodes - def send_heartbeat - begin - dest_addr = resolved_host - @resolved_once = true - rescue ::SocketError => e - if !@resolved_once && @sender.ignore_network_errors_at_startup - @log.warn "failed to resolve node name in heartbeating", server: @name || @host, error: e - return false - end - raise - end - - case @sender.heartbeat_type - when :transport - connect(dest_addr) do |sock, ri| - ensure_established_connection(sock, ri) - - ## don't send any data to not cause a compatibility problem - # sock.write FORWARD_TCP_HEARTBEAT_DATA - - # successful tcp connection establishment is considered as valid heartbeat. - # When heartbeat is succeeded after detached, return true. It rebuilds weight array. - heartbeat(true) - end - when :udp - @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, dest_addr) - # response is going to receive at on_udp_heatbeat_response_recv - false - when :none # :none doesn't use this class - raise "BUG: heartbeat_type none must not use Node" - else - raise "BUG: unknown heartbeat_type '#{@sender.heartbeat_type}'" - end - end - - def resolved_host - case @sender.expire_dns_cache - when 0 - # cache is disabled - resolve_dns! - - when nil - # persistent cache - @resolved_host ||= resolve_dns! - - else - now = Fluent::EventTime.now - rh = @resolved_host - if !rh || now - @resolved_time >= @sender.expire_dns_cache - rh = @resolved_host = resolve_dns! - @resolved_time = now - end - rh - end - end - - def resolve_dns! - addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM) - addrinfo = @sender.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first - @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_udp_heatbeat_response_recv - addrinfo[3] - end - private :resolve_dns! - - def tick - now = Time.now.to_f - unless available? - if @failure.hard_timeout?(now) - @failure.clear - end - return nil - end - - if @failure.hard_timeout?(now) - @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, hard_timeout: true - disable! - @resolved_host = nil # expire cached host - @failure.clear - return true - end - - if @sender.phi_failure_detector - phi = @failure.phi(now) - if phi > @sender.phi_threshold - @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, phi: phi, phi_threshold: @sender.phi_threshold - disable! - @resolved_host = nil # expire cached host - @failure.clear - return true - end - end - false - end - - def heartbeat(detect=true) - now = Time.now.to_f - @failure.add(now) - if detect && !available? && @failure.sample_size > @sender.recover_sample_size - @available = true - @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port - true - else - nil - end - end - - private - - def ensure_established_connection(sock, request_info) - if request_info.state != :established - establish_connection(sock, request_info) - - if request_info.state != :established - raise ConnectionClosedError, "failed to establish connection with node #{@name}" - end - end - end - - def connect(host = nil, ack: false, &block) - @connection_manager.connect(host: host || resolved_host, port: port, hostname: @hostname, ack: ack, &block) - end - end - - # Override Node to disable heartbeat - class NoneHeartbeatNode < Node - def available? - true - end - - def tick - false - end - - def heartbeat(detect=true) - true - end - end - end -end diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index e10a2049f..c83972f11 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -1,7 +1,7 @@ #!/usr/local/bin/ruby # frozen_string_literal: true -require 'fluent/plugin/output' +require "fluent/plugin/output" module Fluent::Plugin class OutputMDM < Output @@ -19,7 +19,7 @@ def initialize require_relative "constants" require_relative "arc_k8s_cluster_identity" require_relative "proxy_utils" - + require_relative "extension_utils" @@token_resource_url = "https://monitoring.azure.com/" # AAD auth supported only in public cloud and handle other clouds when enabled # this is unified new token audience for LA AAD MSI auth & metrics @@ -52,6 +52,7 @@ def initialize # Setting useMsi to false by default @useMsi = false @isAADMSIAuth = false + @isWindows = false @metrics_flushed_count = 0 @cluster_identity = nil @@ -88,6 +89,9 @@ def start aks_region = aks_region.gsub(" ", "") end + @isWindows = isWindows() + @isAADMSIAuth = ExtensionUtils.isAADMSIAuthMode() + if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" @@ -108,11 +112,19 @@ def start @log.info "POST Request url: #{@@post_request_url}" ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMPluginStart", {}) - # arc k8s cluster uses cluster identity if (!!@isArcK8sCluster) - @log.info "using cluster identity token since cluster is azure arc k8s cluster" - @cluster_identity = ArcK8sClusterIdentity.new - @cached_access_token = @cluster_identity.get_cluster_identity_token + if @isAADMSIAuth && !@isWindows + @log.info "using IMDS sidecar endpoint for MSI token since its Arc k8s and Linux node" + @useMsi = true + msi_endpoint = @@imds_msi_endpoint_template % { resource: @@token_resource_audience } + @parsed_token_uri = URI.parse(msi_endpoint) + @cached_access_token = get_access_token + else + # switch to IMDS endpoint for the windows once the Arc K8s team supports the IMDS sidecar for windows + @log.info "using cluster identity token since cluster is azure arc k8s cluster" + @cluster_identity = ArcK8sClusterIdentity.new + @cached_access_token = @cluster_identity.get_cluster_identity_token + end else # azure json file only used for aks and doesnt exist in non-azure envs file = File.read(@@azure_json_path) @@ -132,7 +144,6 @@ def start else # in case of aad msi auth user_assigned_client_id will be empty @log.info "using aad msi auth" - @isAADMSIAuth = true msi_endpoint = @@imds_msi_endpoint_template % { resource: @@token_resource_audience } end @parsed_token_uri = URI.parse(msi_endpoint) @@ -153,48 +164,59 @@ def get_access_token if (Time.now > @get_access_token_backoff_expiry) http_access_token = nil retries = 0 + properties = {} begin if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # Refresh token 5 minutes from expiration @log.info "Refreshing access token for out_mdm plugin.." - - if (!!@useMsi) - properties = {} - if (!!@isAADMSIAuth) - @log.info "Using aad msi auth to get the token to post MDM data" - properties["aadAuthMSIMode"] = "true" + if (!!@isAADMSIAuth) + properties["aadAuthMSIMode"] = "true" + end + if @isAADMSIAuth && @isWindows + @log.info "reading the token from IMDS token file since its windows.." + if File.exist?(Constants::IMDS_TOKEN_PATH_FOR_WINDOWS) && File.readable?(Constants::IMDS_TOKEN_PATH_FOR_WINDOWS) + token_content = File.read(Constants::IMDS_TOKEN_PATH_FOR_WINDOWS).strip + parsed_json = JSON.parse(token_content) + @token_expiry_time = Time.now + @@token_refresh_back_off_interval * 60 # set the expiry time to be ~ thirty minutes from current time + @cached_access_token = parsed_json["access_token"] + @log.info "Successfully got access token" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", properties) else - @log.info "Using msi to get the token to post MDM data" + raise "Either MSI Token file path doesnt exist or not readble" end - ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", properties) - @log.info "Opening TCP connection" - http_access_token = Net::HTTP.start(@parsed_token_uri.host, @parsed_token_uri.port, :use_ssl => false) - # http_access_token.use_ssl = false - token_request = Net::HTTP::Get.new(@parsed_token_uri.request_uri) - token_request["Metadata"] = true else - @log.info "Using SP to get the token to post MDM data" - ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-SP", {}) - @log.info "Opening TCP connection" - http_access_token = Net::HTTP.start(@parsed_token_uri.host, @parsed_token_uri.port, :use_ssl => true) - # http_access_token.use_ssl = true - token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) - token_request.set_form_data( - { - "grant_type" => @@grant_type, - "client_id" => @data_hash["aadClientId"], - "client_secret" => @data_hash["aadClientSecret"], - "resource" => @@token_resource_url, - } - ) - end + if (!!@useMsi) + @log.info "Using msi to get the token to post MDM data" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", properties) + @log.info "Opening TCP connection" + http_access_token = Net::HTTP.start(@parsed_token_uri.host, @parsed_token_uri.port, :use_ssl => false) + # http_access_token.use_ssl = false + token_request = Net::HTTP::Get.new(@parsed_token_uri.request_uri) + token_request["Metadata"] = true + else + @log.info "Using SP to get the token to post MDM data" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-SP", {}) + @log.info "Opening TCP connection" + http_access_token = Net::HTTP.start(@parsed_token_uri.host, @parsed_token_uri.port, :use_ssl => true) + # http_access_token.use_ssl = true + token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) + token_request.set_form_data( + { + "grant_type" => @@grant_type, + "client_id" => @data_hash["aadClientId"], + "client_secret" => @data_hash["aadClientSecret"], + "resource" => @@token_resource_url, + } + ) + end - @log.info "making request to get token.." - token_response = http_access_token.request(token_request) - # Handle the case where the response is not 200 - parsed_json = JSON.parse(token_response.body) - @token_expiry_time = Time.now + @@token_refresh_back_off_interval * 60 # set the expiry time to be ~ thirty minutes from current time - @cached_access_token = parsed_json["access_token"] - @log.info "Successfully got access token" + @log.info "making request to get token.." + token_response = http_access_token.request(token_request) + # Handle the case where the response is not 200 + parsed_json = JSON.parse(token_response.body) + @token_expiry_time = Time.now + @@token_refresh_back_off_interval * 60 # set the expiry time to be ~ thirty minutes from current time + @cached_access_token = parsed_json["access_token"] + @log.info "Successfully got access token" + end end rescue => err @log.info "Exception in get_access_token: #{err}" @@ -316,10 +338,15 @@ def write(chunk) def send_to_mdm(post_body) begin if (!!@isArcK8sCluster) - if @cluster_identity.nil? - @cluster_identity = ArcK8sClusterIdentity.new + if @isAADMSIAuth && !@isWindows + access_token = get_access_token + else + # switch to IMDS sidecar endpoint for the windows once the Arc K8s team supports + if @cluster_identity.nil? + @cluster_identity = ArcK8sClusterIdentity.new + end + access_token = @cluster_identity.get_cluster_identity_token end - access_token = @cluster_identity.get_cluster_identity_token else access_token = get_access_token end @@ -336,7 +363,7 @@ def send_to_mdm(post_body) ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMSendSuccessful", {}) @last_telemetry_sent_time = Time.now end - rescue Net::HTTPClientException => e # see https://docs.ruby-lang.org/en/2.6.0/NEWS.html about deprecating HTTPServerException and adding HTTPClientException + rescue Net::HTTPClientException => e # see https://docs.ruby-lang.org/en/2.6.0/NEWS.html about deprecating HTTPServerException and adding HTTPClientException if !response.nil? && !response.body.nil? #body will have actual error @log.info "Failed to Post Metrics to MDM : #{e} Response.body: #{response.body}" else @@ -368,5 +395,18 @@ def send_to_mdm(post_body) raise e end end + + def isWindows() + isWindows = false + begin + os_type = ENV["OS_TYPE"] + if !os_type.nil? && !os_type.empty? && os_type.strip.casecmp("windows") == 0 + isWindows = true + end + rescue => error + @log.warn "Error in MDM isWindows method: #{error}" + end + return isWindows + end end # class OutputMDM end # module Fluent diff --git a/test/e2e/conformance.yaml b/test/e2e/conformance.yaml index 71e40a6a2..bf9a3727a 100644 --- a/test/e2e/conformance.yaml +++ b/test/e2e/conformance.yaml @@ -3,7 +3,7 @@ sonobuoy-config: plugin-name: azure-arc-ci-conformance result-format: junit spec: - image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:ciconftest10202021 + image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:ciconftest04282022 imagePullPolicy: Always name: plugin resources: {} diff --git a/test/e2e/src/core/e2e_tests.sh b/test/e2e/src/core/e2e_tests.sh index dd9d93073..db89adf9a 100644 --- a/test/e2e/src/core/e2e_tests.sh +++ b/test/e2e/src/core/e2e_tests.sh @@ -111,8 +111,13 @@ addArcConnectedK8sExtension() { } addArcK8sCLIExtension() { - echo "adding Arc K8s k8s-extension extension" - az extension add --name k8s-extension + if [ ! -z "$K8S_EXTENSION_WHL_URL" ]; then + echo "adding Arc K8s k8s-extension cli extension from whl file path ${K8S_EXTENSION_WHL_URL}" + az extension add --source $K8S_EXTENSION_WHL_URL -y + else + echo "adding Arc K8s k8s-extension cli extension" + az extension add --name k8s-extension + fi } createArcCIExtension() { @@ -125,7 +130,11 @@ createArcCIExtension() { basicparameters="$basicparameters --version $CI_ARC_VERSION" fi - az k8s-extension create $basicparameters --configuration-settings omsagent.ISTEST=true + if [ ! -z "$USE_AAD_AUTH" ]; then + az k8s-extension create $basicparameters --configuration-settings omsagent.ISTEST=true omsagent.useAADAuth=$USE_AAD_AUTH + else + az k8s-extension create $basicparameters --configuration-settings omsagent.ISTEST=true + fi } showArcCIExtension() { diff --git a/test/e2e/src/tests/test_e2e_workflows.py b/test/e2e/src/tests/test_e2e_workflows.py index 02ad8cf14..1515534e1 100755 --- a/test/e2e/src/tests/test_e2e_workflows.py +++ b/test/e2e/src/tests/test_e2e_workflows.py @@ -38,7 +38,16 @@ def test_e2e_workflows(env_dict): if len(pod_list.items) <= 0: pytest.fail("number of items in pod list should be greater than 0") + if len(pod_list.items[0].spec.containers) < 1: + pytest.fail("number of containers in pod item should be at least 1") + envVars = pod_list.items[0].spec.containers[0].env + if (len(pod_list.items[0].spec.containers) > 1): + for container in pod_list.items[0].spec.containers: + if (container.name == constants.OMSAGENT_MAIN_CONTAINER_NAME): + envVars = container.env + break + if not envVars: pytest.fail("environment variables should be defined in the replicaset pod") diff --git a/test/e2e/src/tests/test_node_metrics_e2e_workflow.py b/test/e2e/src/tests/test_node_metrics_e2e_workflow.py index dfcc89dde..264abad6b 100755 --- a/test/e2e/src/tests/test_node_metrics_e2e_workflow.py +++ b/test/e2e/src/tests/test_node_metrics_e2e_workflow.py @@ -12,6 +12,8 @@ pytestmark = pytest.mark.agentests # validation of node metrics e2e workflow + + def test_node_metrics_e2e_workflow(env_dict): print("Starting node metrics e2e workflow test.") append_result_output("test_node_metrics_e2e_workflow start \n", @@ -39,7 +41,16 @@ def test_node_metrics_e2e_workflow(env_dict): if len(pod_list.items) <= 0: pytest.fail("number of items in pod list should be greater than 0") + if len(pod_list.items[0].spec.containers) < 1: + pytest.fail("number of containers in pod item should be at least 1") + envVars = pod_list.items[0].spec.containers[0].env + if (len(pod_list.items[0].spec.containers) > 1): + for container in pod_list.items[0].spec.containers: + if (container.name == constants.OMSAGENT_MAIN_CONTAINER_NAME): + envVars = container.env + break + if not envVars: pytest.fail( "environment variables should be defined in the replicaset pod") @@ -71,9 +82,11 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("access_token shouldnt be null or empty") waitTimeSeconds = env_dict['AGENT_WAIT_TIME_SECS'] - print("start: waiting for seconds: {} for agent workflows to get emitted".format(waitTimeSeconds)) + print("start: waiting for seconds: {} for agent workflows to get emitted".format( + waitTimeSeconds)) time.sleep(int(waitTimeSeconds)) - print("complete: waiting for seconds: {} for agent workflows to get emitted".format(waitTimeSeconds)) + print("complete: waiting for seconds: {} for agent workflows to get emitted".format( + waitTimeSeconds)) # validate metrics e2e workflow now = datetime.utcnow() @@ -105,8 +118,8 @@ def test_node_metrics_e2e_workflow(env_dict): "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format( - response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -122,18 +135,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_MEMORY_RSS_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_MEMORY_RSS_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_MEMORY_RSS_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_MEMORY_RSS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORY_RSS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_MEMORY_RSS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) # node metric - memoryRssPercentage custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( @@ -154,8 +170,8 @@ def test_node_metrics_e2e_workflow(env_dict): "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format( - response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -171,18 +187,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) # node metric - memoryWorkingSetBytes custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( @@ -199,11 +218,12 @@ def test_node_metrics_e2e_workflow(env_dict): headers=Headers) if not response: - pytest.fail("response of the metrics query API shouldnt be null or empty") + pytest.fail( + "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format( - response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -219,18 +239,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_MEMORY_WS_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_MEMORY_WS_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_MEMORY_WS_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_MEMORY_WS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORYE_WS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_MEMORYE_WS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) # node metric - memoryWorkingSetPercentage custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( @@ -247,11 +270,12 @@ def test_node_metrics_e2e_workflow(env_dict): headers=Headers) if not response: - pytest.fail("response of the metrics query API shouldnt be null or empty") + pytest.fail( + "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format( - response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -267,18 +291,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) # node metric - cpuUsageMilliCores custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( @@ -295,10 +322,12 @@ def test_node_metrics_e2e_workflow(env_dict): headers=Headers) if not response: - pytest.fail("response of the metrics query API shouldnt be null or empty") + pytest.fail( + "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format(response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -314,18 +343,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) # node metric - cpuUsagePercentage custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( @@ -342,10 +374,12 @@ def test_node_metrics_e2e_workflow(env_dict): headers=Headers) if not response: - pytest.fail("response of the metrics query API shouldnt be null or empty") + pytest.fail( + "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format(response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -361,18 +395,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_CPU_USAGE_PERCENTAGE_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_CPU_USAGE_PERCENTAGE_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_CPU_USAGE_PERCENTAGE_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_CPU_USAGE_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_CPU_USAGE_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_CPU_USAGE_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) # node metric - nodesCount custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( @@ -389,10 +426,12 @@ def test_node_metrics_e2e_workflow(env_dict): headers=Headers) if not response: - pytest.fail("response of the metrics query API shouldnt be null or empty") + pytest.fail( + "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format(response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -408,18 +447,21 @@ def test_node_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.NODE_COUNT_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.NODE_COUNT_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.NODE_COUNT_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.NODE_COUNT_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_COUNT_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.NODE_COUNT_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) append_result_output("test_node_metrics_e2e_workflow end \n", env_dict['TEST_AGENT_LOG_FILE']) diff --git a/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py b/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py index 81e2b77a9..4be36b8a9 100755 --- a/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py +++ b/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py @@ -12,6 +12,8 @@ pytestmark = pytest.mark.agentests # validation of pod metrics e2e workflows + + def test_pod_metrics_e2e_workflow(env_dict): print("Starting pod metrics e2e workflows test.") append_result_output("test_pod_metrics_e2e_workflow start \n", @@ -39,7 +41,16 @@ def test_pod_metrics_e2e_workflow(env_dict): if len(pod_list.items) <= 0: pytest.fail("number of items in pod list should be greater than 0") + if len(pod_list.items[0].spec.containers) < 1: + pytest.fail("number of containers in pod item should be at least 1") + envVars = pod_list.items[0].spec.containers[0].env + if (len(pod_list.items[0].spec.containers) > 1): + for container in pod_list.items[0].spec.containers: + if (container.name == constants.OMSAGENT_MAIN_CONTAINER_NAME): + envVars = container.env + break + if not envVars: pytest.fail( "environment variables should be defined in the replicaset pod") @@ -71,9 +82,11 @@ def test_pod_metrics_e2e_workflow(env_dict): pytest.fail("access_token shouldnt be null or empty") waitTimeSeconds = env_dict['AGENT_WAIT_TIME_SECS'] - print("start: waiting for seconds: {} for agent workflows to get emitted".format(waitTimeSeconds)) + print("start: waiting for seconds: {} for agent workflows to get emitted".format( + waitTimeSeconds)) time.sleep(int(waitTimeSeconds)) - print("complete: waiting for seconds: {} for agent workflows to get emitted".format(waitTimeSeconds)) + print("complete: waiting for seconds: {} for agent workflows to get emitted".format( + waitTimeSeconds)) # validate metrics e2e workflow now = datetime.utcnow() @@ -104,8 +117,8 @@ def test_pod_metrics_e2e_workflow(env_dict): "response of the metrics query API shouldnt be null or empty") if response.status_code != 200: - pytest.fail("metrics query API failed with an error code: {}".format( - response.status_code)) + pytest.fail("metrics query API failed with an error code: {}".format( + response.status_code)) responseJSON = response.json() if not responseJSON: @@ -121,18 +134,21 @@ def test_pod_metrics_e2e_workflow(env_dict): pytest.fail("response JSON shouldnt be null or empty") if len(responseValues) <= 0: - pytest.fail("length of value array in the response should be greater than 0") + pytest.fail( + "length of value array in the response should be greater than 0") for responseVal in responseValues: metricName = responseVal['name']['value'] if metricName != constants.POD_COUNT_METRIC_NAME: - pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format(metricName, constants.POD_COUNT_METRIC_NAME)) + pytest.fail("got the metricname: {0} but expected metricname:{1} in the response".format( + metricName, constants.POD_COUNT_METRIC_NAME)) timeseries = responseVal['timeseries'] if not timeseries: pytest.fail("metric series shouldnt be null or empty for metric:{0} in namespace: {1}".format( constants.POD_COUNT_METRIC_NAME, constants.POD_METRICS_NAMESPACE)) if len(timeseries) <= 0: - pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.POD_COUNT_METRIC_NAME, constants.POD_METRICS_NAMESPACE)) + pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format( + constants.POD_COUNT_METRIC_NAME, constants.POD_METRICS_NAMESPACE)) append_result_output("test_pod_metrics_e2e_workflow end \n", env_dict['TEST_AGENT_LOG_FILE']) diff --git a/test/onboarding-templates-legacy-auth/existingClusterOnboarding.json b/test/onboarding-templates-legacy-auth/existingClusterOnboarding.json new file mode 100644 index 000000000..c996e6042 --- /dev/null +++ b/test/onboarding-templates-legacy-auth/existingClusterOnboarding.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "aksResourceId": { + "type": "string", + "metadata": { + "description": "AKS Cluster Resource ID" + } + }, + "aksResourceLocation": { + "type": "string", + "metadata": { + "description": "Location of the AKS resource e.g. \"East US\"" + } + }, + "workspaceResourceId": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Resource ID" + } + } + }, + "resources": [ + { + "name": "[split(parameters('aksResourceId'),'/')[8]]", + "type": "Microsoft.ContainerService/managedClusters", + "location": "[parameters('aksResourceLocation')]", + "apiVersion": "2018-03-31", + "properties": { + "mode": "Incremental", + "id": "[parameters('aksResourceId')]", + "addonProfiles": { + "omsagent": { + "enabled": false, + "config": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/test/onboarding-templates-legacy-auth/existingClusterParam.json b/test/onboarding-templates-legacy-auth/existingClusterParam.json new file mode 100644 index 000000000..fb5d81c73 --- /dev/null +++ b/test/onboarding-templates-legacy-auth/existingClusterParam.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "aksResourceId": { + "value": "/subscriptions//resourcegroups//providers/Microsoft.ContainerService/managedClusters/" + }, + "aksResourceLocation": { + "value": "" + }, + "workspaceResourceId": { + "value": "/subscriptions//resourceGroups//providers/Microsoft.OperationalInsights/workspaces/" + } + } +} \ No newline at end of file diff --git a/test/prometheus-scraping/win-prometheus-ref-app-ltsc2019.yml b/test/prometheus-scraping/win-prometheus-ref-app-ltsc2019.yml new file mode 100644 index 000000000..6dde6346c --- /dev/null +++ b/test/prometheus-scraping/win-prometheus-ref-app-ltsc2019.yml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: win-prometheus-reference-app-ltsc2019 +spec: + selector: + matchLabels: + app: win-prometheus-reference-app-ltsc2019 + replicas: 1 + template: + metadata: + annotations: + prometheus.io/port: '2112' + prometheus.io/scrape: 'true' + labels: + app: win-prometheus-reference-app-ltsc2019 + spec: + containers: + - name: win-prometheus-reference-app-golang + image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:win-prometheus-reference-app-golang + env: + - name: RUN_PERF_TEST + value: "false" + - name: SCRAPE_INTERVAL + value: "60" + - name: METRIC_COUNT + value: "125000" + ports: + - containerPort: 2112 + protocol: TCP + - containerPort: 2113 + protocol: TCP + - name: win-prometheus-reference-app-python + image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:win-prometheus-reference-app-python + ports: + - containerPort: 2114 + protocol: TCP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows + - key: kubernetes.azure.com/os-sku + operator: NotIn + values: + - Windows2022 \ No newline at end of file diff --git a/test/prometheus-scraping/win-prometheus-ref-app-ltsc2022.yml b/test/prometheus-scraping/win-prometheus-ref-app-ltsc2022.yml new file mode 100644 index 000000000..259a959ff --- /dev/null +++ b/test/prometheus-scraping/win-prometheus-ref-app-ltsc2022.yml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: win-prometheus-reference-app-ltsc2022 +spec: + selector: + matchLabels: + app: win-prometheus-reference-app-ltsc2022 + replicas: 1 + template: + metadata: + annotations: + prometheus.io/port: '2112' + prometheus.io/scrape: 'true' + labels: + app: win-prometheus-reference-app-ltsc2022 + spec: + containers: + - name: win-prometheus-reference-app-golang + image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:win-prometheus-reference-app-golang + env: + - name: RUN_PERF_TEST + value: "false" + - name: SCRAPE_INTERVAL + value: "60" + - name: METRIC_COUNT + value: "125000" + ports: + - containerPort: 2112 + protocol: TCP + - containerPort: 2113 + protocol: TCP + - name: win-prometheus-reference-app-python + image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:win-prometheus-reference-app-python + ports: + - containerPort: 2114 + protocol: TCP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows + - key: kubernetes.azure.com/os-sku + operator: In + values: + - Windows2022 \ No newline at end of file diff --git a/test/scenario/log-app-win-ltsc2019.yml b/test/scenario/log-app-win-ltsc2019.yml new file mode 100644 index 000000000..67381fa35 --- /dev/null +++ b/test/scenario/log-app-win-ltsc2019.yml @@ -0,0 +1,50 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: windows-log-ltsc2019 + labels: + name: windows-log-ltsc2019 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: second-log-app + namespace: windows-log-ltsc2019 +spec: + replicas: 1 + selector: + matchLabels: + app: second-log-app + template: + metadata: + labels: + app: second-log-app + spec: + volumes: + - name: html + emptyDir: {} + containers: + - name: second-log-app + image: mcr.microsoft.com/windows/servercore:ltsc2019 + command: ["powershell", "-c"] + args: + - "$counter = 1; For(;;) { echo $counter; $counter++; Start-Sleep -Seconds 1; }" + env: + - name: RANOMD_ENV_VAR_1 + value: "#123312'@$98" + - name: RANOMD_ENV_VAR_2 + value: "test" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows + - key: kubernetes.azure.com/os-sku + operator: NotIn + values: + - Windows2022 \ No newline at end of file diff --git a/test/scenario/log-app-win-ltsc2022.yml b/test/scenario/log-app-win-ltsc2022.yml new file mode 100644 index 000000000..bdbddbaa4 --- /dev/null +++ b/test/scenario/log-app-win-ltsc2022.yml @@ -0,0 +1,50 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: windows-log-ltsc2022 + labels: + name: windows-log-ltsc2022 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: second-log-app + namespace: windows-log-ltsc2022 +spec: + replicas: 1 + selector: + matchLabels: + app: second-log-app + template: + metadata: + labels: + app: second-log-app + spec: + volumes: + - name: html + emptyDir: {} + containers: + - name: second-log-app + image: mcr.microsoft.com/windows/servercore:ltsc2022 + command: ["powershell", "-c"] + args: + - "$counter = 1; For(;;) { echo $counter; $counter++; Start-Sleep -Seconds 1; }" + env: + - name: RANOMD_ENV_VAR_1 + value: "#123312'@$98" + - name: RANOMD_ENV_VAR_2 + value: "test" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows + - key: kubernetes.azure.com/os-sku + operator: In + values: + - Windows2022 diff --git a/test/unit-tests/plugins/filter_health_model_builder_test.rb b/test/unit-tests/plugins/filter_health_model_builder_test.rb deleted file mode 100644 index c5b17306a..000000000 --- a/test/unit-tests/plugins/filter_health_model_builder_test.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'test/unit' -require 'json' -# require_relative '../../../source/plugins/ruby/health' - -Dir[File.join(__dir__, '../../../source/plugins/ruby/health', '*.rb')].each { |file| require file } - -class FilterHealthModelBuilderTest < Test::Unit::TestCase - include HealthModel - - def test_event_stream - health_definition_path = 'C:\AzureMonitor\ContainerInsights\Docker-Provider\installer\conf\health_model_definition.json' - health_model_definition = ParentMonitorProvider.new(HealthModelDefinitionParser.new(health_definition_path).parse_file) - monitor_factory = MonitorFactory.new - hierarchy_builder = HealthHierarchyBuilder.new(health_model_definition, monitor_factory) - # TODO: Figure out if we need to add NodeMonitorHierarchyReducer to the list of finalizers. For now, dont compress/optimize, since it becomes impossible to construct the model on the UX side - state_finalizers = [AggregateMonitorStateFinalizer.new] - monitor_set = MonitorSet.new - model_builder = HealthModelBuilder.new(hierarchy_builder, state_finalizers, monitor_set) - - i = 1 - loop do - mock_data_path = "C:/AzureMonitor/ContainerInsights/Docker-Provider/source/plugins/ruby/mock_data-#{i}.json" - file = File.read(mock_data_path) - data = JSON.parse(file) - - health_monitor_records = [] - data.each do |record| - health_monitor_record = HealthMonitorRecord.new( - record[HealthMonitorRecordFields::MONITOR_ID], - record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], - record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], - record[HealthMonitorRecordFields::DETAILS]["state"], - record[HealthMonitorRecordFields::MONITOR_LABELS], - record[HealthMonitorRecordFields::MONITOR_CONFIG], - record[HealthMonitorRecordFields::DETAILS] - ) - state_transitions.push(state_transition) - end - - model_builder.process_state_transitions(state_transitions) - changed_monitors = model_builder.finalize_model - changed_monitors.keys.each{|key| - puts key - } - i = i + 1 - if i == 6 - break - end - end - puts "Done" - end -end diff --git a/test/unit-tests/plugins/health/aggregate_monitor_spec.rb b/test/unit-tests/plugins/health/aggregate_monitor_spec.rb deleted file mode 100644 index a12a0aa7f..000000000 --- a/test/unit-tests/plugins/health/aggregate_monitor_spec.rb +++ /dev/null @@ -1,256 +0,0 @@ -require_relative '../test_helpers' - -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe "AggregateMonitor Spec" do - it "is_aggregate_monitor is true for AggregateMonitor" do - # Arrange/Act - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "worstOf", [], {}) - # Assert - assert_equal monitor.is_aggregate_monitor, true - end - - it "add_member_monitor tests -- adds a member monitor as a child monitor" do - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "worstOf", [], {}) - #Act - monitor.add_member_monitor("child_monitor_1") - #Assert - assert_equal monitor.get_member_monitors.include?("child_monitor_1"), true - - #Act - monitor.add_member_monitor("child_monitor_1") - #Assert - assert_equal monitor.get_member_monitors.size, 1 - end - - it "remove_member_monitor tests -- removes a member monitor as a child monitor" do - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "worstOf", [], {}) - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - - #Act - monitor.remove_member_monitor("child_monitor_1") - #Assert - assert_equal monitor.get_member_monitors.size, 1 - - #Act - monitor.remove_member_monitor("unknown_child") - #Assert - assert_equal monitor.get_member_monitors.size, 1 - end - - it "calculate_details tests -- calculates rollup details based on member monitor states" do - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "worstOf", [], {}) - - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "fail", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - - #Act - monitor.calculate_details(monitor_set) - #Assert - assert_equal monitor.details["details"], {"pass"=>["child_monitor_1"], "fail"=>["child_monitor_2"]} - - #Arrange - child_monitor_3 = UnitMonitor.new("monitor_3", "child_monitor_3", "pass", "time", {}, {}, {}) - monitor_set.add_or_update(child_monitor_3) - monitor.add_member_monitor("child_monitor_3") - - #Act - monitor.calculate_details(monitor_set) - #Assert - assert_equal monitor.details["details"], {"pass"=>["child_monitor_1", "child_monitor_3"], "fail"=>["child_monitor_2"]} - end - - it "calculate_state tests -- raises when right aggregation_algorithm NOT specified" do - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "", [], {}) - #Assert - assert_raises do - monitor.calculate_state(monitor_set) - end - end - - it "calculate_state tests -- calculate_worst_of_state " do - # Arrange -- pass, fail = fail - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "worstOf", [], {}) - - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "fail", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "fail" - - #Arrange -- pass, pass = pass - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "pass", "time", {}, {}, {}) - monitor_set.add_or_update(child_monitor_2) - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "pass" - - #Arrange -- pass, warn = warn - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "warn", "time", {}, {}, {}) - monitor_set.add_or_update(child_monitor_2) - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "warn" - - #Arrange -- warn, fail = fail - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "warn", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "fail", "time", {}, {}, {}) - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "fail" - - #Arrange -- warn, unknown = unknown - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "warn", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "unknown", "time", {}, {}, {}) - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "warn" - - #Arrange -- pass, unknown = unknown - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "unknown", "time", {}, {}, {}) - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "unknown" - end - - it "calculate_state tests -- calculate_percentage_state " do - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "percentage", {"state_threshold" => 90.0}, {}) - - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "fail", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "fail" - - #Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "percentage", {"state_threshold" => 50.0}, {}) - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "fail", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "pass" - - #Arrange -- single child monitor - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "percentage", {"state_threshold" => 33.3}, {}) - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor.add_member_monitor("child_monitor_1") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "pass" - - - #Arrange -- remove none state - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :none, :time, "percentage", {"state_threshold" => 100.0}, {}) - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "none", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "pass" - - - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "percentage", {"state_threshold" => 50.0}, {}) - - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "fail", "time", {}, {}, {}) - child_monitor_3 = UnitMonitor.new("monitor_3", "child_monitor_3", "fail", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - monitor_set.add_or_update(child_monitor_3) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - monitor.add_member_monitor("child_monitor_3") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "fail" - - - # Arrange - monitor = AggregateMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, "percentage", {"state_threshold" => 90.0}, {}) - - child_monitor_1 = UnitMonitor.new("monitor_1", "child_monitor_1", "pass", "time", {}, {}, {}) - child_monitor_2 = UnitMonitor.new("monitor_2", "child_monitor_2", "pass", "time", {}, {}, {}) - child_monitor_3 = UnitMonitor.new("monitor_3", "child_monitor_3", "pass", "time", {}, {}, {}) - - monitor_set = MonitorSet.new - monitor_set.add_or_update(child_monitor_1) - monitor_set.add_or_update(child_monitor_2) - monitor_set.add_or_update(child_monitor_3) - - monitor.add_member_monitor("child_monitor_1") - monitor.add_member_monitor("child_monitor_2") - monitor.add_member_monitor("child_monitor_3") - #Act - monitor.calculate_state(monitor_set) - #Assert - assert_equal monitor.state, "pass" - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/aggregate_monitor_state_finalizer_spec.rb b/test/unit-tests/plugins/health/aggregate_monitor_state_finalizer_spec.rb deleted file mode 100644 index 71e4aa16a..000000000 --- a/test/unit-tests/plugins/health/aggregate_monitor_state_finalizer_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "AggregateMonitorStateFinalizer spec" do - it 'computes the right state and details' do - #arrange - monitor_set = Mock.new - - #mock unit monitors - child1 = Mock.new - def child1.state; "pass"; end - def child1.monitor_id; "child1";end - def child1.monitor_instance_id; "child1"; end - def child1.nil?; false; end - def child1.is_aggregate_monitor; false; end - - child2 = Mock.new - def child2.state; "fail"; end - def child2.monitor_id; "child2";end - def child2.monitor_instance_id; "child2"; end - def child2.nil?; false; end - def child2.is_aggregate_monitor; false; end - - parent_monitor = AggregateMonitor.new("parent_monitor", "parent_monitor", :none, :time, "worstOf", nil, {}) - parent_monitor.add_member_monitor("child1") - parent_monitor.add_member_monitor("child2") - - top_level_monitor = AggregateMonitor.new("cluster", "cluster", :none, :time, "worstOf", nil, {}) - top_level_monitor.add_member_monitor("parent_monitor") - - monitor_set.expect(:get_map, {"cluster" => top_level_monitor, "parent_monitor" => parent_monitor, "child1" => child1, "child2" => child2}) - monitor_set.expect(:get_monitor, top_level_monitor, ["cluster"]) - monitor_set.expect(:get_monitor, parent_monitor, ["parent_monitor"]) - monitor_set.expect(:get_monitor, child1, ["child1"]) - monitor_set.expect(:get_monitor, child2, ["child2"]) - monitor_set.expect(:get_monitor, child1, ["child1"]) - monitor_set.expect(:get_monitor, child2, ["child2"]) - monitor_set.expect(:get_monitor, parent_monitor, ["parent_monitor"]) - - - monitor_set.expect(:get_monitor, parent_monitor, ["parent_monitor"]) - monitor_set.expect(:get_monitor, child1, ["child1"]) - monitor_set.expect(:get_monitor, child2, ["child2"]) - - #act - finalizer = AggregateMonitorStateFinalizer.new - finalizer.finalize(monitor_set) - #assert - - assert_equal parent_monitor.state, "fail" - assert_equal parent_monitor.details, {"details"=>{"pass"=>["child1"], "fail"=>["child2"]}, "state"=>"fail", "timestamp"=>:time} - - assert_equal top_level_monitor.state, "fail" - assert_equal top_level_monitor.details, {"details"=>{"fail"=>["parent_monitor"]}, "state"=>"fail", "timestamp"=>:time} - - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/ca.crt b/test/unit-tests/plugins/health/ca.crt deleted file mode 100644 index 9daeafb98..000000000 --- a/test/unit-tests/plugins/health/ca.crt +++ /dev/null @@ -1 +0,0 @@ -test diff --git a/test/unit-tests/plugins/health/cadvisor_perf.json b/test/unit-tests/plugins/health/cadvisor_perf.json deleted file mode 100644 index 35eae32b6..000000000 --- a/test/unit-tests/plugins/health/cadvisor_perf.json +++ /dev/null @@ -1,2540 +0,0 @@ -[ - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:39Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 14061568 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:44Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 7249920 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:45Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 14442496 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:49Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 5988352 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:43Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/f65e6a62-c5c8-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 40284160 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:41Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/69e68b21-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 101965824 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 3203072 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:42Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 9658368 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:42Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9543dbb7-a1f2-11e9-8b08-d602e29755d5/metrics-server", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 21491712 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639906 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639899 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639895 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639903 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/f65e6a62-c5c8-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566580259 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/69e68b21-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566589936 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1563224142 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1563224144 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9543dbb7-a1f2-11e9-8b08-d602e29755d5/metrics-server", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639893 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:39Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 349987 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:44Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 773186 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:45Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 2718196 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:49Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 2007695 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:43Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/f65e6a62-c5c8-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 674463 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:41Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/69e68b21-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 2159553 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 3575667 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:42Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 0 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:42Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9543dbb7-a1f2-11e9-8b08-d602e29755d5/metrics-server", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 633968 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:39Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 11546624 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:39Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 11546624 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:44Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 5652480 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:45Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 10981376 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:49Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 2875392 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:43Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/f65e6a62-c5c8-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 20627456 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:41Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/69e68b21-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 69353472 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 462848 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:42Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/e690309f-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 8212480 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:42Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9543dbb7-a1f2-11e9-8b08-d602e29755d5/metrics-server", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 16543744 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:45Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-1", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 814518272 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:45Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-1", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 82091339.40983607 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:45Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-1", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 2089115648 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:50Z", - "Host": "aks-nodepool1-19574989-1", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-1", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1552408751.22 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:56Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b1e04e1c-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 85528576 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:54Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/49e373c8-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 25415680 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:53Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/65a6f978-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 111738880 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:55Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster-nanny", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 8417280 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:01Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 19492864 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9583b2ab-a1f2-11e9-8b08-d602e29755d5/main", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 12918784 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:46Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 3379200 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 9818112 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b1e04e1c-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566590024 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/49e373c8-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566580398 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/65a6f978-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566589942 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster-nanny", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566580342 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566580337 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9583b2ab-a1f2-11e9-8b08-d602e29755d5/main", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639936 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1563224072 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1563224077 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:56Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b1e04e1c-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 4447595 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:54Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/49e373c8-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 2765529 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:53Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/65a6f978-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 5565414 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:55Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster-nanny", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 863810 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:01Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 886196 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9583b2ab-a1f2-11e9-8b08-d602e29755d5/main", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 855014 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:46Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 1794634 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 0 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:56Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b1e04e1c-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 76308480 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:54Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/49e373c8-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 21319680 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:53Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/65a6f978-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 78180352 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:55Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster-nanny", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 7909376 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:01Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/24ab7e32-c5c9-11e9-8736-86290fd7dd1f/heapster", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 18968576 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/9583b2ab-a1f2-11e9-8b08-d602e29755d5/main", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 9871360 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:46Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 462848 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/bb3d3ef2-a742-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 8212480 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-0", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 865943552 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-0", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 95432166.25 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:12:57Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-0", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 2191216640 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:03Z", - "Host": "aks-nodepool1-19574989-0", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-0", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1552408749.66 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:07Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b2a0e1b3-bd3f-11e9-b2a7-d61658c73830/tunnel-front", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 17743872 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:12Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/114f7246-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 24162304 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:07Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/azureproxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 11472896 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:06Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/redirector", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 3821568 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/8dbd5e8b-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 92057600 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b2a0e1b3-bd3f-11e9-b2a7-d61658c73830/tunnel-front", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1565641691 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/114f7246-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566580300 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/azureproxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1565204288 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/redirector", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1565204284 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/8dbd5e8b-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566589995 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:07Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b2a0e1b3-bd3f-11e9-b2a7-d61658c73830/tunnel-front", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 35140951 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:12Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/114f7246-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 983407 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:07Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/azureproxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 0 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:06Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/redirector", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 4221562 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/8dbd5e8b-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 1881274 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:07Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b2a0e1b3-bd3f-11e9-b2a7-d61658c73830/tunnel-front", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 4161536 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:12Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/114f7246-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 18952192 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:07Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/azureproxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 8224768 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:06Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/44a61692-b945-11e9-a1b6-127094e7fd94/redirector", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 483328 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/8dbd5e8b-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 74915840 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:14Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-3", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 554704896 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:14Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-3", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 88981130.86666666 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:14Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-3", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 1633976320 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:14:15Z", - "Host": "aks-nodepool1-19574989-3", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-3", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1565204130.6 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/be78d7f6-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 92954624 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:33Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 7446528 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:22Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 14811136 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 15114240 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:35Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 5406720 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:32Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/94e52ab1-a1f2-11e9-8b08-d602e29755d5/autoscaler", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 10043392 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/2c3de48d-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 58052608 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 9904128 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 3645440 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/be78d7f6-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566590079 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639920 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639940 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639904 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639932 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/94e52ab1-a1f2-11e9-8b08-d602e29755d5/autoscaler", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1562639909 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/2c3de48d-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1566580349 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1563224204 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1563224199 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/be78d7f6-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 3004849 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:33Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 796842 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:22Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 708906 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 3451625 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:35Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 2572419 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:32Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/94e52ab1-a1f2-11e9-8b08-d602e29755d5/autoscaler", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 548275 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/2c3de48d-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 1740316 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 0 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 3156661 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/be78d7f6-c5df-11e9-8736-86290fd7dd1f/omsagent", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 66428928 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:33Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/dnsmasq", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 5611520 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:22Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/sidecar", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 11833344 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/kubedns", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 11063296 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:35Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/95046bc6-a1f2-11e9-8b08-d602e29755d5/healthz", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 2551808 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:32Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/94e52ab1-a1f2-11e9-8b08-d602e29755d5/autoscaler", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 9244672 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:37Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/2c3de48d-c5c9-11e9-8736-86290fd7dd1f/kube-proxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 20402176 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/azureproxy", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 8216576 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:31Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/06fef5f6-a743-11e9-a38a-22d1c75c4357/redirector", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 462848 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:30Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-2", - "Collections": [ - { - "CounterName": "memoryRssBytes", - "Value": 853344256 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:30Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-2", - "Collections": [ - { - "CounterName": "cpuUsageNanoCores", - "Value": 114265842.16 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:30Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-2", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 1892982784 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }, - { - "DataItems": [ - { - "Timestamp": "2019-08-23T22:13:40Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SNode", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/aks-nodepool1-19574989-2", - "Collections": [ - { - "CounterName": "restartTimeEpoch", - "Value": 1561082409.36 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - } -] \ No newline at end of file diff --git a/test/unit-tests/plugins/health/cluster_health_state_spec.rb b/test/unit-tests/plugins/health/cluster_health_state_spec.rb deleted file mode 100644 index fd13213b1..000000000 --- a/test/unit-tests/plugins/health/cluster_health_state_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -require 'time' -include HealthModel -include Minitest - -describe "Cluster Health State Spec" do - - it "ClusterHealthState.new throws if cert file is NOT present" do - state = { - "m1" => { - "state" => "pass", - "time" => Time.now.utc.iso8601 - } - } - - token_file_path = 'token' - cert_file_path = '/var/ca.crt' - - proc {ClusterHealthState.new(token_file_path, cert_file_path)}.must_raise - - end - - it "ClusterHealthState.new returns nil if token is NOT present" do - state = { - "m1" => { - "state" => "pass", - "time" => Time.now.utc.iso8601 - } - } - token_file_path = 'token' - cert_file_path = File.join(File.expand_path(File.dirname(__FILE__)), "ca.crt") - - chs = ClusterHealthState.new(token_file_path, cert_file_path) - chs.token.must_be_nil - end -end diff --git a/test/unit-tests/plugins/health/deployments.json b/test/unit-tests/plugins/health/deployments.json deleted file mode 100644 index 75586db04..000000000 --- a/test/unit-tests/plugins/health/deployments.json +++ /dev/null @@ -1,1385 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "2" - }, - "creationTimestamp": "2019-08-23T17:12:00Z", - "generation": 2, - "labels": { - "addonmanager.kubernetes.io/mode": "EnsureExists", - "k8s-app": "heapster", - "kubernetes.io/cluster-service": "true" - }, - "name": "heapster", - "namespace": "kube-system", - "resourceVersion": "19048928", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/heapster", - "uid": "1e98c3d1-c5c9-11e9-8736-86290fd7dd1f" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "k8s-app": "heapster" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "k8s-app": "heapster" - } - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/heapster", - "--source=kubernetes.summary_api:\"\"" - ], - "image": "aksrepos.azurecr.io/mirror/heapster-amd64:v1.5.3", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8082, - "scheme": "HTTP" - }, - "initialDelaySeconds": 180, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "heapster", - "resources": { - "limits": { - "cpu": "88m", - "memory": "204Mi" - }, - "requests": { - "cpu": "88m", - "memory": "204Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - }, - { - "command": [ - "/pod_nanny", - "--config-dir=/etc/config", - "--cpu=80m", - "--extra-cpu=0.5m", - "--memory=140Mi", - "--extra-memory=4Mi", - "--threshold=5", - "--deployment=heapster", - "--container=heapster", - "--poll-period=300000", - "--estimator=exponential" - ], - "env": [ - { - "name": "MY_POD_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.name" - } - } - }, - { - "name": "MY_POD_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - } - ], - "image": "aksrepos.azurecr.io/mirror/addon-resizer:1.8.1", - "imagePullPolicy": "IfNotPresent", - "name": "heapster-nanny", - "resources": { - "limits": { - "cpu": "50m", - "memory": "90Mi" - }, - "requests": { - "cpu": "50m", - "memory": "90Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/config", - "name": "heapster-config-volume" - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "heapster", - "serviceAccountName": "heapster", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "heapster-config" - }, - "name": "heapster-config-volume" - } - ] - } - } - }, - "status": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2019-08-23T17:12:00Z", - "lastUpdateTime": "2019-08-23T17:12:00Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 2, - "readyReplicas": 1, - "replicas": 1, - "updatedReplicas": 1 - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "5", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"kube-dns-autoscaler\",\"kubernetes.io/cluster-service\":\"true\"},\"name\":\"kube-dns-autoscaler\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"kube-dns-autoscaler\"}},\"template\":{\"metadata\":{\"annotations\":{\"scheduler.alpha.kubernetes.io/critical-pod\":\"\",\"seccomp.security.alpha.kubernetes.io/pod\":\"docker/default\"},\"labels\":{\"k8s-app\":\"kube-dns-autoscaler\"}},\"spec\":{\"containers\":[{\"command\":[\"/cluster-proportional-autoscaler\",\"--namespace=kube-system\",\"--configmap=kube-dns-autoscaler\",\"--target=deployment/kube-dns-v20\",\"--default-params={\\\"ladder\\\":{\\\"coresToReplicas\\\":[[1,2],[512,3],[1024,4],[2048,5]],\\\"nodesToReplicas\\\":[[1,2],[8,3],[16,4],[32,5]]}}\",\"--logtostderr=true\",\"--v=2\"],\"image\":\"aksrepos.azurecr.io/mirror/cluster-proportional-autoscaler-amd64:1.1.2-r2\",\"name\":\"autoscaler\",\"resources\":{\"requests\":{\"cpu\":\"20m\",\"memory\":\"10Mi\"}}}],\"dnsPolicy\":\"Default\",\"imagePullSecrets\":[{\"name\":\"emptyacrsecret\"}],\"priorityClassName\":\"system-node-critical\",\"serviceAccountName\":\"kube-dns-autoscaler\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"}]}}}}\n" - }, - "creationTimestamp": "2019-03-12T16:38:30Z", - "generation": 5, - "labels": { - "addonmanager.kubernetes.io/mode": "Reconcile", - "k8s-app": "kube-dns-autoscaler", - "kubernetes.io/cluster-service": "true" - }, - "name": "kube-dns-autoscaler", - "namespace": "kube-system", - "resourceVersion": "15144046", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/kube-dns-autoscaler", - "uid": "4509acaf-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "k8s-app": "kube-dns-autoscaler" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "annotations": { - "scheduler.alpha.kubernetes.io/critical-pod": "", - "seccomp.security.alpha.kubernetes.io/pod": "docker/default" - }, - "creationTimestamp": null, - "labels": { - "k8s-app": "kube-dns-autoscaler" - } - }, - "spec": { - "containers": [ - { - "command": [ - "/cluster-proportional-autoscaler", - "--namespace=kube-system", - "--configmap=kube-dns-autoscaler", - "--target=deployment/kube-dns-v20", - "--default-params={\"ladder\":{\"coresToReplicas\":[[1,2],[512,3],[1024,4],[2048,5]],\"nodesToReplicas\":[[1,2],[8,3],[16,4],[32,5]]}}", - "--logtostderr=true", - "--v=2" - ], - "image": "aksrepos.azurecr.io/mirror/cluster-proportional-autoscaler-amd64:1.1.2-r2", - "imagePullPolicy": "IfNotPresent", - "name": "autoscaler", - "resources": { - "requests": { - "cpu": "20m", - "memory": "10Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-dns-autoscaler", - "serviceAccountName": "kube-dns-autoscaler", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ] - } - } - }, - "status": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2019-03-12T16:38:30Z", - "lastUpdateTime": "2019-03-12T16:38:30Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 5, - "readyReplicas": 1, - "replicas": 1, - "updatedReplicas": 1 - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "6", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"kube-dns\",\"kubernetes.io/cluster-service\":\"true\",\"version\":\"v20\"},\"name\":\"kube-dns-v20\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"kube-dns\",\"version\":\"v20\"}},\"template\":{\"metadata\":{\"annotations\":{\"prometheus.io/port\":\"10055\",\"prometheus.io/scrape\":\"true\"},\"labels\":{\"k8s-app\":\"kube-dns\",\"kubernetes.io/cluster-service\":\"true\",\"version\":\"v20\"}},\"spec\":{\"affinity\":{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"labelSelector\":null,\"matchExpressions\":[{\"key\":\"kubernetes.azure.com/cluster\",\"operator\":\"Exists\"}]}]}},\"podAntiAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"podAffinityTerm\":{\"labelSelector\":{\"matchExpressions\":[{\"key\":\"k8s-app\",\"operator\":\"In\",\"values\":[\"kube-dns\"]}]},\"topologyKey\":\"kubernetes.io/hostname\"},\"weight\":100}]}},\"containers\":[{\"args\":[\"--kubecfg-file=/config/kubeconfig\",\"--config-dir=/kube-dns-config\",\"--domain=cluster.local.\",\"--dns-port=10053\",\"--v=2\"],\"env\":[{\"name\":\"PROMETHEUS_PORT\",\"value\":\"10055\"}],\"image\":\"aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13\",\"livenessProbe\":{\"failureThreshold\":5,\"httpGet\":{\"path\":\"/healthcheck/kubedns\",\"port\":10054,\"scheme\":\"HTTP\"},\"initialDelaySeconds\":60,\"successThreshold\":1,\"timeoutSeconds\":5},\"name\":\"kubedns\",\"ports\":[{\"containerPort\":10053,\"name\":\"dns-local\",\"protocol\":\"UDP\"},{\"containerPort\":10053,\"name\":\"dns-tcp-local\",\"protocol\":\"TCP\"},{\"containerPort\":10055,\"name\":\"metrics\",\"protocol\":\"TCP\"}],\"readinessProbe\":{\"httpGet\":{\"path\":\"/readiness\",\"port\":8081,\"scheme\":\"HTTP\"},\"initialDelaySeconds\":30,\"timeoutSeconds\":5},\"resources\":{\"limits\":{\"memory\":\"170Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"70Mi\"}},\"volumeMounts\":[{\"mountPath\":\"/kube-dns-config\",\"name\":\"kube-dns-config\"},{\"mountPath\":\"/config\",\"name\":\"kubedns-kubecfg\",\"readOnly\":true}]},{\"args\":[\"-v=2\",\"-logtostderr\",\"-configDir=/kube-dns-config\",\"-restartDnsmasq=true\",\"--\",\"-k\",\"--cache-size=1000\",\"--no-negcache\",\"--no-resolv\",\"--server=127.0.0.1#10053\",\"--server=/cluster.local/127.0.0.1#10053\",\"--server=/in-addr.arpa/127.0.0.1#10053\",\"--server=/ip6.arpa/127.0.0.1#10053\",\"--log-facility=-\"],\"image\":\"aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10\",\"name\":\"dnsmasq\",\"ports\":[{\"containerPort\":53,\"name\":\"dns\",\"protocol\":\"UDP\"},{\"containerPort\":53,\"name\":\"dns-tcp\",\"protocol\":\"TCP\"}],\"volumeMounts\":[{\"mountPath\":\"/kube-dns-config\",\"name\":\"kube-dns-config\"}]},{\"args\":[\"--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1 \\u003e/dev/null || exit 1; done\",\"--url=/healthz-dnsmasq\",\"--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1:10053 \\u003e/dev/null || exit 1; done\",\"--url=/healthz-kubedns\",\"--port=8080\",\"--quiet\"],\"env\":[{\"name\":\"PROBE_DOMAINS\",\"value\":\"bing.com kubernetes.default.svc.cluster.local\"}],\"image\":\"aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2\",\"livenessProbe\":{\"failureThreshold\":5,\"httpGet\":{\"path\":\"/healthz-dnsmasq\",\"port\":8080,\"scheme\":\"HTTP\"},\"initialDelaySeconds\":60,\"successThreshold\":1,\"timeoutSeconds\":5},\"name\":\"healthz\",\"ports\":[{\"containerPort\":8080,\"protocol\":\"TCP\"}],\"resources\":{\"limits\":{\"memory\":\"50Mi\"},\"requests\":{\"cpu\":\"10m\",\"memory\":\"50Mi\"}}},{\"args\":[\"--v=2\",\"--logtostderr\",\"--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV\",\"--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV\"],\"image\":\"aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10\",\"livenessProbe\":{\"httpGet\":{\"path\":\"/metrics\",\"port\":10054,\"scheme\":\"HTTP\"},\"initialDelaySeconds\":60,\"successThreshold\":1,\"timeoutSeconds\":5},\"name\":\"sidecar\",\"ports\":[{\"containerPort\":10054,\"name\":\"metrics\",\"protocol\":\"TCP\"}],\"resources\":{\"requests\":{\"cpu\":\"10m\",\"memory\":\"20Mi\"}}}],\"dnsPolicy\":\"Default\",\"imagePullSecrets\":[{\"name\":\"emptyacrsecret\"}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\"},\"priorityClassName\":\"system-node-critical\",\"serviceAccountName\":\"kube-dns\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"}],\"volumes\":[{\"configMap\":{\"name\":\"kube-dns\",\"optional\":true},\"name\":\"kube-dns-config\"},{\"configMap\":{\"name\":\"kubedns-kubecfg\"},\"name\":\"kubedns-kubecfg\"}]}}}}\n" - }, - "creationTimestamp": "2019-03-12T16:38:30Z", - "generation": 7, - "labels": { - "addonmanager.kubernetes.io/mode": "Reconcile", - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "version": "v20" - }, - "name": "kube-dns-v20", - "namespace": "kube-system", - "resourceVersion": "15144054", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/kube-dns-v20", - "uid": "4523fcd7-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 2, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "k8s-app": "kube-dns", - "version": "v20" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "annotations": { - "prometheus.io/port": "10055", - "prometheus.io/scrape": "true" - }, - "creationTimestamp": null, - "labels": { - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "version": "v20" - } - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - }, - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "k8s-app", - "operator": "In", - "values": [ - "kube-dns" - ] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - }, - "weight": 100 - } - ] - } - }, - "containers": [ - { - "args": [ - "--kubecfg-file=/config/kubeconfig", - "--config-dir=/kube-dns-config", - "--domain=cluster.local.", - "--dns-port=10053", - "--v=2" - ], - "env": [ - { - "name": "PROMETHEUS_PORT", - "value": "10055" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthcheck/kubedns", - "port": 10054, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "kubedns", - "ports": [ - { - "containerPort": 10053, - "name": "dns-local", - "protocol": "UDP" - }, - { - "containerPort": 10053, - "name": "dns-tcp-local", - "protocol": "TCP" - }, - { - "containerPort": 10055, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readiness", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "170Mi" - }, - "requests": { - "cpu": "100m", - "memory": "70Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/kube-dns-config", - "name": "kube-dns-config" - }, - { - "mountPath": "/config", - "name": "kubedns-kubecfg", - "readOnly": true - } - ] - }, - { - "args": [ - "-v=2", - "-logtostderr", - "-configDir=/kube-dns-config", - "-restartDnsmasq=true", - "--", - "-k", - "--cache-size=1000", - "--no-negcache", - "--no-resolv", - "--server=127.0.0.1#10053", - "--server=/cluster.local/127.0.0.1#10053", - "--server=/in-addr.arpa/127.0.0.1#10053", - "--server=/ip6.arpa/127.0.0.1#10053", - "--log-facility=-" - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10", - "imagePullPolicy": "IfNotPresent", - "name": "dnsmasq", - "ports": [ - { - "containerPort": 53, - "name": "dns", - "protocol": "UDP" - }, - { - "containerPort": 53, - "name": "dns-tcp", - "protocol": "TCP" - } - ], - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/kube-dns-config", - "name": "kube-dns-config" - } - ] - }, - { - "args": [ - "--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1 \u003e/dev/null || exit 1; done", - "--url=/healthz-dnsmasq", - "--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1:10053 \u003e/dev/null || exit 1; done", - "--url=/healthz-kubedns", - "--port=8080", - "--quiet" - ], - "env": [ - { - "name": "PROBE_DOMAINS", - "value": "bing.com kubernetes.default.svc.cluster.local" - } - ], - "image": "aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthz-dnsmasq", - "port": 8080, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "healthz", - "ports": [ - { - "containerPort": 8080, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "50Mi" - }, - "requests": { - "cpu": "10m", - "memory": "50Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - }, - { - "args": [ - "--v=2", - "--logtostderr", - "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV", - "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV" - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/metrics", - "port": 10054, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "sidecar", - "ports": [ - { - "containerPort": 10054, - "name": "metrics", - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "20Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-dns", - "serviceAccountName": "kube-dns", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "kube-dns", - "optional": true - }, - "name": "kube-dns-config" - }, - { - "configMap": { - "defaultMode": 420, - "name": "kubedns-kubecfg" - }, - "name": "kubedns-kubecfg" - } - ] - } - } - }, - "status": { - "availableReplicas": 2, - "conditions": [ - { - "lastTransitionTime": "2019-07-23T14:46:03Z", - "lastUpdateTime": "2019-07-23T14:46:03Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 7, - "readyReplicas": 2, - "replicas": 2, - "updatedReplicas": 2 - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "6", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"kubernetes-dashboard\",\"kubernetes.io/cluster-service\":\"true\"},\"name\":\"kubernetes-dashboard\",\"namespace\":\"kube-system\"},\"spec\":{\"replicas\":1,\"strategy\":{\"rollingUpdate\":{\"maxSurge\":0,\"maxUnavailable\":1},\"type\":\"RollingUpdate\"},\"template\":{\"metadata\":{\"labels\":{\"k8s-app\":\"kubernetes-dashboard\",\"kubernetes.io/cluster-service\":\"true\"}},\"spec\":{\"affinity\":{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"labelSelector\":null,\"matchExpressions\":[{\"key\":\"kubernetes.azure.com/cluster\",\"operator\":\"Exists\"}]}]}}},\"containers\":[{\"image\":\"aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64:v1.10.1\",\"livenessProbe\":{\"failureThreshold\":3,\"httpGet\":{\"path\":\"/\",\"port\":9090,\"scheme\":\"HTTP\"},\"initialDelaySeconds\":30,\"periodSeconds\":10,\"successThreshold\":1,\"timeoutSeconds\":30},\"name\":\"main\",\"ports\":[{\"containerPort\":9090,\"name\":\"http\",\"protocol\":\"TCP\"}],\"resources\":{\"limits\":{\"cpu\":\"100m\",\"memory\":\"500Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"50Mi\"}}}],\"imagePullSecrets\":[{\"name\":\"emptyacrsecret\"}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\"},\"priorityClassName\":\"system-node-critical\",\"serviceAccountName\":\"kubernetes-dashboard\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"}]}}}}\n" - }, - "creationTimestamp": "2019-03-12T16:38:31Z", - "generation": 6, - "labels": { - "addonmanager.kubernetes.io/mode": "Reconcile", - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true" - }, - "name": "kubernetes-dashboard", - "namespace": "kube-system", - "resourceVersion": "15831521", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/kubernetes-dashboard", - "uid": "45b9cc8d-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 0, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true" - } - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "image": "aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64:v1.10.1", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/", - "port": 9090, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 30 - }, - "name": "main", - "ports": [ - { - "containerPort": 9090, - "name": "http", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "100m", - "memory": "500Mi" - }, - "requests": { - "cpu": "100m", - "memory": "50Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - } - ], - "dnsPolicy": "ClusterFirst", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kubernetes-dashboard", - "serviceAccountName": "kubernetes-dashboard", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ] - } - } - }, - "status": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2019-03-12T16:38:32Z", - "lastUpdateTime": "2019-03-12T16:38:32Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 6, - "readyReplicas": 1, - "replicas": 1, - "updatedReplicas": 1 - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "5", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"metrics-server\",\"kubernetes.io/cluster-service\":\"true\"},\"name\":\"metrics-server\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"metrics-server\"}},\"template\":{\"metadata\":{\"labels\":{\"k8s-app\":\"metrics-server\"},\"name\":\"metrics-server\"},\"spec\":{\"affinity\":{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"labelSelector\":null,\"matchExpressions\":[{\"key\":\"kubernetes.azure.com/cluster\",\"operator\":\"Exists\"}]}]}}},\"containers\":[{\"command\":[\"/metrics-server\",\"--source=kubernetes.summary_api:''\"],\"image\":\"aksrepos.azurecr.io/mirror/metrics-server-amd64:v0.2.1\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"metrics-server\"}],\"imagePullSecrets\":[{\"name\":\"emptyacrsecret\"}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\"},\"priorityClassName\":\"system-node-critical\",\"serviceAccountName\":\"metrics-server\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"}]}}}}\n" - }, - "creationTimestamp": "2019-03-12T16:38:31Z", - "generation": 5, - "labels": { - "addonmanager.kubernetes.io/mode": "Reconcile", - "k8s-app": "metrics-server", - "kubernetes.io/cluster-service": "true" - }, - "name": "metrics-server", - "namespace": "kube-system", - "resourceVersion": "15144043", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/metrics-server", - "uid": "45556857-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "k8s-app": "metrics-server" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "k8s-app": "metrics-server" - }, - "name": "metrics-server" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/metrics-server", - "--source=kubernetes.summary_api:''" - ], - "image": "aksrepos.azurecr.io/mirror/metrics-server-amd64:v0.2.1", - "imagePullPolicy": "IfNotPresent", - "name": "metrics-server", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - } - ], - "dnsPolicy": "ClusterFirst", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "metrics-server", - "serviceAccountName": "metrics-server", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ] - } - } - }, - "status": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2019-03-12T16:38:31Z", - "lastUpdateTime": "2019-03-12T16:38:31Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 5, - "readyReplicas": 1, - "replicas": 1, - "updatedReplicas": 1 - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "7", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"name\":\"omsagent-rs\",\"namespace\":\"kube-system\"},\"spec\":{\"replicas\":1,\"selector\":{\"matchLabels\":{\"rsName\":\"omsagent-rs\"}},\"strategy\":{\"type\":\"RollingUpdate\"},\"template\":{\"metadata\":{\"annotations\":{\"agentVersion\":\"1.10.0.1\",\"dockerProviderVersion\":\"6.0.0-0\",\"schema-versions\":\"v1\"},\"labels\":{\"rsName\":\"omsagent-rs\"}},\"spec\":{\"containers\":[{\"env\":[{\"name\":\"AKS_RESOURCE_ID\",\"value\":\"/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test\"},{\"name\":\"AKS_REGION\",\"value\":\"eastus\"},{\"name\":\"CONTROLLER_TYPE\",\"value\":\"ReplicaSet\"},{\"name\":\"NODE_IP\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"status.hostIP\"}}}],\"image\":\"mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019\",\"imagePullPolicy\":\"IfNotPresent\",\"livenessProbe\":{\"exec\":{\"command\":[\"/bin/bash\",\"-c\",\"/opt/livenessprobe.sh\"]},\"initialDelaySeconds\":60,\"periodSeconds\":60},\"name\":\"omsagent\",\"ports\":[{\"containerPort\":25225,\"protocol\":\"TCP\"},{\"containerPort\":25224,\"protocol\":\"UDP\"},{\"containerPort\":25227,\"name\":\"in-rs-tcp\",\"protocol\":\"TCP\"}],\"resources\":{\"limits\":{\"cpu\":\"150m\",\"memory\":\"500Mi\"},\"requests\":{\"cpu\":\"110m\",\"memory\":\"250Mi\"}},\"securityContext\":{\"privileged\":true},\"volumeMounts\":[{\"mountPath\":\"/var/run/host\",\"name\":\"docker-sock\"},{\"mountPath\":\"/var/log\",\"name\":\"host-log\"},{\"mountPath\":\"/var/lib/docker/containers\",\"name\":\"containerlog-path\"},{\"mountPath\":\"/etc/kubernetes/host\",\"name\":\"azure-json-path\"},{\"mountPath\":\"/etc/omsagent-secret\",\"name\":\"omsagent-secret\",\"readOnly\":true},{\"mountPath\":\"/etc/config\",\"name\":\"omsagent-rs-config\"},{\"mountPath\":\"/etc/config/settings\",\"name\":\"settings-vol-config\",\"readOnly\":true}]}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\",\"kubernetes.io/role\":\"agent\"},\"serviceAccountName\":\"omsagent\",\"volumes\":[{\"hostPath\":{\"path\":\"/var/run\"},\"name\":\"docker-sock\"},{\"hostPath\":{\"path\":\"/etc/hostname\"},\"name\":\"container-hostname\"},{\"hostPath\":{\"path\":\"/var/log\"},\"name\":\"host-log\"},{\"hostPath\":{\"path\":\"/var/lib/docker/containers\"},\"name\":\"containerlog-path\"},{\"hostPath\":{\"path\":\"/etc/kubernetes\"},\"name\":\"azure-json-path\"},{\"name\":\"omsagent-secret\",\"secret\":{\"secretName\":\"omsagent-secret\"}},{\"configMap\":{\"name\":\"omsagent-rs-config\"},\"name\":\"omsagent-rs-config\"},{\"configMap\":{\"name\":\"container-azm-ms-agentconfig\",\"optional\":true},\"name\":\"settings-vol-config\"}]}}}}\n" - }, - "creationTimestamp": "2019-08-19T22:44:22Z", - "generation": 7, - "labels": { - "rsName": "omsagent-rs" - }, - "name": "omsagent-rs", - "namespace": "kube-system", - "resourceVersion": "19063500", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/omsagent-rs", - "uid": "e32d7e82-c2d2-11e9-8736-86290fd7dd1f" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "rsName": "omsagent-rs" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "annotations": { - "agentVersion": "1.10.0.1", - "dockerProviderVersion": "6.0.0-0", - "schema-versions": "v1" - }, - "creationTimestamp": null, - "labels": { - "rsName": "omsagent-rs" - } - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "AKS_RESOURCE_ID", - "value": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test" - }, - { - "name": "AKS_REGION", - "value": "eastus" - }, - { - "name": "CONTROLLER_TYPE", - "value": "ReplicaSet" - }, - { - "name": "NODE_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/bin/bash", - "-c", - "/opt/livenessprobe.sh" - ] - }, - "failureThreshold": 3, - "initialDelaySeconds": 60, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "omsagent", - "ports": [ - { - "containerPort": 25225, - "protocol": "TCP" - }, - { - "containerPort": 25224, - "protocol": "UDP" - }, - { - "containerPort": 25227, - "name": "in-rs-tcp", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "500Mi" - }, - "requests": { - "cpu": "110m", - "memory": "250Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/host", - "name": "docker-sock" - }, - { - "mountPath": "/var/log", - "name": "host-log" - }, - { - "mountPath": "/var/lib/docker/containers", - "name": "containerlog-path" - }, - { - "mountPath": "/etc/kubernetes/host", - "name": "azure-json-path" - }, - { - "mountPath": "/etc/omsagent-secret", - "name": "omsagent-secret", - "readOnly": true - }, - { - "mountPath": "/etc/config", - "name": "omsagent-rs-config" - }, - { - "mountPath": "/etc/config/settings", - "name": "settings-vol-config", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "nodeSelector": { - "beta.kubernetes.io/os": "linux", - "kubernetes.io/role": "agent" - }, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "omsagent", - "serviceAccountName": "omsagent", - "terminationGracePeriodSeconds": 30, - "volumes": [ - { - "hostPath": { - "path": "/var/run", - "type": "" - }, - "name": "docker-sock" - }, - { - "hostPath": { - "path": "/etc/hostname", - "type": "" - }, - "name": "container-hostname" - }, - { - "hostPath": { - "path": "/var/log", - "type": "" - }, - "name": "host-log" - }, - { - "hostPath": { - "path": "/var/lib/docker/containers", - "type": "" - }, - "name": "containerlog-path" - }, - { - "hostPath": { - "path": "/etc/kubernetes", - "type": "" - }, - "name": "azure-json-path" - }, - { - "name": "omsagent-secret", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-secret" - } - }, - { - "configMap": { - "defaultMode": 420, - "name": "omsagent-rs-config" - }, - "name": "omsagent-rs-config" - }, - { - "configMap": { - "defaultMode": 420, - "name": "container-azm-ms-agentconfig", - "optional": true - }, - "name": "settings-vol-config" - } - ] - } - } - }, - "status": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2019-08-19T22:44:22Z", - "lastUpdateTime": "2019-08-19T22:44:22Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 7, - "readyReplicas": 1, - "replicas": 1, - "updatedReplicas": 1 - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": { - "deployment.kubernetes.io/revision": "9", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"extensions/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"component\":\"tunnel\",\"kubernetes.io/cluster-service\":\"true\",\"tier\":\"node\"},\"name\":\"tunnelfront\",\"namespace\":\"kube-system\"},\"spec\":{\"replicas\":1,\"selector\":{\"matchLabels\":{\"component\":\"tunnel\"}},\"template\":{\"metadata\":{\"labels\":{\"component\":\"tunnel\"}},\"spec\":{\"affinity\":{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"labelSelector\":null,\"matchExpressions\":[{\"key\":\"kubernetes.azure.com/cluster\",\"operator\":\"Exists\"}]}]}}},\"containers\":[{\"env\":[{\"name\":\"OVERRIDE_TUNNEL_SERVER_NAME\",\"value\":\"t_dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io\"},{\"name\":\"TUNNEL_CLUSTERUSER_NAME\",\"value\":\"28957308\"},{\"name\":\"TUNNELGATEWAY_SERVER_NAME\",\"value\":\"dilipr-hea-dilipr-health-te-72c8e8-0b16acad.tun.eastus.azmk8s.io\"},{\"name\":\"TUNNELGATEWAY_SSH_PORT\",\"value\":\"22\"},{\"name\":\"TUNNELGATEWAY_TLS_PORT\",\"value\":\"443\"},{\"name\":\"KUBE_CONFIG\",\"value\":\"/etc/kubernetes/kubeconfig/kubeconfig\"}],\"image\":\"aksrepos.azurecr.io/prod/hcp-tunnel-front:v1.9.2-v4.0.7\",\"imagePullPolicy\":\"IfNotPresent\",\"livenessProbe\":{\"exec\":{\"command\":[\"/lib/tunnel-front/check-tunnel-connection.sh\"]},\"failureThreshold\":12,\"initialDelaySeconds\":10,\"periodSeconds\":60},\"name\":\"tunnel-front\",\"resources\":{\"requests\":{\"cpu\":\"10m\",\"memory\":\"64Mi\"}},\"securityContext\":{\"privileged\":true},\"volumeMounts\":[{\"mountPath\":\"/etc/kubernetes/kubeconfig\",\"name\":\"kubeconfig\",\"readOnly\":true},{\"mountPath\":\"/etc/kubernetes/certs\",\"name\":\"certificates\",\"readOnly\":true}]}],\"dnsPolicy\":\"Default\",\"imagePullSecrets\":[{\"name\":\"emptyacrsecret\"}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\"},\"priorityClassName\":\"system-node-critical\",\"serviceAccountName\":\"tunnelfront\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"}],\"volumes\":[{\"configMap\":{\"name\":\"tunnelfront-kubecfg\",\"optional\":true},\"name\":\"kubeconfig\"},{\"hostPath\":{\"path\":\"/etc/kubernetes/certs\"},\"name\":\"certificates\"}]}}}}\n" - }, - "creationTimestamp": "2019-03-12T16:38:32Z", - "generation": 9, - "labels": { - "addonmanager.kubernetes.io/mode": "Reconcile", - "component": "tunnel", - "kubernetes.io/cluster-service": "true", - "tier": "node" - }, - "name": "tunnelfront", - "namespace": "kube-system", - "resourceVersion": "17628811", - "selfLink": "/apis/extensions/v1beta1/namespaces/kube-system/deployments/tunnelfront", - "uid": "45e524e6-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "progressDeadlineSeconds": 2147483647, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "component": "tunnel" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "component": "tunnel" - } - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "OVERRIDE_TUNNEL_SERVER_NAME", - "value": "t_dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "TUNNEL_CLUSTERUSER_NAME", - "value": "28957308" - }, - { - "name": "TUNNELGATEWAY_SERVER_NAME", - "value": "dilipr-hea-dilipr-health-te-72c8e8-0b16acad.tun.eastus.azmk8s.io" - }, - { - "name": "TUNNELGATEWAY_SSH_PORT", - "value": "22" - }, - { - "name": "TUNNELGATEWAY_TLS_PORT", - "value": "443" - }, - { - "name": "KUBE_CONFIG", - "value": "/etc/kubernetes/kubeconfig/kubeconfig" - } - ], - "image": "aksrepos.azurecr.io/prod/hcp-tunnel-front:v1.9.2-v4.0.7", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/lib/tunnel-front/check-tunnel-connection.sh" - ] - }, - "failureThreshold": 12, - "initialDelaySeconds": 10, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "tunnel-front", - "resources": { - "requests": { - "cpu": "10m", - "memory": "64Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/kubernetes/kubeconfig", - "name": "kubeconfig", - "readOnly": true - }, - { - "mountPath": "/etc/kubernetes/certs", - "name": "certificates", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "tunnelfront", - "serviceAccountName": "tunnelfront", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "tunnelfront-kubecfg", - "optional": true - }, - "name": "kubeconfig" - }, - { - "hostPath": { - "path": "/etc/kubernetes/certs", - "type": "" - }, - "name": "certificates" - } - ] - } - } - }, - "status": { - "availableReplicas": 1, - "conditions": [ - { - "lastTransitionTime": "2019-03-12T16:38:32Z", - "lastUpdateTime": "2019-03-12T16:38:32Z", - "message": "Deployment has minimum availability.", - "reason": "MinimumReplicasAvailable", - "status": "True", - "type": "Available" - } - ], - "observedGeneration": 9, - "readyReplicas": 1, - "replicas": 1, - "updatedReplicas": 1 - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": "" - } -} diff --git a/test/unit-tests/plugins/health/health_container_cpu_memory_aggregator_spec.rb b/test/unit-tests/plugins/health/health_container_cpu_memory_aggregator_spec.rb deleted file mode 100644 index 89eebb509..000000000 --- a/test/unit-tests/plugins/health/health_container_cpu_memory_aggregator_spec.rb +++ /dev/null @@ -1,190 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe 'HealthContainerCpuMemoryAggregator spec' do - - it 'dedupes and drops older records' do - formatted_records = JSON.parse'[{ - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "CounterName": "memoryRssBytes", - "CounterValue": 14061568, - "Timestamp": "2019-08-23T23:13:39Z" - }, - { - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/952488f3-a1f2-11e9-8b08-d602e29755d5/sidecar", - "CounterName": "memoryRssBytes", - "CounterValue": 14061568, - "Timestamp": "2019-08-23T22:13:39Z" - }]' - - resources = HealthKubernetesResources.instance - nodes = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'nodes.json'))) - pods = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'pods.json'))) - deployments = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'deployments.json'))) - - resources.pod_inventory = pods - resources.node_inventory = nodes - resources.set_replicaset_inventory(deployments) - resources.build_pod_uid_lookup #call this in in_kube_health every min - - cluster_labels = { - 'container.azm.ms/cluster-region' => 'eastus', - 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - 'container.azm.ms/cluster-name' => 'dilipr-health-test' - } - cluster_id = 'fake_cluster_id' - provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - aggregator = HealthContainerCpuMemoryAggregator.new(resources, provider) - deduped_records = aggregator.dedupe_records(formatted_records) - deduped_records.size.must_equal 1 - deduped_records[0]["Timestamp"].must_equal "2019-08-23T23:13:39Z" - end - - it 'aggregates based on container name' do - file = File.read(File.join(File.expand_path(File.dirname(__FILE__)),'cadvisor_perf.json')) - records = JSON.parse(file) - records = records.select{|record| record['DataItems'][0]['ObjectName'] == 'K8SContainer'} - formatted_records = [] - formatter = HealthContainerCpuMemoryRecordFormatter.new - records.each{|record| - formatted_record = formatter.get_record_from_cadvisor_record(record) - formatted_records.push(formatted_record) - } - - resources = HealthKubernetesResources.instance - nodes = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'nodes.json'))) - pods = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'pods.json'))) - deployments = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'deployments.json'))) - - resources.pod_inventory = pods - resources.node_inventory = nodes - resources.set_replicaset_inventory(deployments) - resources.build_pod_uid_lookup #call this in in_kube_health every min - - cluster_labels = { - 'container.azm.ms/cluster-region' => 'eastus', - 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - 'container.azm.ms/cluster-name' => 'dilipr-health-test' - } - - cluster_id = 'fake_cluster_id' - - provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - - aggregator = HealthContainerCpuMemoryAggregator.new(resources, provider) - deduped_records = aggregator.dedupe_records(formatted_records) - aggregator.aggregate(deduped_records) - aggregator.compute_state - records = aggregator.get_records - records.size.must_equal 30 - #records have all the required details - records.each{|record| - record["Details"]["details"]["container"].wont_be_nil - record["Details"]["details"]["workload_name"].wont_be_nil - record["Details"]["details"]["workload_kind"].wont_be_nil - record["Details"]["details"]["namespace"].wont_be_nil - record["Details"]["timestamp"].wont_be_nil - record["Details"]["state"].wont_be_nil - record["MonitorTypeId"].wont_be_nil - record["MonitorInstanceId"].wont_be_nil - record["TimeFirstObserved"].wont_be_nil - record["TimeGenerated"].wont_be_nil - } - end - - it "calculates the state correctly" do - file = File.read(File.join(File.expand_path(File.dirname(__FILE__)),'cadvisor_perf.json')) - records = JSON.parse(file) - records = records.select{|record| record['DataItems'][0]['ObjectName'] == 'K8SContainer'} - formatted_records = [] - formatter = HealthContainerCpuMemoryRecordFormatter.new - records.each{|record| - formatted_record = formatter.get_record_from_cadvisor_record(record) - formatted_records.push(formatted_record) - } - - resources = HealthKubernetesResources.instance - nodes = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'nodes.json'))) - pods = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'pods.json'))) - deployments = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'deployments.json'))) - - resources.pod_inventory = pods - resources.node_inventory = nodes - resources.set_replicaset_inventory(deployments) - resources.build_pod_uid_lookup #call this in in_kube_health every min - - cluster_labels = { - 'container.azm.ms/cluster-region' => 'eastus', - 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - 'container.azm.ms/cluster-name' => 'dilipr-health-test' - } - - cluster_id = 'fake_cluster_id' - - provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - - aggregator = HealthContainerCpuMemoryAggregator.new(resources, provider) - deduped_records = aggregator.dedupe_records(formatted_records) - aggregator.aggregate(deduped_records) - aggregator.compute_state - records = aggregator.get_records - - #omsagent has limit set. So its state should be set to pass. - #sidecar has no limit set. its state should be set to warning - omsagent_record = records.select{|r| r["MonitorTypeId"] == MonitorId::CONTAINER_CPU_MONITOR_ID && r["Details"]["details"]["container"] == "omsagent"}[0] - sidecar_record = records.select{|r| r["MonitorTypeId"] == MonitorId::CONTAINER_CPU_MONITOR_ID && r["Details"]["details"]["container"] == "sidecar"}[0] - omsagent_record['Details']['state'].must_equal HealthMonitorStates::PASS #limit is set - sidecar_record['Details']['state'].must_equal HealthMonitorStates::PASS - end - - - it "calculates the state as unknown when signals are missing" do - file = File.read(File.join(File.expand_path(File.dirname(__FILE__)),'cadvisor_perf.json')) - records = JSON.parse(file) - records = records.select{|record| record['DataItems'][0]['ObjectName'] == 'K8SContainer'} - formatted_records = [] - formatter = HealthContainerCpuMemoryRecordFormatter.new - records.each{|record| - formatted_record = formatter.get_record_from_cadvisor_record(record) - formatted_records.push(formatted_record) - } - - formatted_records = formatted_records.reject{|r| r["InstanceName"] == "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/69e68b21-c5df-11e9-8736-86290fd7dd1f/omsagent" && r["CounterName"] == "cpuUsageNanoCores"} - formatted_records = formatted_records.reject{|r| r["InstanceName"] == "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/b1e04e1c-c5df-11e9-8736-86290fd7dd1f/omsagent" && r["CounterName"] == "cpuUsageNanoCores"} - - resources = HealthKubernetesResources.instance - nodes = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'nodes.json'))) - pods = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'pods.json'))) - deployments = JSON.parse(File.read(File.join(File.expand_path(File.dirname(__FILE__)),'deployments.json'))) - - resources.pod_inventory = pods - resources.node_inventory = nodes - resources.set_replicaset_inventory(deployments) - resources.build_pod_uid_lookup #call this in in_kube_health every min - - cluster_labels = { - 'container.azm.ms/cluster-region' => 'eastus', - 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - 'container.azm.ms/cluster-name' => 'dilipr-health-test' - } - - cluster_id = 'fake_cluster_id' - - provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - - aggregator = HealthContainerCpuMemoryAggregator.new(resources, provider) - deduped_records = aggregator.dedupe_records(formatted_records) - aggregator.aggregate(deduped_records) - aggregator.compute_state - records = aggregator.get_records - - #removed(missed) omsagent records should result in state being unknown - omsagent_record = records.select{|r| r["MonitorTypeId"] == MonitorId::CONTAINER_CPU_MONITOR_ID && r["Details"]["details"]["container"] == "omsagent" && !r["Details"]["details"]["workload_name"].include?("omsagent-rs") }[0] - omsagent_record['Details']['state'].must_equal HealthMonitorStates::UNKNOWN #limit is set - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_container_cpu_memory_record_formatter_spec.rb b/test/unit-tests/plugins/health/health_container_cpu_memory_record_formatter_spec.rb deleted file mode 100644 index e19eb15dc..000000000 --- a/test/unit-tests/plugins/health/health_container_cpu_memory_record_formatter_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "HealthContainerCpuMemoryRecordFormatter spec" do - it 'returns the record in expected format when cadvisor record is well formed' do - formatter = HealthContainerCpuMemoryRecordFormatter.new - cadvisor_record = JSON.parse('{ - "DataItems": [ - { - "Timestamp": "2019-08-01T23:19:19Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/6708e4ac-b49a-11e9-8a49-52a94e80d897/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 85143552 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }') - record = formatter.get_record_from_cadvisor_record(cadvisor_record) - record.wont_equal nil - record["InstanceName"].must_equal "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/6708e4ac-b49a-11e9-8a49-52a94e80d897/omsagent" - record["CounterName"].must_equal "memoryWorkingSetBytes" - record["CounterValue"].must_equal 85143552 - record["Timestamp"].must_equal "2019-08-01T23:19:19Z" - end - - it 'returns nil for invalid cadvisor record' do - formatter = HealthContainerCpuMemoryRecordFormatter.new - cadvisor_record = JSON.parse('{ - "DataItms": [ - { - "Timestamp": "2019-08-01T23:19:19Z", - "Host": "aks-nodepool1-19574989-2", - "ObjectName": "K8SContainer", - "InstanceName": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test/6708e4ac-b49a-11e9-8a49-52a94e80d897/omsagent", - "Collections": [ - { - "CounterName": "memoryWorkingSetBytes", - "Value": 85143552 - } - ] - } - ], - "DataType": "LINUX_PERF_BLOB", - "IPName": "LogManagement" - }') - record = formatter.get_record_from_cadvisor_record(cadvisor_record) - record.must_be_nil - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_hierarchy_builder_spec.rb b/test/unit-tests/plugins/health/health_hierarchy_builder_spec.rb deleted file mode 100644 index 615826c03..000000000 --- a/test/unit-tests/plugins/health/health_hierarchy_builder_spec.rb +++ /dev/null @@ -1,11 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "HealthHierarchyBuilder spec" do - it 'builds right hierarchy given a child monitor and a parent monitor provider' do - - end - -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_kubernetes_resource_spec.rb b/test/unit-tests/plugins/health/health_kubernetes_resource_spec.rb deleted file mode 100644 index a1a013052..000000000 --- a/test/unit-tests/plugins/health/health_kubernetes_resource_spec.rb +++ /dev/null @@ -1,222 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe "HealthKubernetesResources spec" do - it "returns the right set of nodes and workloads given node and pod inventory" do - - #arrange - nodes_json = '{ - "items": [ - { - "metadata": { - "name": "aks-nodepool1-19574989-0" - } - }, - { - "metadata": { - "name": "aks-nodepool1-19574989-1" - } - } - ] - }' - - pods_json = '{ - "items": [ - { - "metadata": { - "name": "diliprdeploymentnodeapps-c4fdfb446-mzcsr", - "generateName": "diliprdeploymentnodeapps-c4fdfb446-", - "namespace": "default", - "selfLink": "/api/v1/namespaces/default/pods/diliprdeploymentnodeapps-c4fdfb446-mzcsr", - "uid": "ee31a9ce-526e-11e9-a899-6a5520730c61", - "resourceVersion": "4597573", - "creationTimestamp": "2019-03-29T22:06:40Z", - "labels": { - "app": "diliprsnodeapppod", - "diliprPodLabel1": "p1", - "diliprPodLabel2": "p2", - "pod-template-hash": "709896002" - }, - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "diliprdeploymentnodeapps-c4fdfb446", - "uid": "ee1e78e0-526e-11e9-a899-6a5520730c61", - "controller": true, - "blockOwnerDeletion": true - } - ] - }, - "apiVersion": "v1", - "kind": "Pod" - }, - { - "metadata": { - "name": "pi-m8ccw", - "generateName": "pi-", - "namespace": "default", - "selfLink": "/api/v1/namespaces/default/pods/pi-m8ccw", - "uid": "9fb16aaa-7ccc-11e9-8d23-32c49ee6f300", - "resourceVersion": "7940877", - "creationTimestamp": "2019-05-22T20:03:10Z", - "labels": { - "controller-uid": "9fad836f-7ccc-11e9-8d23-32c49ee6f300", - "job-name": "pi" - }, - "ownerReferences": [ - { - "apiVersion": "batch/v1", - "kind": "Job", - "name": "pi", - "uid": "9fad836f-7ccc-11e9-8d23-32c49ee6f300", - "controller": true, - "blockOwnerDeletion": true - } - ] - }, - "apiVersion": "v1", - "kind": "Pod" - }, - { - "metadata": { - "name": "rss-site", - "namespace": "default", - "selfLink": "/api/v1/namespaces/default/pods/rss-site", - "uid": "68a34ea4-7ce4-11e9-8d23-32c49ee6f300", - "resourceVersion": "7954135", - "creationTimestamp": "2019-05-22T22:53:26Z", - "labels": { - "app": "web" - }, - "annotations": { - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Pod\",\"metadata\":{\"annotations\":{},\"labels\":{\"app\":\"web\"},\"name\":\"rss-site\",\"namespace\":\"default\"},\"spec\":{\"containers\":[{\"image\":\"nginx\",\"name\":\"front-end\",\"ports\":[{\"containerPort\":80}]},{\"image\":\"nickchase/rss-php-nginx:v1\",\"name\":\"rss-reader\",\"ports\":[{\"containerPort\":88}]}]}}\n" - } - }, - "apiVersion": "v1", - "kind": "Pod" - }, - { - "metadata": { - "name": "kube-proxy-4hjws", - "generateName": "kube-proxy-", - "namespace": "kube-system", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-proxy-4hjws", - "uid": "8cf7c410-88f4-11e9-b1b0-5eb4a3e9de7d", - "resourceVersion": "9661065", - "creationTimestamp": "2019-06-07T07:19:12Z", - "labels": { - "component": "kube-proxy", - "controller-revision-hash": "1271944371", - "pod-template-generation": "16", - "tier": "node" - }, - "annotations": { - "aks.microsoft.com/release-time": "seconds:1559735217 nanos:797729016 ", - "remediator.aks.microsoft.com/kube-proxy-restart": "7" - }, - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "kind": "DaemonSet", - "name": "kube-proxy", - "uid": "45640bf6-44e5-11e9-9920-423525a6b683", - "controller": true, - "blockOwnerDeletion": true - } - ] - }, - "apiVersion": "v1", - "kind": "Pod" - } - ] - }' - deployments_json = '{ - "items": [ - { - "metadata": { - "name": "diliprdeploymentnodeapps", - "namespace": "default", - "selfLink": "/apis/extensions/v1beta1/namespaces/default/deployments/diliprdeploymentnodeapps", - "uid": "ee1b111d-526e-11e9-a899-6a5520730c61", - "resourceVersion": "4597575", - "generation": 1, - "creationTimestamp": "2019-03-29T22:06:40Z", - "labels": { - "diliprdeploymentLabel1": "d1", - "diliprdeploymentLabel2": "d2" - }, - "annotations": { - "deployment.kubernetes.io/revision": "1", - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"diliprdeploymentLabel1\":\"d1\",\"diliprdeploymentLabel2\":\"d2\"},\"name\":\"diliprdeploymentnodeapps\",\"namespace\":\"default\"},\"spec\":{\"replicas\":1,\"selector\":{\"matchLabels\":{\"app\":\"diliprsnodeapppod\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"diliprsnodeapppod\",\"diliprPodLabel1\":\"p1\",\"diliprPodLabel2\":\"p2\"}},\"spec\":{\"containers\":[{\"image\":\"rdilip83/logeverysecond:v2\",\"name\":\"diliprcontainerhelloapp\"}]}}}}\n" - } - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": { - "app": "diliprsnodeapppod" - } - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "app": "diliprsnodeapppod", - "diliprPodLabel1": "p1", - "diliprPodLabel2": "p2" - } - }, - "spec": { - "containers": [ - { - "name": "diliprcontainerhelloapp", - "image": "rdilip83/logeverysecond:v2", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "imagePullPolicy": "IfNotPresent" - } - ], - "restartPolicy": "Always", - "terminationGracePeriodSeconds": 30, - "dnsPolicy": "ClusterFirst", - "securityContext": {}, - "schedulerName": "default-scheduler" - } - }, - "strategy": { - "type": "RollingUpdate", - "rollingUpdate": { - "maxUnavailable": "25%", - "maxSurge": "25%" - } - }, - "revisionHistoryLimit": 2, - "progressDeadlineSeconds": 600 - }, - "apiVersion": "extensions/v1beta1", - "kind": "Deployment" - } - ] - }' - nodes = JSON.parse(nodes_json) - pods = JSON.parse(pods_json) - deployments = JSON.parse(deployments_json) - resources = HealthKubernetesResources.instance - resources.node_inventory = nodes - resources.pod_inventory = pods - resources.set_replicaset_inventory(deployments) - #act - parsed_nodes = resources.get_nodes - parsed_workloads = resources.get_workload_names - - #assert - assert_equal parsed_nodes.size, 2 - assert_equal parsed_workloads.size, 3 - - assert_equal parsed_nodes, ['aks-nodepool1-19574989-0', 'aks-nodepool1-19574989-1'] - parsed_workloads.sort.must_equal ['default~~diliprdeploymentnodeapps-c4fdfb446', 'default~~rss-site', 'kube-system~~kube-proxy'].sort - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_missing_signal_generator_spec.rb b/test/unit-tests/plugins/health/health_missing_signal_generator_spec.rb deleted file mode 100644 index 125d02fe0..000000000 --- a/test/unit-tests/plugins/health/health_missing_signal_generator_spec.rb +++ /dev/null @@ -1,79 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each {|file| require file} -include HealthModel -include Minitest - -describe "HealthMissingSignalGenerator spec" do - it 'generates missing node signals' do - #arrange - resources = Mock.new - resources.expect(:get_nodes, ["node1"]) - resources.expect(:get_workload_names, ["default~~workload1"]) - - provider = Mock.new - provider.expect(:get_node_labels, {HealthMonitorLabels::HOSTNAME => "node1"}, ["node1"]) - - node1_cpu_record = Mock.new - def node1_cpu_record.monitor_id; "node_cpu_utilization"; end - def node1_cpu_record.monitor_instance_id; "node_cpu_utilization"; end - def node1_cpu_record.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - def node1_cpu_record.config; {}; end - def node1_cpu_record.state; "pass"; end - - node1_memory_record = Mock.new - def node1_memory_record.monitor_id; "node_memory_utilization"; end - def node1_memory_record.monitor_instance_id; "node_memory_utilization"; end - def node1_memory_record.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - def node1_memory_record.config; {}; end - def node1_memory_record.state; "pass"; end - - node1_condition_record = Mock.new - def node1_condition_record.monitor_id; "node_condition"; end - def node1_condition_record.monitor_instance_id; "node_condition-0c593682737a955dc8e0947ad12754fe"; end - def node1_condition_record.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - def node1_condition_record.config; {}; end - def node1_condition_record.state; "pass"; end - - - workload1_pods_ready_record = Mock.new - def workload1_pods_ready_record.monitor_id; "user_workload_pods_ready"; end - def workload1_pods_ready_record.monitor_instance_id; "user_workload_pods_ready-workload1"; end - def workload1_pods_ready_record.labels; {HealthMonitorLabels::NAMESPACE => "default", HealthMonitorLabels::WORKLOAD_NAME => "workload1"}; end - def workload1_pods_ready_record.config; {}; end - def workload1_pods_ready_record.state; "pass"; end - - generator = HealthMissingSignalGenerator.new - generator.update_last_received_records([node1_cpu_record, node1_memory_record, node1_condition_record, workload1_pods_ready_record]) - - #act - missing = generator.get_missing_signals('fake_cluster_id', [node1_cpu_record, node1_memory_record], resources, provider) - - #assert - assert_equal missing.size, 2 - - assert_equal missing[0].monitor_id, "node_condition" - assert_equal missing[0].state, "unknown" - assert_equal missing[0].monitor_instance_id, "node_condition-0c593682737a955dc8e0947ad12754fe" - - assert_equal missing[1].monitor_id, "user_workload_pods_ready" - assert_equal missing[1].state, "unknown" - assert_equal missing[1].monitor_instance_id, "user_workload_pods_ready-workload1" - - #arrange - resources.expect(:get_nodes, ["node1"]) - resources.expect(:get_workload_names, ["default~~workload1"]) - provider.expect(:get_node_labels, {HealthMonitorLabels::HOSTNAME => "node1"}, ["node1"]) - generator.update_last_received_records([node1_cpu_record, node1_memory_record]) - #act - missing = generator.get_missing_signals('fake_cluster_id', [node1_cpu_record, node1_memory_record], resources, provider) - #assert - assert_equal missing.size, 2 - assert_equal missing[0].monitor_id, "node_condition" - assert_equal missing[0].state, "unknown" - assert_equal missing[0].monitor_instance_id, "node_condition-0c593682737a955dc8e0947ad12754fe" - - assert_equal missing[1].monitor_id, "user_workload_pods_ready" - assert_equal missing[1].state, "none" - assert_equal missing[1].monitor_instance_id, "user_workload_pods_ready-workload1" - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_model_buffer_spec.rb b/test/unit-tests/plugins/health/health_model_buffer_spec.rb deleted file mode 100644 index a19969082..000000000 --- a/test/unit-tests/plugins/health/health_model_buffer_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -require_relative '../../../../source/plugins/ruby/health/health_model_buffer' -require_relative '../test_helpers' - -include HealthModel - -describe "HealthModelBuffer Spec" do - it "get_buffer returns the correct buffer data" do - # Arrange - buffer = HealthModelBuffer.new - # Act - buffer.add_to_buffer(['mockRecord']) - # Assert - assert_equal buffer.get_buffer.length, 1 - - #Act - buffer.add_to_buffer(['mockRecord1', 'mockRecord2']) - #Assert - assert_equal buffer.get_buffer.length, 3 - - #Act - buffer.reset_buffer - #Assert - assert_equal buffer.get_buffer.length, 0 - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_model_builder_spec.rb b/test/unit-tests/plugins/health/health_model_builder_spec.rb deleted file mode 100644 index c21524982..000000000 --- a/test/unit-tests/plugins/health/health_model_builder_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -require_relative '../test_helpers' -# consider doing this in test_helpers.rb so that this code is common -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "HealthModelBuilder spec" do - it "Verify hierarchy builder and finalizer public methods are called" do - #arrange - mock_hierarchy_builder = Mock::new - health_record = Mock::new - mock_monitor_set = Mock::new - mock_state_finalizer = Mock::new - mock_hierarchy_builder.expect(:process_record, nil, [health_record, mock_monitor_set]) - mock_state_finalizer.expect(:finalize, {}, [mock_monitor_set]) - def mock_monitor_set.get_map; {}; end - - #act - builder = HealthModelBuilder.new(mock_hierarchy_builder, [mock_state_finalizer], mock_monitor_set) - builder.process_records([health_record]) - builder.finalize_model - #assert - assert mock_hierarchy_builder.verify - assert mock_state_finalizer.verify - end - - it "Verify finalize_model raises if state_finalizers is empty" do - #arrange - mock_hierarchy_builder = Mock.new - mock_monitor_set = Mock.new - builder = HealthModelBuilder.new(mock_hierarchy_builder, [], mock_monitor_set) - #act and assert - assert_raises do - builder.finalize_model - end - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_model_builder_test.rb b/test/unit-tests/plugins/health/health_model_builder_test.rb deleted file mode 100644 index 42f1b60a8..000000000 --- a/test/unit-tests/plugins/health/health_model_builder_test.rb +++ /dev/null @@ -1,516 +0,0 @@ -require 'test/unit' -require 'json' -# require_relative '../../../source/plugins/ruby/health' - -Dir[File.join(__dir__, '../../../../source/plugins/ruby/health', '*.rb')].each { |file| require file } - -class FilterHealthModelBuilderTest < Test::Unit::TestCase - include HealthModel - - # def test_event_stream - # #setup - # health_definition_path = File.join(__dir__, '../../../../installer/conf/health_model_definition.json') - # health_model_definition = ParentMonitorProvider.new(HealthModelDefinitionParser.new(health_definition_path).parse_file) - # monitor_factory = MonitorFactory.new - # hierarchy_builder = HealthHierarchyBuilder.new(health_model_definition, monitor_factory) - # # TODO: Figure out if we need to add NodeMonitorHierarchyReducer to the list of finalizers. For now, dont compress/optimize, since it becomes impossible to construct the model on the UX side - # state_finalizers = [AggregateMonitorStateFinalizer.new] - # monitor_set = MonitorSet.new - # model_builder = HealthModelBuilder.new(hierarchy_builder, state_finalizers, monitor_set) - - # nodes_file_map = { - # #"extra" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/extra_nodes.json", - # "first" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"first-nosecondnode" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # "second" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # "third" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"fourth" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"missing" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"kube_api_down" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # } - - # pods_file_map = { - # #"extra" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/extra_pods.json", - # "first" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"first-nosecondnode" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # "second" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # "third" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"fourth" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"missing" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"kube_api_down" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # } - - # cluster_labels = { - # 'container.azm.ms/cluster-region' => 'eastus', - # 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - # 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - # 'container.azm.ms/cluster-name' => 'dilipr-health-test' - # } - - # cluster_id = 'fake_cluster_id' - - # #test - # state = HealthMonitorState.new() - # generator = HealthMissingSignalGenerator.new - - # for scenario in ["first", "second", "third"] - # mock_data_path = File.join(__dir__, "../../../../health_records/#{scenario}_daemon_set_signals.json") - # file = File.read(mock_data_path) - # records = JSON.parse(file) - - # node_inventory = JSON.parse(File.read(nodes_file_map[scenario])) - # pod_inventory = JSON.parse(File.read(pods_file_map[scenario])) - # deployment_inventory = JSON.parse(File.read(File.join(__dir__, "../../../../inventory/deployments.json"))) - # resources = HealthKubernetesResources.instance - # resources.node_inventory = node_inventory - # resources.pod_inventory = pod_inventory - # resources.set_replicaset_inventory(deployment_inventory) - - # workload_names = resources.get_workload_names - # provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - - # health_monitor_records = [] - # records.each do |record| - # monitor_instance_id = record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] - # monitor_id = record[HealthMonitorRecordFields::MONITOR_ID] - # health_monitor_record = HealthMonitorRecord.new( - # record[HealthMonitorRecordFields::MONITOR_ID], - # record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], - # record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], - # record[HealthMonitorRecordFields::DETAILS]["state"], - # provider.get_labels(record), - # provider.get_config(monitor_id), - # record[HealthMonitorRecordFields::DETAILS] - # ) - - # state.update_state(health_monitor_record, - # provider.get_config(health_monitor_record.monitor_id) - # ) - - # # get the health state based on the monitor's operational state - # # update state calls updates the state of the monitor based on configuration and history of the the monitor records - # health_monitor_record.state = state.get_state(monitor_instance_id).new_state - # health_monitor_records.push(health_monitor_record) - # instance_state = state.get_state(monitor_instance_id) - # #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - # end - - - # #handle kube api down - # kube_api_down_handler = HealthKubeApiDownHandler.new - # health_monitor_records = kube_api_down_handler.handle_kube_api_down(health_monitor_records) - - # # Dedupe daemonset signals - # # Remove unit monitor signals for “gone” objects - # reducer = HealthSignalReducer.new() - # reduced_records = reducer.reduce_signals(health_monitor_records, resources) - - # cluster_id = 'fake_cluster_id' - - # #get the list of 'none' and 'unknown' signals - # missing_signals = generator.get_missing_signals(cluster_id, reduced_records, resources, provider) - # #update state for missing signals - # missing_signals.each{|signal| - # state.update_state(signal, - # provider.get_config(signal.monitor_id) - # ) - # } - # generator.update_last_received_records(reduced_records) - # reduced_records.push(*missing_signals) - - # # build the health model - # all_records = reduced_records - # model_builder.process_records(all_records) - # all_monitors = model_builder.finalize_model - - # # update the state for aggregate monitors (unit monitors are updated above) - # all_monitors.each{|monitor_instance_id, monitor| - # if monitor.is_aggregate_monitor - # state.update_state(monitor, - # provider.get_config(monitor.monitor_id) - # ) - # end - - # instance_state = state.get_state(monitor_instance_id) - # #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - # should_send = instance_state.should_send - - # # always send cluster monitor as a heartbeat - # if !should_send && monitor_instance_id != MonitorId::CLUSTER - # all_monitors.delete(monitor_instance_id) - # end - # } - - # records_to_send = [] - # all_monitors.keys.each{|key| - # record = provider.get_record(all_monitors[key], state) - # #puts "#{record["MonitorInstanceId"]} #{record["OldState"]} #{record["NewState"]}" - # } - - # if scenario == "first" - # assert_equal 50, all_monitors.size - # elsif scenario == "second" - # assert_equal 34, all_monitors.size - # elsif scenario == "third" - # assert_equal 5, all_monitors.size - # end - # # for each key in monitor.keys, - # # get the state from health_monitor_state - # # generate the record to send - # serializer = HealthStateSerializer.new(File.join(__dir__, '../../../../health_records\health_model_state.json')) - # serializer.serialize(state) - - # deserializer = HealthStateDeserializer.new(File.join(__dir__, '../../../../health_records\health_model_state.json')) - # deserialized_state = deserializer.deserialize - - # after_state = HealthMonitorState.new - # after_state.initialize_state(deserialized_state) - # end - # end - - # def test_event_stream_aks_engine - - # #setup - # health_definition_path = File.join(__dir__, '../../../../installer\conf\health_model_definition.json') - # health_model_definition = ParentMonitorProvider.new(HealthModelDefinitionParser.new(health_definition_path).parse_file) - # monitor_factory = MonitorFactory.new - # hierarchy_builder = HealthHierarchyBuilder.new(health_model_definition, monitor_factory) - # state_finalizers = [AggregateMonitorStateFinalizer.new] - # monitor_set = MonitorSet.new - # model_builder = HealthModelBuilder.new(hierarchy_builder, state_finalizers, monitor_set) - - # nodes_file_map = { - # #"extra" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/extra_nodes.json", - # #"first" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"first-nosecondnode" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"second" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"third" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"fourth" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"missing" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # #"kube_api_down" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/nodes.json", - # "aks-engine-1" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/aks-engine/nodes.json", - # "aks-engine-2" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/aks-engine/nodes.json", - # "aks-engine-3" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/aks-engine/nodes.json", - # } - - # pods_file_map = { - # #"extra" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/extra_pods.json", - # #"first" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"first-nosecondnode" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"second" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"third" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"fourth" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"missing" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # #"kube_api_down" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/pods.json", - # "aks-engine-1" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/aks-engine/pods.json", - # "aks-engine-2" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/aks-engine/pods.json", - # "aks-engine-3" => "C:/AzureMonitor/ContainerInsights/Docker-Provider/inventory/aks-engine/pods.json", - # } - - # cluster_labels = { - # 'container.azm.ms/cluster-region' => 'eastus', - # 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - # 'container.azm.ms/cluster-resource-group' => 'aks-engine-health', - # 'container.azm.ms/cluster-name' => 'aks-engine-health' - # } - - # cluster_id = 'fake_cluster_id' - - # #test - # state = HealthMonitorState.new() - # generator = HealthMissingSignalGenerator.new - - # for scenario in 1..3 - # mock_data_path = File.join(__dir__, "../../../../health_records/aks-engine/aks-engine-#{scenario}.json") - # file = File.read(mock_data_path) - # records = JSON.parse(file) - - # node_inventory = JSON.parse(File.read(nodes_file_map["aks-engine-#{scenario}"])) - # pod_inventory = JSON.parse(File.read(pods_file_map["aks-engine-#{scenario}"])) - # deployment_inventory = JSON.parse(File.read(File.join(__dir__, "../../../../inventory/aks-engine/deployments.json"))) - # resources = HealthKubernetesResources.instance - # resources.node_inventory = node_inventory - # resources.pod_inventory = pod_inventory - # resources.deployment_inventory = deployment_inventory - - # workload_names = resources.get_workload_names - # provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - - # health_monitor_records = [] - # records.each do |record| - # monitor_instance_id = record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] - # monitor_id = record[HealthMonitorRecordFields::MONITOR_ID] - # health_monitor_record = HealthMonitorRecord.new( - # record[HealthMonitorRecordFields::MONITOR_ID], - # record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], - # record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], - # record[HealthMonitorRecordFields::DETAILS]["state"], - # provider.get_labels(record), - # provider.get_config(monitor_id), - # record[HealthMonitorRecordFields::DETAILS] - # ) - - # state.update_state(health_monitor_record, - # provider.get_config(health_monitor_record.monitor_id) - # ) - - # # get the health state based on the monitor's operational state - # # update state calls updates the state of the monitor based on configuration and history of the the monitor records - # health_monitor_record.state = state.get_state(monitor_instance_id).new_state - # health_monitor_records.push(health_monitor_record) - # instance_state = state.get_state(monitor_instance_id) - # #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - # end - - - # #handle kube api down - # kube_api_down_handler = HealthKubeApiDownHandler.new - # health_monitor_records = kube_api_down_handler.handle_kube_api_down(health_monitor_records) - - # # Dedupe daemonset signals - # # Remove unit monitor signals for “gone” objects - # reducer = HealthSignalReducer.new() - # reduced_records = reducer.reduce_signals(health_monitor_records, resources) - - # cluster_id = 'fake_cluster_id' - - # #get the list of 'none' and 'unknown' signals - # missing_signals = generator.get_missing_signals(cluster_id, reduced_records, resources, provider) - # #update state for missing signals - # missing_signals.each{|signal| - # state.update_state(signal, - # provider.get_config(signal.monitor_id) - # ) - # } - # generator.update_last_received_records(reduced_records) - # reduced_records.push(*missing_signals) - - # # build the health model - # all_records = reduced_records - # model_builder.process_records(all_records) - # all_monitors = model_builder.finalize_model - - # # update the state for aggregate monitors (unit monitors are updated above) - # all_monitors.each{|monitor_instance_id, monitor| - # if monitor.is_aggregate_monitor - # state.update_state(monitor, - # provider.get_config(monitor.monitor_id) - # ) - # end - - # instance_state = state.get_state(monitor_instance_id) - # #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - # should_send = instance_state.should_send - - # # always send cluster monitor as a heartbeat - # if !should_send && monitor_instance_id != MonitorId::CLUSTER - # all_monitors.delete(monitor_instance_id) - # end - # } - - # records_to_send = [] - # all_monitors.keys.each{|key| - # record = provider.get_record(all_monitors[key], state) - # #puts "#{record["MonitorInstanceId"]} #{record["OldState"]} #{record["NewState"]}" - # } - - # if scenario == 1 - # assert_equal 58, all_monitors.size - # elsif scenario == 2 - # assert_equal 37, all_monitors.size - # elsif scenario == 3 - # assert_equal 6, all_monitors.size - # end - # # for each key in monitor.keys, - # # get the state from health_monitor_state - # # generate the record to send - # serializer = HealthStateSerializer.new(File.join(__dir__, '../../../../health_records\health_model_state_aks-engine.json')) - # serializer.serialize(state) - - # deserializer = HealthStateDeserializer.new(File.join(__dir__, '../../../../health_records\health_model_state_aks-engine.json')) - # deserialized_state = deserializer.deserialize - - # after_state = HealthMonitorState.new - # after_state.initialize_state(deserialized_state) - # end - # end - - # def test_container_memory_cpu_with_model - # health_definition_path = File.join(__dir__, '../../../../installer/conf/health_model_definition.json') - # health_model_definition = ParentMonitorProvider.new(HealthModelDefinitionParser.new(health_definition_path).parse_file) - # monitor_factory = MonitorFactory.new - # hierarchy_builder = HealthHierarchyBuilder.new(health_model_definition, monitor_factory) - # # TODO: Figure out if we need to add NodeMonitorHierarchyReducer to the list of finalizers. For now, dont compress/optimize, since it becomes impossible to construct the model on the UX side - # state_finalizers = [AggregateMonitorStateFinalizer.new] - # monitor_set = MonitorSet.new - # model_builder = HealthModelBuilder.new(hierarchy_builder, state_finalizers, monitor_set) - - # nodes_file_map = { - # "first" => "C:/Users/dilipr/desktop/health/container_cpu_memory/nodes.json", - # "second" => "C:/Users/dilipr/desktop/health/container_cpu_memory/nodes.json", - # "third" => "C:/Users/dilipr/desktop/health/container_cpu_memory/nodes.json", - # } - - # pods_file_map = { - # "first" => "C:/Users/dilipr/desktop/health/container_cpu_memory/pods.json", - # "second" => "C:/Users/dilipr/desktop/health/container_cpu_memory/pods.json", - # "third" => "C:/Users/dilipr/desktop/health/container_cpu_memory/pods.json", - # } - - # cluster_labels = { - # 'container.azm.ms/cluster-region' => 'eastus', - # 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - # 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - # 'container.azm.ms/cluster-name' => 'dilipr-health-test' - # } - - # cluster_id = 'fake_cluster_id' - - # #test - # state = HealthMonitorState.new() - # generator = HealthMissingSignalGenerator.new - - # mock_data_path = "C:/Users/dilipr/desktop/health/container_cpu_memory/daemonset.json" - # file = File.read(mock_data_path) - # records = JSON.parse(file) - - # node_inventory = JSON.parse(File.read("C:/Users/dilipr/desktop/health/container_cpu_memory/nodes.json")) - # pod_inventory = JSON.parse(File.read("C:/Users/dilipr/desktop/health/container_cpu_memory/pods.json")) - # deployment_inventory = JSON.parse(File.read("C:/Users/dilipr/desktop/health/container_cpu_memory/deployments.json")) - # resources = HealthKubernetesResources.instance - # resources.node_inventory = node_inventory - # resources.pod_inventory = pod_inventory - # resources.set_replicaset_inventory(deployment_inventory) - - # workload_names = resources.get_workload_names - # provider = HealthMonitorProvider.new(cluster_id, cluster_labels, resources, File.join(__dir__, "../../../../installer/conf/healthmonitorconfig.json")) - - - # #container memory cpu records - # file = File.read('C:/Users/dilipr/desktop/health/container_cpu_memory/cadvisor_perf.json') - # cadvisor_records = JSON.parse(file) - # cadvisor_records = cadvisor_records.select{|record| record['DataItems'][0]['ObjectName'] == 'K8SContainer'} - # formatted_records = [] - # formatter = HealthContainerCpuMemoryRecordFormatter.new - # cadvisor_records.each{|record| - # formatted_record = formatter.get_record_from_cadvisor_record(record) - # formatted_records.push(formatted_record) - # } - - # resources.build_pod_uid_lookup #call this in in_kube_health every min - - # cluster_labels = { - # 'container.azm.ms/cluster-region' => 'eastus', - # 'container.azm.ms/cluster-subscription-id' => '72c8e8ca-dc16-47dc-b65c-6b5875eb600a', - # 'container.azm.ms/cluster-resource-group' => 'dilipr-health-test', - # 'container.azm.ms/cluster-name' => 'dilipr-health-test' - # } - - # cluster_id = 'fake_cluster_id' - - # aggregator = HealthContainerCpuMemoryAggregator.new(resources, provider) - # deduped_records = aggregator.dedupe_records(formatted_records) - # aggregator.aggregate(deduped_records) - # aggregator.compute_state - # container_cpu_memory_records = aggregator.get_records - - # records.concat(container_cpu_memory_records) - - # health_monitor_records = [] - # records.each do |record| - # monitor_instance_id = record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] - # monitor_id = record[HealthMonitorRecordFields::MONITOR_ID] - # health_monitor_record = HealthMonitorRecord.new( - # record[HealthMonitorRecordFields::MONITOR_ID], - # record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], - # record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], - # record[HealthMonitorRecordFields::DETAILS]["state"], - # provider.get_labels(record), - # provider.get_config(monitor_id), - # record[HealthMonitorRecordFields::DETAILS] - # ) - - # state.update_state(health_monitor_record, - # provider.get_config(health_monitor_record.monitor_id) - # ) - - # # get the health state based on the monitor's operational state - # # update state calls updates the state of the monitor based on configuration and history of the the monitor records - # health_monitor_record.state = state.get_state(monitor_instance_id).new_state - # health_monitor_records.push(health_monitor_record) - # #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - # end - - # #handle kube api down - # kube_api_down_handler = HealthKubeApiDownHandler.new - # health_monitor_records = kube_api_down_handler.handle_kube_api_down(health_monitor_records) - - # # Dedupe daemonset signals - # # Remove unit monitor signals for “gone” objects - # reducer = HealthSignalReducer.new() - # reduced_records = reducer.reduce_signals(health_monitor_records, resources) - - # cluster_id = 'fake_cluster_id' - - # #get the list of 'none' and 'unknown' signals - # missing_signals = generator.get_missing_signals(cluster_id, reduced_records, resources, provider) - # #update state for missing signals - # missing_signals.each{|signal| - # state.update_state(signal, - # provider.get_config(signal.monitor_id) - # ) - # } - # generator.update_last_received_records(reduced_records) - # reduced_records.push(*missing_signals) - - # # build the health model - # all_records = reduced_records - # model_builder.process_records(all_records) - # all_monitors = model_builder.finalize_model - - # # update the state for aggregate monitors (unit monitors are updated above) - # all_monitors.each{|monitor_instance_id, monitor| - # if monitor.is_aggregate_monitor - # state.update_state(monitor, - # provider.get_config(monitor.monitor_id) - # ) - # end - - # instance_state = state.get_state(monitor_instance_id) - # #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - # should_send = instance_state.should_send - - # # always send cluster monitor as a heartbeat - # if !should_send && monitor_instance_id != MonitorId::CLUSTER - # all_monitors.delete(monitor_instance_id) - # end - # } - - # records_to_send = [] - # all_monitors.keys.each{|key| - # record = provider.get_record(all_monitors[key], state) - # #puts "#{record["MonitorInstanceId"]} #{record["OldState"]} #{record["NewState"]}" - # } - # end - - def test_get_workload_name - # node_inventory = JSON.parse(File.read("C:/AzureMonitor/ContainerInsights/Docker-Provider/test/code/plugin/health/dilipr-health-test-nodes.json")) - # pod_inventory = JSON.parse(File.read('C:/AzureMonitor/ContainerInsights/Docker-Provider/test/code/plugin/health/dilipr-health-test-pods.json')) - # replicaset_inventory = JSON.parse(File.read('C:/AzureMonitor/ContainerInsights/Docker-Provider/test/code/plugin/health/dilipr-health-test-rs.json')) - node_inventory = JSON.parse(File.read("C:/AzureMonitor/ContainerInsights/Docker-Provider/test/code/plugin/health/jobyaks2-nodes.json")) - pod_inventory = JSON.parse(File.read('C:/AzureMonitor/ContainerInsights/Docker-Provider/test/code/plugin/health/jobyaks2-pods.json')) - replicaset_inventory = JSON.parse(File.read('C:/AzureMonitor/ContainerInsights/Docker-Provider/test/code/plugin/health/jobyaks2-rs.json')) - resources = HealthKubernetesResources.instance - resources.node_inventory = node_inventory - resources.pod_inventory = pod_inventory - resources.set_replicaset_inventory(replicaset_inventory) - pod_inventory['items'].each{|pod| - workload_name = resources.get_workload_name(pod) - puts "POD #{pod['metadata']['name']} Workload Name #{workload_name}" - } - - pods_ready_hash = HealthMonitorUtils.get_pods_ready_hash(resources) - - puts JSON.pretty_generate(pods_ready_hash) - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_model_definition_parser_spec.rb b/test/unit-tests/plugins/health/health_model_definition_parser_spec.rb deleted file mode 100644 index 52a98a113..000000000 --- a/test/unit-tests/plugins/health/health_model_definition_parser_spec.rb +++ /dev/null @@ -1,24 +0,0 @@ -require_relative '../test_helpers' -# consider doing this in test_helpers.rb so that this code is common -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe "HealthModelDefinitionParser spec " do - it "parses the definition file correctly with the right conditions" do - #arrange - - parser = HealthModelDefinitionParser.new(File.join(File.expand_path(File.dirname(__FILE__)), 'test_health_model_definition.json')) - #act - model_definition = parser.parse_file - - #assert - assert_equal model_definition['conditional_monitor_id'].key?("conditions"), true - assert_equal model_definition['conditional_monitor_id']["conditions"].size, 2 - assert_equal model_definition['conditional_monitor_id'].key?("parent_monitor_id"), false - - #assert - assert_equal model_definition['monitor_id'].key?("conditions"), false - assert_equal model_definition['monitor_id'].key?("parent_monitor_id"), true - end - -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_monitor_state_spec.rb b/test/unit-tests/plugins/health/health_monitor_state_spec.rb deleted file mode 100644 index 34c61a2df..000000000 --- a/test/unit-tests/plugins/health/health_monitor_state_spec.rb +++ /dev/null @@ -1,176 +0,0 @@ -require_relative '../test_helpers' -# consider doing this in test_helpers.rb so that this code is common -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "HealthMonitorState spec" do - it 'updates should_send to true for monitors which hasnt been sent before' do - #arrange - state = HealthMonitorState.new - mock_monitor = Mock.new - def mock_monitor.state; "pass"; end - def mock_monitor.monitor_id; "monitor_id"; end - def mock_monitor.monitor_instance_id; "monitor_instance_id"; end - def mock_monitor.transition_date_time; Time.now.utc.iso8601; end - def mock_monitor.details; {"state" => "pass", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - - #act - state.update_state(mock_monitor, {}) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - monitor_state.old_state.must_equal "none" - monitor_state.new_state.must_equal "pass" - end - - it 'updates should_send to true for monitors which need no consistent state change' do - #arrange - state = HealthMonitorState.new - mock_monitor = Mock.new - def mock_monitor.state; "pass"; end - def mock_monitor.monitor_id; "monitor_id"; end - def mock_monitor.monitor_instance_id; "monitor_instance_id"; end - def mock_monitor.transition_date_time; Time.now.utc.iso8601; end - def mock_monitor.details; {"state" => "pass", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - - #act - state.update_state(mock_monitor, {}) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - monitor_state.old_state.must_equal "none" - monitor_state.new_state.must_equal "pass" - - #arrange - def mock_monitor.state; "fail"; end - def mock_monitor.details; {"state" => "fail", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - #act - state.update_state(mock_monitor, {}) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - monitor_state.old_state.must_equal "pass" - monitor_state.new_state.must_equal "fail" - end - - it 'updates should_send to false for monitors which need consistent state change and has no consistent state change' do - #arrange - state = HealthMonitorState.new - mock_monitor = Mock.new - def mock_monitor.state; "pass"; end - def mock_monitor.monitor_id; "monitor_id"; end - def mock_monitor.monitor_instance_id; "monitor_instance_id"; end - def mock_monitor.transition_date_time; Time.now.utc.iso8601; end - def mock_monitor.details; {"state" => "pass", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - - config = JSON.parse('{ - "WarnIfGreaterThanPercentage": 80.0, - "FailIfGreaterThanPercentage": 90.0, - "ConsecutiveSamplesForStateTransition": 3 - }') - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - - #arrange - def mock_monitor.state; "fail"; end - def mock_monitor.details; {"state" => "fail", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal false - end - - it 'updates should_send to true for monitors which need consistent state change and has a consistent state change' do - #arrange - state = HealthMonitorState.new - mock_monitor = Mock.new - def mock_monitor.state; "pass"; end - def mock_monitor.monitor_id; "monitor_id"; end - def mock_monitor.monitor_instance_id; "monitor_instance_id"; end - def mock_monitor.transition_date_time; Time.now.utc.iso8601; end - def mock_monitor.details; {"state" => "pass", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - - config = JSON.parse('{ - "WarnIfGreaterThanPercentage": 80.0, - "FailIfGreaterThanPercentage": 90.0, - "ConsecutiveSamplesForStateTransition": 3 - }') - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - - #arrange - def mock_monitor.state; "fail"; end - def mock_monitor.details; {"state" => "fail", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal false - - #act - state.update_state(mock_monitor, config) - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - monitor_state.old_state.must_equal "none" - monitor_state.new_state.must_equal "fail" - end - - it 'updates should_send to false for monitors which need consistent state change and has NO state change' do - #arrange - state = HealthMonitorState.new - mock_monitor = Mock.new - def mock_monitor.state; "pass"; end - def mock_monitor.monitor_id; "monitor_id"; end - def mock_monitor.monitor_instance_id; "monitor_instance_id"; end - def mock_monitor.transition_date_time; Time.now.utc.iso8601; end - def mock_monitor.details; {"state" => "pass", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - - config = JSON.parse('{ - "WarnIfGreaterThanPercentage": 80.0, - "FailIfGreaterThanPercentage": 90.0, - "ConsecutiveSamplesForStateTransition": 3 - }') - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal true - monitor_state.old_state.must_equal "none" - monitor_state.new_state.must_equal "none" - - - #arrange - def mock_monitor.state; "pass"; end - def mock_monitor.details; {"state" => "pass", "timestamp" => Time.now.utc.iso8601, "details" => {}}; end - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal false - - #act - state.update_state(mock_monitor, config) - monitor_state.should_send.must_equal true - monitor_state.old_state.must_equal "none" - monitor_state.new_state.must_equal "pass" - - #act - state.update_state(mock_monitor, config) - monitor_state = state.get_state("monitor_instance_id") - #assert - monitor_state.should_send.must_equal false - monitor_state.old_state.must_equal "none" - monitor_state.new_state.must_equal "pass" - end - -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/health_signal_reducer_spec.rb b/test/unit-tests/plugins/health/health_signal_reducer_spec.rb deleted file mode 100644 index 90f4ab352..000000000 --- a/test/unit-tests/plugins/health/health_signal_reducer_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -require_relative '../test_helpers' -# consider doing this in test_helpers.rb so that this code is common -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "HealthSignalReducer spec" do - it "returns the right set of records -- no reduction" do - #arrange - record1 = Mock.new - def record1.monitor_id; "node_cpu_utilization"; end - def record1.monitor_instance_id; "node_cpu_utilization-node1"; end - def record1.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - inventory = Mock.new - def inventory.get_nodes; ["node1"]; end - def inventory.get_workload_names; []; end - reducer = HealthSignalReducer.new - #act - reduced = reducer.reduce_signals([record1], inventory) - #Assert - assert_equal reduced.size, 1 - end - - it "returns only the latest record if multiple records are present for the same monitor" do - #arrange - record1 = Mock.new - def record1.monitor_id; "node_cpu_utilization"; end - def record1.monitor_instance_id; "node_cpu_utilization-node1"; end - def record1.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - def record1.transition_date_time; Time.now.utc.iso8601 ; end - - - record2 = Mock.new - def record2.monitor_id; "node_cpu_utilization"; end - def record2.monitor_instance_id; "node_cpu_utilization-node1"; end - def record2.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - def record2.transition_date_time; "#{Time.now.utc.iso8601}" ; end - - inventory = Mock.new - def inventory.get_nodes; ["node1"]; end - def inventory.get_workload_names; []; end - reducer = HealthSignalReducer.new - #act - reduced = reducer.reduce_signals([record1, record2], inventory) - #Assert - assert_equal reduced.size, 1 - end - - it "returns only those records if the node is present in the inventory" do - #arrange - record1 = Mock.new - def record1.monitor_id; "node_cpu_utilization"; end - def record1.monitor_instance_id; "node_cpu_utilization-node1"; end - def record1.labels; {HealthMonitorLabels::HOSTNAME => "node1"}; end - inventory = Mock.new - def inventory.get_nodes; ["node2"]; end - def inventory.get_workload_names; []; end - - #act - reducer = HealthSignalReducer.new - #assert - assert_equal reducer.reduce_signals([record1], inventory).size, 0 - end - - it "returns only those records if the workdload name is present in the inventory" do - #arrange - record1 = Mock.new - def record1.monitor_id; "user_workload_pods_ready"; end - def record1.monitor_instance_id; "user_workload_pods_ready-workload1"; end - def record1.labels; {HealthMonitorLabels::NAMESPACE => "default", HealthMonitorLabels::WORKLOAD_NAME => "workload1"}; end - def record1.transition_date_time; Time.now.utc.iso8601 ; end - - inventory = Mock.new - def inventory.get_nodes; ["node2"]; end - def inventory.get_workload_names; ["default~~workload1"]; end - reducer = HealthSignalReducer.new - - #act - reduced = reducer.reduce_signals([record1], inventory) - - #assert - assert_equal reduced.size, 1 - - #arrange - record2 = Mock.new - def record2.monitor_id; "user_workload_pods_ready"; end - def record2.monitor_instance_id; "user_workload_pods_ready-workload2"; end - def record2.labels; {HealthMonitorLabels::NAMESPACE => "default1", HealthMonitorLabels::WORKLOAD_NAME => "workload2"}; end - def record1.transition_date_time; Time.now.utc.iso8601 ; end - #act - reduced = reducer.reduce_signals([record1, record2], inventory) - #assert - assert_equal reduced.size, 1 - end - -end diff --git a/test/unit-tests/plugins/health/kube_api_down_handler_spec.rb b/test/unit-tests/plugins/health/kube_api_down_handler_spec.rb deleted file mode 100644 index 5ace7c724..000000000 --- a/test/unit-tests/plugins/health/kube_api_down_handler_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe "KubeApiDownHandler spec" do - it "updates states for monitors in monitors_to_change" do - #arrange - record1 = HealthMonitorRecord.new("node_condition", "node_condition-node1", Time.now.utc.iso8601, "pass", {}, {}, {}) - record2 = HealthMonitorRecord.new("kube_api_status", "kube_api_status", Time.now.utc.iso8601, "fail", {}, {}, {}) - record3 = HealthMonitorRecord.new("user_workload_pods_ready", "user_workload_pods_ready-workload1", Time.now.utc.iso8601, "pass", {}, {}, {}) - record4 = HealthMonitorRecord.new("system_workload_pods_ready", "system_workload_pods_ready-workload2", Time.now.utc.iso8601, "pass", {}, {}, {}) - record5 = HealthMonitorRecord.new("subscribed_capacity_cpu", "subscribed_capacity_cpu", Time.now.utc.iso8601, "pass", {}, {}, {}) - record6 = HealthMonitorRecord.new("subscribed_capacity_memory", "subscribed_capacity_memory", Time.now.utc.iso8601, "pass", {}, {}, {}) - handler = HealthKubeApiDownHandler.new - - #act - handler.handle_kube_api_down([record1, record2, record3, record4, record5, record6]) - #assert - assert_equal record1.state, HealthMonitorStates::UNKNOWN - assert_equal record3.state, HealthMonitorStates::UNKNOWN - assert_equal record4.state, HealthMonitorStates::UNKNOWN - assert_equal record5.state, HealthMonitorStates::UNKNOWN - assert_equal record6.state, HealthMonitorStates::UNKNOWN - - end -end diff --git a/test/unit-tests/plugins/health/monitor_factory_spec.rb b/test/unit-tests/plugins/health/monitor_factory_spec.rb deleted file mode 100644 index 4d4ac5b31..000000000 --- a/test/unit-tests/plugins/health/monitor_factory_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -require_relative '../test_helpers' -# consider doing this in test_helpers.rb so that this code is common -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe "MonitorFactory Spec" do - it "returns UnitMonitor for create_unit_monitor" do - #Arrange - factory = MonitorFactory.new() - monitor_record = HealthMonitorRecord.new(:monitor_id, :monitor_instance_id, :time, :pass, {}, {}, {}) - #act - monitor = factory.create_unit_monitor(monitor_record) - # assert - monitor.must_be_kind_of(UnitMonitor) - end - - it "returns AggregateMonitor for create_aggregate_monitor" do - #arrange - factory = MonitorFactory.new() - mock = Minitest::Mock.new - def mock.state; :pass; end - def mock.transition_date_time; :time; end - #act - monitor = factory.create_aggregate_monitor(:monitor_id, :monitor_instance_id, :pass, {}, {}, mock) - #assert - monitor.must_be_kind_of(AggregateMonitor) - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/monitor_set_spec.rb b/test/unit-tests/plugins/health/monitor_set_spec.rb deleted file mode 100644 index b5cd01f50..000000000 --- a/test/unit-tests/plugins/health/monitor_set_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -require_relative '../test_helpers' -# consider doing this in test_helpers.rb so that this code is common -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel - -describe "MonitorSet Spec" do - it "add_or_update -- adds a monitor" do - #arrange - set = MonitorSet.new - mock_monitor = MiniTest::Mock.new - def mock_monitor.monitor_instance_id; "monitor_instance_id_1"; end - def mock_monitor.state; :pass;end - #act - set.add_or_update(mock_monitor) - #assert - assert_equal set.get_map.size, 1 - assert_equal set.get_map.key?("monitor_instance_id_1"), true - end - - it "add_or_update -- updates a monitor" do - #arrange - set = MonitorSet.new - mock_monitor = MiniTest::Mock.new - def mock_monitor.monitor_instance_id; "monitor_instance_id_1"; end - def mock_monitor.state; :pass;end - #act - set.add_or_update(mock_monitor) - #assert - assert_equal set.get_map["monitor_instance_id_1"].state, :pass - - #act - def mock_monitor.state; :fail;end - set.add_or_update(mock_monitor) - #assert - assert_equal set.get_map["monitor_instance_id_1"].state, :fail - end - - it "delete -- delete a monitor" do - #arrange - set = MonitorSet.new - mock_monitor = MiniTest::Mock.new - def mock_monitor.monitor_instance_id; "monitor_instance_id_1"; end - def mock_monitor.state; :pass;end - set.add_or_update(mock_monitor) - - #act - set.delete("monitor_instance_id_1") - #assert - assert_equal set.get_map.size, 0 - end - - it "get_map -- returns a hash" do - #arrange - set = MonitorSet.new - #act and assert - set.get_map.must_be_kind_of(Hash) - end -end diff --git a/test/unit-tests/plugins/health/nodes.json b/test/unit-tests/plugins/health/nodes.json deleted file mode 100644 index f1721e076..000000000 --- a/test/unit-tests/plugins/health/nodes.json +++ /dev/null @@ -1,1966 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "annotations": { - "node.alpha.kubernetes.io/ttl": "0", - "volumes.kubernetes.io/controller-managed-attach-detach": "true" - }, - "creationTimestamp": "2019-03-12T16:40:36Z", - "labels": { - "agentpool": "nodepool1", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/instance-type": "Standard_DS1_v2", - "beta.kubernetes.io/os": "linux", - "failure-domain.beta.kubernetes.io/region": "eastus", - "failure-domain.beta.kubernetes.io/zone": "0", - "kubernetes.azure.com/cluster": "MC_dilipr-health-test_dilipr-health-test_eastus", - "kubernetes.io/hostname": "aks-nodepool1-19574989-0", - "kubernetes.io/role": "agent", - "node-role.kubernetes.io/agent": "", - "storageprofile": "managed", - "storagetier": "Premium_LRS" - }, - "name": "aks-nodepool1-19574989-0", - "resourceVersion": "19068106", - "selfLink": "/api/v1/nodes/aks-nodepool1-19574989-0", - "uid": "9012b16c-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "podCIDR": "10.244.1.0/24", - "providerID": "azure:///subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/MC_dilipr-health-test_dilipr-health-test_eastus/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-19574989-0" - }, - "status": { - "addresses": [ - { - "address": "aks-nodepool1-19574989-0", - "type": "Hostname" - }, - { - "address": "10.240.0.4", - "type": "InternalIP" - } - ], - "allocatable": { - "cpu": "940m", - "ephemeral-storage": "28043041951", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "2504708Ki", - "pods": "110" - }, - "capacity": { - "cpu": "1", - "ephemeral-storage": "30428648Ki", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "3524612Ki", - "pods": "110" - }, - "conditions": [ - { - "lastHeartbeatTime": "2019-03-12T16:42:18Z", - "lastTransitionTime": "2019-03-12T16:42:18Z", - "message": "RouteController created a route", - "reason": "RouteCreated", - "status": "False", - "type": "NetworkUnavailable" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-07-29T08:16:01Z", - "message": "kubelet has sufficient disk space available", - "reason": "KubeletHasSufficientDisk", - "status": "False", - "type": "OutOfDisk" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-07-29T08:16:01Z", - "message": "kubelet has sufficient memory available", - "reason": "KubeletHasSufficientMemory", - "status": "False", - "type": "MemoryPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-07-29T08:16:01Z", - "message": "kubelet has no disk pressure", - "reason": "KubeletHasNoDiskPressure", - "status": "False", - "type": "DiskPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-03-12T16:40:36Z", - "message": "kubelet has sufficient PID available", - "reason": "KubeletHasSufficientPID", - "status": "False", - "type": "PIDPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-07-29T08:16:01Z", - "message": "kubelet is posting ready status. AppArmor enabled", - "reason": "KubeletReady", - "status": "True", - "type": "Ready" - } - ], - "daemonEndpoints": { - "kubeletEndpoint": { - "Port": 10250 - } - }, - "images": [ - { - "names": [ - "nickchase/rss-php-nginx@sha256:48da56a77fe4ecff4917121365d8e0ce615ebbdfe31f48a996255f5592894e2b", - "nickchase/rss-php-nginx:v1" - ], - "sizeBytes": 677038498 - }, - { - "names": [ - "rdilip83/logeverysecond@sha256:6fe5624808609c507178a77f94384fb9794a4d6b7d102ed8016a4baf608164a1", - "rdilip83/logeverysecond:v2" - ], - "sizeBytes": 674931590 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "k8s.gcr.io/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "k8s.gcr.io/hyperkube-amd64:v1.11.8" - ], - "sizeBytes": 615263658 - }, - { - "names": [ - "microsoft/oms@sha256:de83d1df24cb86a3a3110bd03abbd5704d7a7345565b1996f49ff001a3665385", - "microsoft/oms:healthpreview04262019" - ], - "sizeBytes": 514907213 - }, - { - "names": [ - "rdilip83/fixrubyerror@sha256:6b7f36cf6258b311015493ab025f06577d758c45bc5010d022ac160b9f40ea5d", - "rdilip83/fixrubyerror:latest" - ], - "sizeBytes": 494068028 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019" - ], - "sizeBytes": 494067935 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:fb2b90ce9bf7186fd9dfae97f5f72f9b9c80c8a0493af3cff74179cd4ff847c0", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08212019" - ], - "sizeBytes": 494067572 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c646e180483d295ffac114fb9df513db02553af7879681814d5910764653dd2d", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08202019" - ], - "sizeBytes": 494067210 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c21b596a22a1338ed293d01681f327acc871ee502ed779ec1109d6a93375bb3b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08192019" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "rdilip83/cifeatureprod08192019@sha256:7815bba9a805e4e8df33356fd532671de45525ce9c6e936e14f9b126e2097ecd", - "rdilip83/cifeatureprod08192019:v1" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:59e34aab9f6e16a87e880b1ee1c9dd5434ee40dd29502e74aceefabf51443717", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:internaltesthealth08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:6387d0dedf4de0bab430f681ef61361f63a20e1c4c287a9b60ea5460283ac6cf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ci_feature_prod_health08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/hc08192019@sha256:014d936771508d499ac4c15043e23b16bce8de0019fb2048b99540cbe9084895", - "rdilip83/hc08192019:1" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/health-rc@sha256:8ad12bce5ffd27b301bc6fe4355c8affa6fce080ae7e2291dec3a0ed11bb9483", - "rdilip83/health-rc:3" - ], - "sizeBytes": 494052863 - }, - { - "names": [ - "rdilip83/health_ci_feature_image@sha256:1a574d25884483083e8cbaacbf0cb7c4e442dc736d480615c65f5c71f8969b13", - "rdilip83/health_ci_feature_image:v1" - ], - "sizeBytes": 494052147 - }, - { - "names": [ - "rdilip83/healthrc@sha256:816c8cef09822daf050a0fca6f92e7ac19147ff4bf1a823d43fe70f73470cc0c", - "rdilip83/healthrc:v3" - ], - "sizeBytes": 494052138 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:d35aac044d1adc3d02269fde78f8dfd923db94b81288447cf6fdd482970a333b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthmerge08142019" - ], - "sizeBytes": 494052135 - }, - { - "names": [ - "rdilip83/healthrc@sha256:a130780e56ac0edb3ca29477e12edd5e9b5d08b5732dbd59ede9beb58e21eca7", - "rdilip83/healthrc:v2" - ], - "sizeBytes": 494051682 - }, - { - "names": [ - "rdilip83/healthmerge@sha256:24d270b0f59fb484c283922474736c3cba50f8aad0270bc0a3acd14284694eea", - "rdilip83/healthmerge:v8" - ], - "sizeBytes": 494010139 - }, - { - "names": [ - "rdilip83/health-rc@sha256:b1d24728eb808d301da426b76b7f7b79606204c4c2b695a24ac670be8276d55d", - "rdilip83/health-rc:1" - ], - "sizeBytes": 494000891 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:a0666957cccbfdf5784accd1133408bf017c28a6e694d9a2ae74da94eef2d285", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - ], - "sizeBytes": 493994261 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:32c9b35a6809c54d5296e2ca2b122b35a4ad8c852622174cc5a9f92cc27e56e4", - "rdilip83/mergehealth:v3" - ], - "sizeBytes": 493988815 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:a3521e8f36e007b3cb949e0356a75394ac61fd2024ca1ec4827b8d54fb068534", - "rdilip83/mergehealth:v1" - ], - "sizeBytes": 493981585 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0438e4690e042b195917e160b8949aeb339520ee19c898a8bb9452f36d1f84f1", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview07182019" - ], - "sizeBytes": 493977357 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:9ebc410a36856176921dba81b5bd43132469209b315f52be346690435419b9bb" - ], - "sizeBytes": 493946790 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:4e51195a9c77bd166fc90ee5f6143a4604b502ab7ef0f06431dec10c341b10f3", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview06272019" - ], - "sizeBytes": 493893635 - }, - { - "names": [ - "rdilip83/healthpreview06272019@sha256:d888ba5ff5e5810113a32f9c9812a5e28088cc81b902e95a185fe465a514029c", - "rdilip83/healthpreview06272019:latest" - ], - "sizeBytes": 493893633 - }, - { - "names": [ - "rdilip83/healthpreview06252019-1@sha256:1561876cffe94433a569f29f5231548e039193ebaa7ec640d22439675179e43f", - "rdilip83/healthpreview06252019-1:latest" - ], - "sizeBytes": 493887387 - }, - { - "names": [ - "rdilip83/healthpreview06252019@sha256:6597ff599a78ac452a4138dedb9e08c0ccd3e8b01594b033fd78ba9dbb41fe9e", - "rdilip83/healthpreview06252019:latest" - ], - "sizeBytes": 493887384 - }, - { - "names": [ - "rdilip83/healthpreview06242019@sha256:c4f565d92086d1ee56e6016178fed5c668352dc0ca0047f02910bdcb87a482c4", - "rdilip83/healthpreview06242019:latest" - ], - "sizeBytes": 493850850 - }, - { - "names": [ - "rdilip83/healthpreview06212019-1@sha256:937ce5801a0097a1cbc4eff5399c1973b4c6223ece9279b35207368b99f82b96", - "rdilip83/healthpreview06212019-1:latest" - ], - "sizeBytes": 493850674 - }, - { - "names": [ - "rdilip83/healthpreview06192019@sha256:f92cb5283814d446f0acde6a489648ea197496d5f85b27ca959ec97bce742d8a", - "rdilip83/healthpreview06192019:latest" - ], - "sizeBytes": 493799437 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0f798cb7d56931b231f71e38e7fa5bf898b69e611247a566701f70a5f29a9799", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod07092019" - ], - "sizeBytes": 467692116 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:3734a084fa9681c7e930eb90cad45a8f282c24af63065a720a2327b1683f3ba4", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06142019" - ], - "sizeBytes": 466882569 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:16402c34e2d7de72c2ebc18ec8e9f7933fa25f6a7f83bceb84483ba95e3902f7", - "rdilip83/mergehealth:v2" - ], - "sizeBytes": 448931997 - }, - { - "names": [ - "rdilip83/healthpreview06212019@sha256:5860c9caaf544f2e7c46edad5cdfb69e22398e20dc87cb8a4cd630b5b7000074", - "rdilip83/healthpreview06212019:latest" - ], - "sizeBytes": 448366491 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/hcp-tunnel-front@sha256:68878ee3ea1781b322ea3952c3370e31dd89be8bb0864e2bf27bdba6dc904c41", - "aksrepos.azurecr.io/prod/hcp-tunnel-front@sha256:68878ee3ea1781b322ea3952c3370e31dd89be8bb0864e2bf27bdba6dc904c41", - "aksrepos.azurecr.io/mirror/hcp-tunnel-front:v1.9.2-v4.0.7", - "aksrepos.azurecr.io/prod/hcp-tunnel-front:v1.9.2-v4.0.7" - ], - "sizeBytes": 383483267 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64@sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747", - "k8s.gcr.io/kubernetes-dashboard-amd64@sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747", - "aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64:v1.10.1", - "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1" - ], - "sizeBytes": 121711221 - }, - { - "names": [ - "nginx@sha256:23b4dcdf0d34d4a129755fc6f52e1c6e23bb34ea011b315d87e193033bcd1b68" - ], - "sizeBytes": 109331233 - }, - { - "names": [ - "nginx@sha256:bdbf36b7f1f77ffe7bd2a32e59235dff6ecf131e3b6b5b96061c652f30685f3a", - "nginx:latest" - ], - "sizeBytes": 109258867 - }, - { - "names": [ - "nginx@sha256:b73f527d86e3461fd652f62cf47e7b375196063bbbd503e853af5be16597cb2e", - "nginx:1.15.5" - ], - "sizeBytes": 109083698 - }, - { - "names": [ - "debian@sha256:118cf8f3557e1ea766c02f36f05f6ac3e63628427ea8965fb861be904ec35a6f", - "debian:latest" - ], - "sizeBytes": 100594230 - }, - { - "names": [ - "nginx@sha256:e3456c851a152494c3e4ff5fcc26f240206abac0c9d794affb40e0714846c451", - "nginx:1.7.9" - ], - "sizeBytes": 91664166 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/prod/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "deis/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/mirror/kube-svc-redirect:v1.0.2", - "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2" - ], - "sizeBytes": 82897218 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/heapster-amd64@sha256:fc33c690a3a446de5abc24b048b88050810a58b9e4477fa763a43d7df029301a", - "k8s.gcr.io/heapster-amd64@sha256:fc33c690a3a446de5abc24b048b88050810a58b9e4477fa763a43d7df029301a", - "aksrepos.azurecr.io/mirror/heapster-amd64:v1.5.3", - "k8s.gcr.io/heapster-amd64:v1.5.3" - ], - "sizeBytes": 75318342 - }, - { - "names": [ - "vishiy/hello@sha256:99d60766e39df52d28fe8db9c659633d96ba1d84fd672298dce047d8a86c478a", - "vishiy/hello:err100eps" - ], - "sizeBytes": 54649865 - }, - { - "names": [ - "k8s.gcr.io/k8s-dns-kube-dns-amd64@sha256:618a82fa66cf0c75e4753369a6999032372be7308866fc9afb381789b1e5ad52", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.13" - ], - "sizeBytes": 51157394 - }, - { - "names": [ - "k8s.gcr.io/metrics-server-amd64@sha256:49a9f12f7067d11f42c803dbe61ed2c1299959ad85cb315b25ff7eef8e6b8892", - "k8s.gcr.io/metrics-server-amd64:v0.2.1" - ], - "sizeBytes": 42541759 - }, - { - "names": [ - "k8s.gcr.io/k8s-dns-sidecar-amd64@sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10" - ], - "sizeBytes": 41635309 - }, - { - "names": [ - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64@sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10" - ], - "sizeBytes": 40372149 - } - ], - "nodeInfo": { - "architecture": "amd64", - "bootID": "d8f6c00f-a085-450e-bf5c-12e651a0fcfc", - "containerRuntimeVersion": "docker://3.0.4", - "kernelVersion": "4.15.0-1037-azure", - "kubeProxyVersion": "v1.11.8", - "kubeletVersion": "v1.11.8", - "machineID": "cc9ed99e383540a4b0379995bb779221", - "operatingSystem": "linux", - "osImage": "Ubuntu 16.04.5 LTS", - "systemUUID": "301B3B88-C7BD-3D45-A3CB-3CD66A42EB6F" - } - } - }, - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "annotations": { - "node.alpha.kubernetes.io/ttl": "0", - "volumes.kubernetes.io/controller-managed-attach-detach": "true" - }, - "creationTimestamp": "2019-03-12T16:40:33Z", - "labels": { - "agentpool": "nodepool1", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/instance-type": "Standard_DS1_v2", - "beta.kubernetes.io/os": "linux", - "failure-domain.beta.kubernetes.io/region": "eastus", - "failure-domain.beta.kubernetes.io/zone": "1", - "kubernetes.azure.com/cluster": "MC_dilipr-health-test_dilipr-health-test_eastus", - "kubernetes.io/hostname": "aks-nodepool1-19574989-1", - "kubernetes.io/role": "agent", - "node-role.kubernetes.io/agent": "", - "storageprofile": "managed", - "storagetier": "Premium_LRS" - }, - "name": "aks-nodepool1-19574989-1", - "resourceVersion": "19068104", - "selfLink": "/api/v1/nodes/aks-nodepool1-19574989-1", - "uid": "8e1b5c77-44e5-11e9-9920-423525a6b683" - }, - "spec": { - "podCIDR": "10.244.0.0/24", - "providerID": "azure:///subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/MC_dilipr-health-test_dilipr-health-test_eastus/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-19574989-1" - }, - "status": { - "addresses": [ - { - "address": "aks-nodepool1-19574989-1", - "type": "Hostname" - }, - { - "address": "10.240.0.5", - "type": "InternalIP" - } - ], - "allocatable": { - "cpu": "940m", - "ephemeral-storage": "28043041951", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "2504708Ki", - "pods": "110" - }, - "capacity": { - "cpu": "1", - "ephemeral-storage": "30428648Ki", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "3524612Ki", - "pods": "110" - }, - "conditions": [ - { - "lastHeartbeatTime": "2019-03-12T16:42:30Z", - "lastTransitionTime": "2019-03-12T16:42:30Z", - "message": "RouteController created a route", - "reason": "RouteCreated", - "status": "False", - "type": "NetworkUnavailable" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:21Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet has sufficient disk space available", - "reason": "KubeletHasSufficientDisk", - "status": "False", - "type": "OutOfDisk" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:21Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet has sufficient memory available", - "reason": "KubeletHasSufficientMemory", - "status": "False", - "type": "MemoryPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:21Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet has no disk pressure", - "reason": "KubeletHasNoDiskPressure", - "status": "False", - "type": "DiskPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:21Z", - "lastTransitionTime": "2019-03-12T16:40:33Z", - "message": "kubelet has sufficient PID available", - "reason": "KubeletHasSufficientPID", - "status": "False", - "type": "PIDPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:21Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet is posting ready status. AppArmor enabled", - "reason": "KubeletReady", - "status": "True", - "type": "Ready" - } - ], - "daemonEndpoints": { - "kubeletEndpoint": { - "Port": 10250 - } - }, - "images": [ - { - "names": [ - "perl@sha256:268e7af9853bcc6d2100e2ad76e928c2ca861518217c269b8a762849a8617c12", - "perl:latest" - ], - "sizeBytes": 890592834 - }, - { - "names": [ - "nickchase/rss-php-nginx@sha256:48da56a77fe4ecff4917121365d8e0ce615ebbdfe31f48a996255f5592894e2b", - "nickchase/rss-php-nginx:v1" - ], - "sizeBytes": 677038498 - }, - { - "names": [ - "rdilip83/jsonlogger@sha256:82b67ca5e0650cd5e47f5b51659d61cee035e5d8dcd8a79c50358cd2beb3b5a8", - "rdilip83/jsonlogger:v12" - ], - "sizeBytes": 676594134 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "k8s.gcr.io/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "k8s.gcr.io/hyperkube-amd64:v1.11.8" - ], - "sizeBytes": 615263658 - }, - { - "names": [ - "rdilip83/fixrubyerror@sha256:6b7f36cf6258b311015493ab025f06577d758c45bc5010d022ac160b9f40ea5d", - "rdilip83/fixrubyerror:latest" - ], - "sizeBytes": 494068028 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019" - ], - "sizeBytes": 494067935 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:fb2b90ce9bf7186fd9dfae97f5f72f9b9c80c8a0493af3cff74179cd4ff847c0", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08212019" - ], - "sizeBytes": 494067572 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c646e180483d295ffac114fb9df513db02553af7879681814d5910764653dd2d", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08202019" - ], - "sizeBytes": 494067210 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c21b596a22a1338ed293d01681f327acc871ee502ed779ec1109d6a93375bb3b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08192019" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "rdilip83/cifeatureprod08192019@sha256:7815bba9a805e4e8df33356fd532671de45525ce9c6e936e14f9b126e2097ecd", - "rdilip83/cifeatureprod08192019:v1" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:59e34aab9f6e16a87e880b1ee1c9dd5434ee40dd29502e74aceefabf51443717", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:internaltesthealth08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:6387d0dedf4de0bab430f681ef61361f63a20e1c4c287a9b60ea5460283ac6cf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ci_feature_prod_health08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/hc08192019@sha256:014d936771508d499ac4c15043e23b16bce8de0019fb2048b99540cbe9084895", - "rdilip83/hc08192019:1" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/health-rc@sha256:8ad12bce5ffd27b301bc6fe4355c8affa6fce080ae7e2291dec3a0ed11bb9483", - "rdilip83/health-rc:3" - ], - "sizeBytes": 494052863 - }, - { - "names": [ - "rdilip83/health_ci_feature_image@sha256:1a574d25884483083e8cbaacbf0cb7c4e442dc736d480615c65f5c71f8969b13", - "rdilip83/health_ci_feature_image:v1" - ], - "sizeBytes": 494052147 - }, - { - "names": [ - "rdilip83/healthrc@sha256:816c8cef09822daf050a0fca6f92e7ac19147ff4bf1a823d43fe70f73470cc0c", - "rdilip83/healthrc:v3" - ], - "sizeBytes": 494052138 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:d35aac044d1adc3d02269fde78f8dfd923db94b81288447cf6fdd482970a333b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthmerge08142019" - ], - "sizeBytes": 494052135 - }, - { - "names": [ - "rdilip83/healthrc@sha256:a130780e56ac0edb3ca29477e12edd5e9b5d08b5732dbd59ede9beb58e21eca7", - "rdilip83/healthrc:v2" - ], - "sizeBytes": 494051682 - }, - { - "names": [ - "rdilip83/healthmerge@sha256:24d270b0f59fb484c283922474736c3cba50f8aad0270bc0a3acd14284694eea", - "rdilip83/healthmerge:v8" - ], - "sizeBytes": 494010139 - }, - { - "names": [ - "rdilip83/health-rc@sha256:b1d24728eb808d301da426b76b7f7b79606204c4c2b695a24ac670be8276d55d", - "rdilip83/health-rc:1" - ], - "sizeBytes": 494000891 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:a0666957cccbfdf5784accd1133408bf017c28a6e694d9a2ae74da94eef2d285", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - ], - "sizeBytes": 493994261 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:32c9b35a6809c54d5296e2ca2b122b35a4ad8c852622174cc5a9f92cc27e56e4", - "rdilip83/mergehealth:v3" - ], - "sizeBytes": 493988815 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:a3521e8f36e007b3cb949e0356a75394ac61fd2024ca1ec4827b8d54fb068534", - "rdilip83/mergehealth:v1" - ], - "sizeBytes": 493981585 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0438e4690e042b195917e160b8949aeb339520ee19c898a8bb9452f36d1f84f1", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview07182019" - ], - "sizeBytes": 493977357 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:9ebc410a36856176921dba81b5bd43132469209b315f52be346690435419b9bb" - ], - "sizeBytes": 493946790 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:4e51195a9c77bd166fc90ee5f6143a4604b502ab7ef0f06431dec10c341b10f3", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview06272019" - ], - "sizeBytes": 493893635 - }, - { - "names": [ - "rdilip83/healthpreview06272019@sha256:d888ba5ff5e5810113a32f9c9812a5e28088cc81b902e95a185fe465a514029c", - "rdilip83/healthpreview06272019:latest" - ], - "sizeBytes": 493893633 - }, - { - "names": [ - "rdilip83/healthpreview06252019-1@sha256:1561876cffe94433a569f29f5231548e039193ebaa7ec640d22439675179e43f", - "rdilip83/healthpreview06252019-1:latest" - ], - "sizeBytes": 493887387 - }, - { - "names": [ - "rdilip83/healthpreview06252019@sha256:6597ff599a78ac452a4138dedb9e08c0ccd3e8b01594b033fd78ba9dbb41fe9e", - "rdilip83/healthpreview06252019:latest" - ], - "sizeBytes": 493887384 - }, - { - "names": [ - "rdilip83/healthpreview06242019@sha256:c4f565d92086d1ee56e6016178fed5c668352dc0ca0047f02910bdcb87a482c4", - "rdilip83/healthpreview06242019:latest" - ], - "sizeBytes": 493850850 - }, - { - "names": [ - "rdilip83/healthpreview06212019-1@sha256:937ce5801a0097a1cbc4eff5399c1973b4c6223ece9279b35207368b99f82b96", - "rdilip83/healthpreview06212019-1:latest" - ], - "sizeBytes": 493850674 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0f798cb7d56931b231f71e38e7fa5bf898b69e611247a566701f70a5f29a9799", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod07092019" - ], - "sizeBytes": 467692116 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:3734a084fa9681c7e930eb90cad45a8f282c24af63065a720a2327b1683f3ba4", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06142019" - ], - "sizeBytes": 466882569 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:16402c34e2d7de72c2ebc18ec8e9f7933fa25f6a7f83bceb84483ba95e3902f7", - "rdilip83/mergehealth:v2" - ], - "sizeBytes": 448931997 - }, - { - "names": [ - "deis/hcp-tunnel-front@sha256:68878ee3ea1781b322ea3952c3370e31dd89be8bb0864e2bf27bdba6dc904c41", - "deis/hcp-tunnel-front:v1.9.2-v4.0.7" - ], - "sizeBytes": 383483267 - }, - { - "names": [ - "nginx@sha256:23b4dcdf0d34d4a129755fc6f52e1c6e23bb34ea011b315d87e193033bcd1b68" - ], - "sizeBytes": 109331233 - }, - { - "names": [ - "nginx@sha256:bdbf36b7f1f77ffe7bd2a32e59235dff6ecf131e3b6b5b96061c652f30685f3a", - "nginx:latest" - ], - "sizeBytes": 109258867 - }, - { - "names": [ - "debian@sha256:118cf8f3557e1ea766c02f36f05f6ac3e63628427ea8965fb861be904ec35a6f", - "debian:latest" - ], - "sizeBytes": 100594230 - }, - { - "names": [ - "nginx@sha256:e3456c851a152494c3e4ff5fcc26f240206abac0c9d794affb40e0714846c451", - "nginx:1.7.9" - ], - "sizeBytes": 91664166 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/prod/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "deis/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/mirror/kube-svc-redirect:v1.0.2", - "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2" - ], - "sizeBytes": 82897218 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/heapster-amd64@sha256:fc33c690a3a446de5abc24b048b88050810a58b9e4477fa763a43d7df029301a", - "k8s.gcr.io/heapster-amd64@sha256:fc33c690a3a446de5abc24b048b88050810a58b9e4477fa763a43d7df029301a", - "aksrepos.azurecr.io/mirror/heapster-amd64:v1.5.3", - "k8s.gcr.io/heapster-amd64:v1.5.3" - ], - "sizeBytes": 75318342 - }, - { - "names": [ - "vishiy/hello@sha256:99d60766e39df52d28fe8db9c659633d96ba1d84fd672298dce047d8a86c478a", - "vishiy/hello:err100eps" - ], - "sizeBytes": 54649865 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64@sha256:618a82fa66cf0c75e4753369a6999032372be7308866fc9afb381789b1e5ad52", - "k8s.gcr.io/k8s-dns-kube-dns-amd64@sha256:618a82fa66cf0c75e4753369a6999032372be7308866fc9afb381789b1e5ad52", - "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13", - "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.13" - ], - "sizeBytes": 51157394 - }, - { - "names": [ - "k8s.gcr.io/cluster-proportional-autoscaler-amd64@sha256:003f98d9f411ddfa6ff6d539196355e03ddd69fa4ed38c7ffb8fec6f729afe2d", - "k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2" - ], - "sizeBytes": 49648481 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/metrics-server-amd64@sha256:220c0ed3451cb95e4b2f72dd5dc8d9d39d9f529722e5b29d8286373ce27b117e", - "k8s.gcr.io/metrics-server-amd64@sha256:49a9f12f7067d11f42c803dbe61ed2c1299959ad85cb315b25ff7eef8e6b8892", - "aksrepos.azurecr.io/mirror/metrics-server-amd64:v0.2.1", - "k8s.gcr.io/metrics-server-amd64:v0.2.1" - ], - "sizeBytes": 42541759 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64@sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4", - "k8s.gcr.io/k8s-dns-sidecar-amd64@sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4", - "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10", - "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10" - ], - "sizeBytes": 41635309 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64@sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64@sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8", - "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10", - "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10" - ], - "sizeBytes": 40372149 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/addon-resizer@sha256:8ac3ffa4232046feb297cefc40734641fa2954c16308f9e0d70ec152f22231ca", - "k8s.gcr.io/addon-resizer@sha256:507aa9845ecce1fdde4d61f530c802f4dc2974c700ce0db7730866e442db958d", - "aksrepos.azurecr.io/mirror/addon-resizer:1.8.1", - "k8s.gcr.io/addon-resizer:1.8.1" - ], - "sizeBytes": 32968591 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/nginx@sha256:91d22184f3f9b1be658c2cc2c12d324de7ff12c8b9c9a597905457b4d93b069d", - "nginx@sha256:9d46fd628d54ebe1633ee3cf0fe2acfcc419cfae541c63056530e39cd5620366", - "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "nginx:1.13.12-alpine" - ], - "sizeBytes": 18002931 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/exechealthz-amd64@sha256:34722333f0cd0b891b61c9e0efa31913f22157e341a3aabb79967305d4e78260", - "k8s.gcr.io/exechealthz-amd64@sha256:503e158c3f65ed7399f54010571c7c977ade7fe59010695f48d9650d83488c0a", - "aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2", - "k8s.gcr.io/exechealthz-amd64:1.2" - ], - "sizeBytes": 8374840 - } - ], - "nodeInfo": { - "architecture": "amd64", - "bootID": "4c822e6d-c2e5-4697-9a01-467e04804fc1", - "containerRuntimeVersion": "docker://3.0.4", - "kernelVersion": "4.15.0-1037-azure", - "kubeProxyVersion": "v1.11.8", - "kubeletVersion": "v1.11.8", - "machineID": "1954026de5e6436788f214eb0dfd6a13", - "operatingSystem": "linux", - "osImage": "Ubuntu 16.04.5 LTS", - "systemUUID": "17A6A78E-D3E2-2A4F-852B-C91D933C8D5B" - } - } - }, - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "annotations": { - "node.alpha.kubernetes.io/ttl": "0", - "volumes.kubernetes.io/controller-managed-attach-detach": "true" - }, - "creationTimestamp": "2019-06-21T02:01:53Z", - "labels": { - "agentpool": "nodepool1", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/instance-type": "Standard_DS1_v2", - "beta.kubernetes.io/os": "linux", - "failure-domain.beta.kubernetes.io/region": "eastus", - "failure-domain.beta.kubernetes.io/zone": "0", - "kubernetes.azure.com/cluster": "MC_dilipr-health-test_dilipr-health-test_eastus", - "kubernetes.io/hostname": "aks-nodepool1-19574989-2", - "kubernetes.io/role": "agent", - "node-role.kubernetes.io/agent": "", - "storageprofile": "managed", - "storagetier": "Premium_LRS" - }, - "name": "aks-nodepool1-19574989-2", - "resourceVersion": "19068101", - "selfLink": "/api/v1/nodes/aks-nodepool1-19574989-2", - "uid": "8a62e1bc-93c8-11e9-854d-ee76584a3c00" - }, - "spec": { - "podCIDR": "10.244.12.0/24", - "providerID": "azure:///subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/MC_dilipr-health-test_dilipr-health-test_eastus/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-19574989-2" - }, - "status": { - "addresses": [ - { - "address": "aks-nodepool1-19574989-2", - "type": "Hostname" - }, - { - "address": "10.240.0.7", - "type": "InternalIP" - } - ], - "allocatable": { - "cpu": "940m", - "ephemeral-storage": "28043041951", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "2480548Ki", - "pods": "110" - }, - "capacity": { - "cpu": "1", - "ephemeral-storage": "30428648Ki", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "3500452Ki", - "pods": "110" - }, - "conditions": [ - { - "lastHeartbeatTime": "2019-06-21T02:02:24Z", - "lastTransitionTime": "2019-06-21T02:02:24Z", - "message": "RouteController created a route", - "reason": "RouteCreated", - "status": "False", - "type": "NetworkUnavailable" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:20Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet has sufficient disk space available", - "reason": "KubeletHasSufficientDisk", - "status": "False", - "type": "OutOfDisk" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:20Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet has sufficient memory available", - "reason": "KubeletHasSufficientMemory", - "status": "False", - "type": "MemoryPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:20Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet has no disk pressure", - "reason": "KubeletHasNoDiskPressure", - "status": "False", - "type": "DiskPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:20Z", - "lastTransitionTime": "2019-06-21T02:01:53Z", - "message": "kubelet has sufficient PID available", - "reason": "KubeletHasSufficientPID", - "status": "False", - "type": "PIDPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:20Z", - "lastTransitionTime": "2019-07-23T14:46:10Z", - "message": "kubelet is posting ready status. AppArmor enabled", - "reason": "KubeletReady", - "status": "True", - "type": "Ready" - } - ], - "daemonEndpoints": { - "kubeletEndpoint": { - "Port": 10250 - } - }, - "images": [ - { - "names": [ - "nickchase/rss-php-nginx@sha256:48da56a77fe4ecff4917121365d8e0ce615ebbdfe31f48a996255f5592894e2b", - "nickchase/rss-php-nginx:v1" - ], - "sizeBytes": 677038498 - }, - { - "names": [ - "rdilip83/jsonlogger@sha256:82b67ca5e0650cd5e47f5b51659d61cee035e5d8dcd8a79c50358cd2beb3b5a8", - "rdilip83/jsonlogger:v12" - ], - "sizeBytes": 676594134 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "k8s.gcr.io/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "k8s.gcr.io/hyperkube-amd64:v1.11.8" - ], - "sizeBytes": 615263658 - }, - { - "names": [ - "rdilip83/fixrubyerror@sha256:6b7f36cf6258b311015493ab025f06577d758c45bc5010d022ac160b9f40ea5d", - "rdilip83/fixrubyerror:latest" - ], - "sizeBytes": 494068028 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019" - ], - "sizeBytes": 494067935 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:fb2b90ce9bf7186fd9dfae97f5f72f9b9c80c8a0493af3cff74179cd4ff847c0", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08212019" - ], - "sizeBytes": 494067572 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c646e180483d295ffac114fb9df513db02553af7879681814d5910764653dd2d", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08202019" - ], - "sizeBytes": 494067210 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c21b596a22a1338ed293d01681f327acc871ee502ed779ec1109d6a93375bb3b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08192019" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "rdilip83/cifeatureprod08192019@sha256:7815bba9a805e4e8df33356fd532671de45525ce9c6e936e14f9b126e2097ecd", - "rdilip83/cifeatureprod08192019:v1" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:6387d0dedf4de0bab430f681ef61361f63a20e1c4c287a9b60ea5460283ac6cf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ci_feature_prod_health08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:59e34aab9f6e16a87e880b1ee1c9dd5434ee40dd29502e74aceefabf51443717", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:internaltesthealth08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/hc08192019@sha256:014d936771508d499ac4c15043e23b16bce8de0019fb2048b99540cbe9084895", - "rdilip83/hc08192019:1" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/health-rc@sha256:8ad12bce5ffd27b301bc6fe4355c8affa6fce080ae7e2291dec3a0ed11bb9483", - "rdilip83/health-rc:3" - ], - "sizeBytes": 494052863 - }, - { - "names": [ - "rdilip83/health_ci_feature_image@sha256:1a574d25884483083e8cbaacbf0cb7c4e442dc736d480615c65f5c71f8969b13", - "rdilip83/health_ci_feature_image:v1" - ], - "sizeBytes": 494052147 - }, - { - "names": [ - "rdilip83/healthrc@sha256:816c8cef09822daf050a0fca6f92e7ac19147ff4bf1a823d43fe70f73470cc0c", - "rdilip83/healthrc:v3" - ], - "sizeBytes": 494052138 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:d35aac044d1adc3d02269fde78f8dfd923db94b81288447cf6fdd482970a333b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthmerge08142019" - ], - "sizeBytes": 494052135 - }, - { - "names": [ - "rdilip83/healthrc@sha256:a130780e56ac0edb3ca29477e12edd5e9b5d08b5732dbd59ede9beb58e21eca7", - "rdilip83/healthrc:v2" - ], - "sizeBytes": 494051682 - }, - { - "names": [ - "rdilip83/healthmerge@sha256:24d270b0f59fb484c283922474736c3cba50f8aad0270bc0a3acd14284694eea", - "rdilip83/healthmerge:v8" - ], - "sizeBytes": 494010139 - }, - { - "names": [ - "rdilip83/health-rc@sha256:b1d24728eb808d301da426b76b7f7b79606204c4c2b695a24ac670be8276d55d", - "rdilip83/health-rc:1" - ], - "sizeBytes": 494000891 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:a0666957cccbfdf5784accd1133408bf017c28a6e694d9a2ae74da94eef2d285", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview08012019" - ], - "sizeBytes": 493994261 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:32c9b35a6809c54d5296e2ca2b122b35a4ad8c852622174cc5a9f92cc27e56e4", - "rdilip83/mergehealth:v3" - ], - "sizeBytes": 493988815 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:a3521e8f36e007b3cb949e0356a75394ac61fd2024ca1ec4827b8d54fb068534", - "rdilip83/mergehealth:v1" - ], - "sizeBytes": 493981585 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0438e4690e042b195917e160b8949aeb339520ee19c898a8bb9452f36d1f84f1", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview07182019" - ], - "sizeBytes": 493977357 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:9ebc410a36856176921dba81b5bd43132469209b315f52be346690435419b9bb" - ], - "sizeBytes": 493946790 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:4e51195a9c77bd166fc90ee5f6143a4604b502ab7ef0f06431dec10c341b10f3", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview06272019" - ], - "sizeBytes": 493893635 - }, - { - "names": [ - "rdilip83/healthpreview06272019@sha256:d888ba5ff5e5810113a32f9c9812a5e28088cc81b902e95a185fe465a514029c", - "rdilip83/healthpreview06272019:latest" - ], - "sizeBytes": 493893633 - }, - { - "names": [ - "rdilip83/healthpreview06252019-1@sha256:1561876cffe94433a569f29f5231548e039193ebaa7ec640d22439675179e43f", - "rdilip83/healthpreview06252019-1:latest" - ], - "sizeBytes": 493887387 - }, - { - "names": [ - "rdilip83/healthpreview06252019@sha256:6597ff599a78ac452a4138dedb9e08c0ccd3e8b01594b033fd78ba9dbb41fe9e", - "rdilip83/healthpreview06252019:latest" - ], - "sizeBytes": 493887384 - }, - { - "names": [ - "rdilip83/healthpreview06242019@sha256:c4f565d92086d1ee56e6016178fed5c668352dc0ca0047f02910bdcb87a482c4", - "rdilip83/healthpreview06242019:latest" - ], - "sizeBytes": 493850850 - }, - { - "names": [ - "rdilip83/healthpreview06212019-1@sha256:937ce5801a0097a1cbc4eff5399c1973b4c6223ece9279b35207368b99f82b96", - "rdilip83/healthpreview06212019-1:latest" - ], - "sizeBytes": 493850674 - }, - { - "names": [ - "rdilip83/healthpreview06192019@sha256:f92cb5283814d446f0acde6a489648ea197496d5f85b27ca959ec97bce742d8a", - "rdilip83/healthpreview06192019:latest" - ], - "sizeBytes": 493799437 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0f798cb7d56931b231f71e38e7fa5bf898b69e611247a566701f70a5f29a9799", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod07092019" - ], - "sizeBytes": 467692116 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:3734a084fa9681c7e930eb90cad45a8f282c24af63065a720a2327b1683f3ba4", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06142019" - ], - "sizeBytes": 466882569 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:16402c34e2d7de72c2ebc18ec8e9f7933fa25f6a7f83bceb84483ba95e3902f7", - "rdilip83/mergehealth:v2" - ], - "sizeBytes": 448931997 - }, - { - "names": [ - "rdilip83/healthpreview06212019@sha256:5860c9caaf544f2e7c46edad5cdfb69e22398e20dc87cb8a4cd630b5b7000074", - "rdilip83/healthpreview06212019:latest" - ], - "sizeBytes": 448366491 - }, - { - "names": [ - "deis/hcp-tunnel-front@sha256:68878ee3ea1781b322ea3952c3370e31dd89be8bb0864e2bf27bdba6dc904c41", - "deis/hcp-tunnel-front:v1.9.2-v4.0.7" - ], - "sizeBytes": 383483267 - }, - { - "names": [ - "progrium/stress@sha256:e34d56d60f5caae79333cee395aae93b74791d50e3841986420d23c2ee4697bf", - "progrium/stress:latest" - ], - "sizeBytes": 281783943 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:b6834bb69e8fad88110b1dc57097a45bc79e6f2c5f2c2773c871d07389794771", - "k8s.gcr.io/cluster-autoscaler:v1.12.3" - ], - "sizeBytes": 232229241 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:dc5744fd8c22aebfe40d6b62ab97d18d7bfbfc7ab1782509d69a5a9ec514df2c", - "k8s.gcr.io/cluster-autoscaler:v1.12.2" - ], - "sizeBytes": 232167833 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:e71851267764a068fbb091a4ef3bb874b5ce34db48cb757fcf77779f30ef0207", - "k8s.gcr.io/cluster-autoscaler:v1.3.7" - ], - "sizeBytes": 217353965 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:36a369ca4643542d501bce0addf8b903f2141ae9e2608662b77a3d24f01d7780", - "k8s.gcr.io/cluster-autoscaler:v1.2.2" - ], - "sizeBytes": 208688449 - }, - { - "names": [ - "containernetworking/azure-npm@sha256:4735da6dc0d5393d68be72498f5ce563cb930fa21b26faec8fdc844001057a56", - "containernetworking/azure-npm:v1.0.18" - ], - "sizeBytes": 170727162 - }, - { - "names": [ - "containernetworking/networkmonitor@sha256:d875511410502c3e37804e1f313cc2b0a03d7a03d3d5e6adaf8994b753a76f8e", - "containernetworking/networkmonitor:v0.0.6" - ], - "sizeBytes": 123663837 - }, - { - "names": [ - "containernetworking/networkmonitor@sha256:944408a497c451b0e79d2596dc2e9fe5036cdbba7fa831bff024e1c9ed44190d", - "containernetworking/networkmonitor:v0.0.5" - ], - "sizeBytes": 122043325 - }, - { - "names": [ - "nginx@sha256:bdbf36b7f1f77ffe7bd2a32e59235dff6ecf131e3b6b5b96061c652f30685f3a", - "nginx:latest" - ], - "sizeBytes": 109258867 - }, - { - "names": [ - "debian@sha256:118cf8f3557e1ea766c02f36f05f6ac3e63628427ea8965fb861be904ec35a6f", - "debian:latest" - ], - "sizeBytes": 100594230 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:3da3f17cd4f02fe5696f29a5e6cd4aef7111f20dab9bec54ea35942346cfeb60", - "k8s.gcr.io/kube-addon-manager-amd64:v8.8" - ], - "sizeBytes": 99631084 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:672794ee3582521eb8bc4f257d0f70c92893f1989f39a200f9c84bcfe1aea7c9", - "k8s.gcr.io/kube-addon-manager-amd64:v9.0" - ], - "sizeBytes": 83077558 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/prod/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "deis/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/mirror/kube-svc-redirect:v1.0.2", - "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2" - ], - "sizeBytes": 82897218 - }, - { - "names": [ - "k8s.gcr.io/heapster-amd64@sha256:dccaabb0c20cf05c29baefa1e9bf0358b083ccc0fab492b9b3b47fb7e4db5472", - "k8s.gcr.io/heapster-amd64:v1.5.4" - ], - "sizeBytes": 75318342 - } - ], - "nodeInfo": { - "architecture": "amd64", - "bootID": "ee529550-afa8-43bb-90a6-f157e7e22e18", - "containerRuntimeVersion": "docker://3.0.4", - "kernelVersion": "4.15.0-1045-azure", - "kubeProxyVersion": "v1.11.8", - "kubeletVersion": "v1.11.8", - "machineID": "0e5d932888da4e17a3c58210f6c8c9db", - "operatingSystem": "linux", - "osImage": "Ubuntu 16.04.6 LTS", - "systemUUID": "5DBFC273-947F-0140-AD1F-BF6758D30B37" - } - } - }, - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "annotations": { - "node.alpha.kubernetes.io/ttl": "0", - "volumes.kubernetes.io/controller-managed-attach-detach": "true" - }, - "creationTimestamp": "2019-08-07T18:57:56Z", - "labels": { - "agentpool": "nodepool1", - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/instance-type": "Standard_DS1_v2", - "beta.kubernetes.io/os": "linux", - "failure-domain.beta.kubernetes.io/region": "eastus", - "failure-domain.beta.kubernetes.io/zone": "1", - "kubernetes.azure.com/cluster": "MC_dilipr-health-test_dilipr-health-test_eastus", - "kubernetes.io/hostname": "aks-nodepool1-19574989-3", - "kubernetes.io/role": "agent", - "node-role.kubernetes.io/agent": "", - "storageprofile": "managed", - "storagetier": "Premium_LRS" - }, - "name": "aks-nodepool1-19574989-3", - "resourceVersion": "19068105", - "selfLink": "/api/v1/nodes/aks-nodepool1-19574989-3", - "uid": "448ea0a7-b945-11e9-a1b6-127094e7fd94" - }, - "spec": { - "podCIDR": "10.244.2.0/24", - "providerID": "azure:///subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourceGroups/MC_dilipr-health-test_dilipr-health-test_eastus/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-19574989-3" - }, - "status": { - "addresses": [ - { - "address": "aks-nodepool1-19574989-3", - "type": "Hostname" - }, - { - "address": "10.240.0.6", - "type": "InternalIP" - } - ], - "allocatable": { - "cpu": "940m", - "ephemeral-storage": "28043041951", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "2480544Ki", - "pods": "110" - }, - "capacity": { - "cpu": "1", - "ephemeral-storage": "30428648Ki", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "memory": "3500448Ki", - "pods": "110" - }, - "conditions": [ - { - "lastHeartbeatTime": "2019-08-07T18:59:32Z", - "lastTransitionTime": "2019-08-07T18:59:32Z", - "message": "RouteController created a route", - "reason": "RouteCreated", - "status": "False", - "type": "NetworkUnavailable" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-08-07T18:57:56Z", - "message": "kubelet has sufficient disk space available", - "reason": "KubeletHasSufficientDisk", - "status": "False", - "type": "OutOfDisk" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-08-07T18:57:56Z", - "message": "kubelet has sufficient memory available", - "reason": "KubeletHasSufficientMemory", - "status": "False", - "type": "MemoryPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-08-07T18:57:56Z", - "message": "kubelet has no disk pressure", - "reason": "KubeletHasNoDiskPressure", - "status": "False", - "type": "DiskPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-08-07T18:57:56Z", - "message": "kubelet has sufficient PID available", - "reason": "KubeletHasSufficientPID", - "status": "False", - "type": "PIDPressure" - }, - { - "lastHeartbeatTime": "2019-08-23T20:43:22Z", - "lastTransitionTime": "2019-08-07T18:58:06Z", - "message": "kubelet is posting ready status. AppArmor enabled", - "reason": "KubeletReady", - "status": "True", - "type": "Ready" - } - ], - "daemonEndpoints": { - "kubeletEndpoint": { - "Port": 10250 - } - }, - "images": [ - { - "names": [ - "deis/hcp-tunnel-front@sha256:a067679f0ab376197a344cd410821cf07d69fc322dcd9af4a9229250da725ce2", - "deis/hcp-tunnel-front:v1.9.2-v4.0.4" - ], - "sizeBytes": 640504769 - }, - { - "names": [ - "aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "k8s.gcr.io/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "k8s.gcr.io/hyperkube-amd64:v1.11.8" - ], - "sizeBytes": 615263658 - }, - { - "names": [ - "rdilip83/fixrubyerror@sha256:6b7f36cf6258b311015493ab025f06577d758c45bc5010d022ac160b9f40ea5d", - "rdilip83/fixrubyerror:latest" - ], - "sizeBytes": 494068028 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019" - ], - "sizeBytes": 494067935 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:fb2b90ce9bf7186fd9dfae97f5f72f9b9c80c8a0493af3cff74179cd4ff847c0", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08212019" - ], - "sizeBytes": 494067572 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c646e180483d295ffac114fb9df513db02553af7879681814d5910764653dd2d", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08202019" - ], - "sizeBytes": 494067210 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:c21b596a22a1338ed293d01681f327acc871ee502ed779ec1109d6a93375bb3b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08192019" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "rdilip83/cifeatureprod08192019@sha256:7815bba9a805e4e8df33356fd532671de45525ce9c6e936e14f9b126e2097ecd", - "rdilip83/cifeatureprod08192019:v1" - ], - "sizeBytes": 494055088 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:6387d0dedf4de0bab430f681ef61361f63a20e1c4c287a9b60ea5460283ac6cf", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ci_feature_prod_health08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/hc08192019@sha256:014d936771508d499ac4c15043e23b16bce8de0019fb2048b99540cbe9084895", - "rdilip83/hc08192019:1" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:59e34aab9f6e16a87e880b1ee1c9dd5434ee40dd29502e74aceefabf51443717", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:internaltesthealth08192019" - ], - "sizeBytes": 494053562 - }, - { - "names": [ - "rdilip83/health-rc@sha256:8ad12bce5ffd27b301bc6fe4355c8affa6fce080ae7e2291dec3a0ed11bb9483", - "rdilip83/health-rc:3" - ], - "sizeBytes": 494052863 - }, - { - "names": [ - "rdilip83/health_ci_feature_image@sha256:1a574d25884483083e8cbaacbf0cb7c4e442dc736d480615c65f5c71f8969b13", - "rdilip83/health_ci_feature_image:v1" - ], - "sizeBytes": 494052147 - }, - { - "names": [ - "rdilip83/healthrc@sha256:816c8cef09822daf050a0fca6f92e7ac19147ff4bf1a823d43fe70f73470cc0c", - "rdilip83/healthrc:v3" - ], - "sizeBytes": 494052138 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:d35aac044d1adc3d02269fde78f8dfd923db94b81288447cf6fdd482970a333b", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthmerge08142019" - ], - "sizeBytes": 494052135 - }, - { - "names": [ - "rdilip83/healthrc@sha256:a130780e56ac0edb3ca29477e12edd5e9b5d08b5732dbd59ede9beb58e21eca7", - "rdilip83/healthrc:v2" - ], - "sizeBytes": 494051682 - }, - { - "names": [ - "rdilip83/healthmerge@sha256:24d270b0f59fb484c283922474736c3cba50f8aad0270bc0a3acd14284694eea", - "rdilip83/healthmerge:v8" - ], - "sizeBytes": 494010139 - }, - { - "names": [ - "rdilip83/health-rc@sha256:b1d24728eb808d301da426b76b7f7b79606204c4c2b695a24ac670be8276d55d", - "rdilip83/health-rc:1" - ], - "sizeBytes": 494000891 - }, - { - "names": [ - "rdilip83/mergehealth@sha256:32c9b35a6809c54d5296e2ca2b122b35a4ad8c852622174cc5a9f92cc27e56e4", - "rdilip83/mergehealth:v3" - ], - "sizeBytes": 493988815 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:0438e4690e042b195917e160b8949aeb339520ee19c898a8bb9452f36d1f84f1", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview07182019" - ], - "sizeBytes": 493977357 - }, - { - "names": [ - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:4e51195a9c77bd166fc90ee5f6143a4604b502ab7ef0f06431dec10c341b10f3", - "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:healthpreview06272019" - ], - "sizeBytes": 493893635 - }, - { - "names": [ - "rdilip83/healthpreview06272019@sha256:d888ba5ff5e5810113a32f9c9812a5e28088cc81b902e95a185fe465a514029c", - "rdilip83/healthpreview06272019:latest" - ], - "sizeBytes": 493893633 - }, - { - "names": [ - "aksrepos.azurecr.io/prod/hcp-tunnel-front@sha256:68878ee3ea1781b322ea3952c3370e31dd89be8bb0864e2bf27bdba6dc904c41", - "aksrepos.azurecr.io/prod/hcp-tunnel-front:v1.9.2-v4.0.7" - ], - "sizeBytes": 383483267 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:b6834bb69e8fad88110b1dc57097a45bc79e6f2c5f2c2773c871d07389794771", - "k8s.gcr.io/cluster-autoscaler:v1.12.3" - ], - "sizeBytes": 232229241 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:dc5744fd8c22aebfe40d6b62ab97d18d7bfbfc7ab1782509d69a5a9ec514df2c", - "k8s.gcr.io/cluster-autoscaler:v1.12.2" - ], - "sizeBytes": 232167833 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:193eaf37788dd5f971dd400b7e3d28e650bfd81c90fa46b234f03eb3d43880e3", - "k8s.gcr.io/cluster-autoscaler:v1.12.5" - ], - "sizeBytes": 231543459 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:e71851267764a068fbb091a4ef3bb874b5ce34db48cb757fcf77779f30ef0207", - "k8s.gcr.io/cluster-autoscaler:v1.3.7" - ], - "sizeBytes": 217353965 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:156b7b9bcba24ed474f67d0feaf27f2506013f15b030341bbd41c630283161b8", - "k8s.gcr.io/cluster-autoscaler:v1.3.4" - ], - "sizeBytes": 217264129 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:97896235bf66bde573d6f2ee150e212ea7010d314eb5d2cfb2ff1af93335db30", - "k8s.gcr.io/cluster-autoscaler:v1.3.3" - ], - "sizeBytes": 217259793 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:b416bf3b6687788b4da4c7ede2bcf067b34ad781862ee3d3dac1d720c5fa38b3", - "k8s.gcr.io/cluster-autoscaler:v1.3.9" - ], - "sizeBytes": 216696035 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:f37a2c84614bdd02475ccb020182caec562cde97fdfd9dae58de66ff89614bc5", - "k8s.gcr.io/cluster-autoscaler:v1.3.8" - ], - "sizeBytes": 216693526 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:b0777becbfc7a56e66b079d2767fdc173121a29165523bbbe309bcb2c0a226aa", - "k8s.gcr.io/cluster-autoscaler:v1.2.5" - ], - "sizeBytes": 212991966 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:36a369ca4643542d501bce0addf8b903f2141ae9e2608662b77a3d24f01d7780", - "k8s.gcr.io/cluster-autoscaler:v1.2.2" - ], - "sizeBytes": 208688449 - }, - { - "names": [ - "mcr.microsoft.com/containernetworking/azure-npm@sha256:7b9e7dec6b06a21595f9aa06b319c99b579950619fa869dd85dc637b2235d79f", - "mcr.microsoft.com/containernetworking/azure-npm:v1.0.18" - ], - "sizeBytes": 170727162 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:760232bed2097b5ca742f05b15c94d56ff96ed6b5c93251edc613be045c8d78b", - "k8s.gcr.io/cluster-autoscaler:v1.15.0" - ], - "sizeBytes": 152214996 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:a4e5a8e6d4dc011e6e7a104d6abdfda56274b90357ee9f6e42cc22b70482420b", - "k8s.gcr.io/cluster-autoscaler:v1.14.0" - ], - "sizeBytes": 142102721 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:cbc61e0f6c3ef1c591a0f22ec483826110e2c10acddd5415c0cc2305fd085e69", - "k8s.gcr.io/cluster-autoscaler:v1.14.2" - ], - "sizeBytes": 142099784 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:9dcbd91e79f33c44529de58a0024deb3da23a3a0bc7fd4d028c1255c68f62fb7", - "k8s.gcr.io/cluster-autoscaler:v1.13.2" - ], - "sizeBytes": 136684274 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:e4140dc3ab54e115ab4464331b25022fc5ffb947b568aaf81089efb72506c895", - "k8s.gcr.io/cluster-autoscaler:v1.13.4" - ], - "sizeBytes": 136681463 - }, - { - "names": [ - "k8s.gcr.io/cluster-autoscaler@sha256:7ff5a60304b344f2f29c804c7253632bbc818794f6932236a56db107a6a8f5af", - "k8s.gcr.io/cluster-autoscaler:v1.13.1" - ], - "sizeBytes": 136618018 - }, - { - "names": [ - "mcr.microsoft.com/containernetworking/networkmonitor@sha256:d875511410502c3e37804e1f313cc2b0a03d7a03d3d5e6adaf8994b753a76f8e", - "mcr.microsoft.com/containernetworking/networkmonitor:v0.0.6" - ], - "sizeBytes": 123663837 - }, - { - "names": [ - "mcr.microsoft.com/containernetworking/networkmonitor@sha256:944408a497c451b0e79d2596dc2e9fe5036cdbba7fa831bff024e1c9ed44190d", - "mcr.microsoft.com/containernetworking/networkmonitor:v0.0.5" - ], - "sizeBytes": 122043325 - }, - { - "names": [ - "k8s.gcr.io/kubernetes-dashboard-amd64@sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747", - "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1" - ], - "sizeBytes": 121711221 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:3da3f17cd4f02fe5696f29a5e6cd4aef7111f20dab9bec54ea35942346cfeb60", - "k8s.gcr.io/kube-addon-manager-amd64:v8.8" - ], - "sizeBytes": 99631084 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:2fd1daf3d3cf0e94a753f2263b60dbb0d42b107b5cde0c75ee3fc5c830e016e4", - "k8s.gcr.io/kube-addon-manager-amd64:v8.9" - ], - "sizeBytes": 99240637 - }, - { - "names": [ - "microsoft/virtual-kubelet@sha256:efc397d741d7e590c892c0ea5dccc9a800656c3adb95da4dae25c1cdd5eb6d9f", - "microsoft/virtual-kubelet:latest" - ], - "sizeBytes": 87436458 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:672794ee3582521eb8bc4f257d0f70c92893f1989f39a200f9c84bcfe1aea7c9", - "k8s.gcr.io/kube-addon-manager-amd64:v9.0" - ], - "sizeBytes": 83077558 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:382c220b3531d9f95bf316a16b7282cc2ef929cd8a89a9dd3f5933edafc41a8e", - "k8s.gcr.io/kube-addon-manager-amd64:v9.0.1" - ], - "sizeBytes": 83076194 - }, - { - "names": [ - "aksrepos.azurecr.io/prod/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "deis/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2", - "deis/kube-svc-redirect:v1.0.2" - ], - "sizeBytes": 82897218 - }, - { - "names": [ - "k8s.gcr.io/kube-addon-manager-amd64@sha256:3519273916ba45cfc9b318448d4629819cb5fbccbb0822cce054dd8c1f68cb60", - "k8s.gcr.io/kube-addon-manager-amd64:v8.6" - ], - "sizeBytes": 78384272 - } - ], - "nodeInfo": { - "architecture": "amd64", - "bootID": "47e7c02b-3741-42be-a2a1-76c76aa8ccde", - "containerRuntimeVersion": "docker://3.0.6", - "kernelVersion": "4.15.0-1050-azure", - "kubeProxyVersion": "v1.11.8", - "kubeletVersion": "v1.11.8", - "machineID": "a4a4bc2f5a944cd38aba89365df05227", - "operatingSystem": "linux", - "osImage": "Ubuntu 16.04.6 LTS", - "systemUUID": "BB102B43-9922-264C-8C23-22A7DE0F950F" - } - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": "" - } -} diff --git a/test/unit-tests/plugins/health/parent_monitor_provider_spec.rb b/test/unit-tests/plugins/health/parent_monitor_provider_spec.rb deleted file mode 100644 index 851a18002..000000000 --- a/test/unit-tests/plugins/health/parent_monitor_provider_spec.rb +++ /dev/null @@ -1,146 +0,0 @@ -require_relative '../test_helpers' -Dir[File.join(File.expand_path(File.dirname(__FILE__)), "../../../../source/plugins/ruby/health/*.rb")].reject{|f| f.include?('health_monitor_utils')}.each { |file| require file } -include HealthModel -include Minitest - -describe "ParentMonitorProvider spec" do - it 'returns correct parent_monitor_id for a non-condition case' do - #arrange - definition = JSON.parse('{ - "monitor_id" : { - "parent_monitor_id": "parent_monitor_id", - "labels": [ - "label_1", - "label_2" - ] - } - }' - ) - health_model_definition = ParentMonitorProvider.new(definition) - - monitor = Mock.new - def monitor.monitor_id; "monitor_id"; end - def monitor.monitor_instance_id; "monitor_instance_id"; end - - #act - parent_id = health_model_definition.get_parent_monitor_id(monitor) - #assert - assert_equal parent_id, "parent_monitor_id" - end - - it 'returns raises for an incorrect monitor id' do - #arrange - definition = JSON.parse('{ - "monitor_id" : { - "parent_monitor_id": "parent_monitor_id", - "labels": [ - "label_1", - "label_2" - ] - } - }' - ) - health_model_definition = ParentMonitorProvider.new(definition) - - monitor = Mock.new - def monitor.monitor_id; "monitor_id_!"; end - def monitor.monitor_instance_id; "monitor_instance_id"; end - - #act and assert - assert_raises do - parent_id = health_model_definition.get_parent_monitor_id(monitor) - end - end - - it 'returns correct parent_monitor_id for a conditional case' do - #arrange - definition = JSON.parse('{"conditional_monitor_id": { - "conditions": [ - { - "key": "kubernetes.io/role", - "operator": "==", - "value": "master", - "parent_id": "master_node_pool" - }, - { - "key": "kubernetes.io/role", - "operator": "==", - "value": "agent", - "parent_id": "agent_node_pool" - } - ], - "labels": [ - "kubernetes.io/hostname", - "agentpool", - "kubernetes.io/role", - "container.azm.ms/cluster-region", - "container.azm.ms/cluster-subscription-id", - "container.azm.ms/cluster-resource-group", - "container.azm.ms/cluster-name" - ], - "aggregation_algorithm": "worstOf", - "aggregation_algorithm_params": null - } - - }' - ) - health_model_definition = ParentMonitorProvider.new(definition) - - monitor = Mock.new - def monitor.monitor_id; "conditional_monitor_id"; end - def monitor.monitor_instance_id; "conditional_monitor_instance_id"; end - def monitor.labels; {HealthMonitorLabels::ROLE => "master"}; end - - #act - parent_id = health_model_definition.get_parent_monitor_id(monitor) - #assert - assert_equal parent_id, "master_node_pool" - end - - it 'returns defaultParentMonitorTypeId if conditions are not met' do - #arrange - definition = JSON.parse('{"conditional_monitor_id": { - "conditions": [ - { - "key": "kubernetes.io/role", - "operator": "==", - "value": "master", - "parent_id": "master_node_pool" - }, - { - "key": "kubernetes.io/role", - "operator": "==", - "value": "agent", - "parent_id": "agent_node_pool" - } - ], - "labels": [ - "kubernetes.io/hostname", - "agentpool", - "kubernetes.io/role", - "container.azm.ms/cluster-region", - "container.azm.ms/cluster-subscription-id", - "container.azm.ms/cluster-resource-group", - "container.azm.ms/cluster-name" - ], - "default_parent_monitor_id": "default_parent_monitor_id", - "aggregation_algorithm": "worstOf", - "aggregation_algorithm_params": null - } - - }' - ) - health_model_definition = ParentMonitorProvider.new(definition) - - monitor = Mock.new - def monitor.monitor_id; "conditional_monitor_id"; end - def monitor.monitor_instance_id; "conditional_monitor_instance_id"; end - def monitor.labels; {HealthMonitorLabels::ROLE => "master1"}; end - - #act and assert - - parent_id = health_model_definition.get_parent_monitor_id(monitor) - parent_id.must_equal('default_parent_monitor_id') - - end -end diff --git a/test/unit-tests/plugins/health/pods.json b/test/unit-tests/plugins/health/pods.json deleted file mode 100644 index b7c202a19..000000000 --- a/test/unit-tests/plugins/health/pods.json +++ /dev/null @@ -1,5987 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-08-23T17:12:10Z", - "generateName": "heapster-9bcbfdcf5-", - "labels": { - "k8s-app": "heapster", - "pod-template-hash": "567698791" - }, - "name": "heapster-9bcbfdcf5-zp9tl", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "heapster-9bcbfdcf5", - "uid": "24a0036e-c5c9-11e9-8736-86290fd7dd1f" - } - ], - "resourceVersion": "19048925", - "selfLink": "/api/v1/namespaces/kube-system/pods/heapster-9bcbfdcf5-zp9tl", - "uid": "24ab7e32-c5c9-11e9-8736-86290fd7dd1f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/heapster", - "--source=kubernetes.summary_api:\"\"" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/heapster-amd64:v1.5.3", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8082, - "scheme": "HTTP" - }, - "initialDelaySeconds": 180, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "heapster", - "resources": { - "limits": { - "cpu": "88m", - "memory": "204Mi" - }, - "requests": { - "cpu": "88m", - "memory": "204Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "heapster-token-7z7c5", - "readOnly": true - } - ] - }, - { - "command": [ - "/pod_nanny", - "--config-dir=/etc/config", - "--cpu=80m", - "--extra-cpu=0.5m", - "--memory=140Mi", - "--extra-memory=4Mi", - "--threshold=5", - "--deployment=heapster", - "--container=heapster", - "--poll-period=300000", - "--estimator=exponential" - ], - "env": [ - { - "name": "MY_POD_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.name" - } - } - }, - { - "name": "MY_POD_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/addon-resizer:1.8.1", - "imagePullPolicy": "IfNotPresent", - "name": "heapster-nanny", - "resources": { - "limits": { - "cpu": "50m", - "memory": "90Mi" - }, - "requests": { - "cpu": "50m", - "memory": "90Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/config", - "name": "heapster-config-volume" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "heapster-token-7z7c5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-0", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "heapster", - "serviceAccountName": "heapster", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "heapster-config" - }, - "name": "heapster-config-volume" - }, - { - "name": "heapster-token-7z7c5", - "secret": { - "defaultMode": 420, - "secretName": "heapster-token-7z7c5" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:12:10Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:12:26Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:12:10Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://8ab1ee82d29d0351cb21dbce4db9eb2a270407d2ebe10377be02edd46cb34027", - "image": "aksrepos.azurecr.io/mirror/heapster-amd64:v1.5.3", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/heapster-amd64@sha256:fc33c690a3a446de5abc24b048b88050810a58b9e4477fa763a43d7df029301a", - "lastState": {}, - "name": "heapster", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T17:12:21Z" - } - } - }, - { - "containerID": "docker://42154ff41fed196c3f4b8a485436537330d16bcef23c743a34cf63202d023453", - "image": "aksrepos.azurecr.io/mirror/addon-resizer:1.8.1", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/addon-resizer@sha256:8ac3ffa4232046feb297cefc40734641fa2954c16308f9e0d70ec152f22231ca", - "lastState": {}, - "name": "heapster-nanny", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T17:12:25Z" - } - } - } - ], - "hostIP": "10.240.0.4", - "phase": "Running", - "podIP": "10.244.1.33", - "qosClass": "Guaranteed", - "startTime": "2019-08-23T17:12:10Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "scheduler.alpha.kubernetes.io/critical-pod": "", - "seccomp.security.alpha.kubernetes.io/pod": "docker/default" - }, - "creationTimestamp": "2019-07-09T02:38:06Z", - "generateName": "kube-dns-autoscaler-7d64798d95-", - "labels": { - "k8s-app": "kube-dns-autoscaler", - "pod-template-hash": "3820354851" - }, - "name": "kube-dns-autoscaler-7d64798d95-f9wcv", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "kube-dns-autoscaler-7d64798d95", - "uid": "71655f71-a1f2-11e9-9bc6-127bb0ec03b8" - } - ], - "resourceVersion": "15144041", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-dns-autoscaler-7d64798d95-f9wcv", - "uid": "94e52ab1-a1f2-11e9-8b08-d602e29755d5" - }, - "spec": { - "containers": [ - { - "command": [ - "/cluster-proportional-autoscaler", - "--namespace=kube-system", - "--configmap=kube-dns-autoscaler", - "--target=deployment/kube-dns-v20", - "--default-params={\"ladder\":{\"coresToReplicas\":[[1,2],[512,3],[1024,4],[2048,5]],\"nodesToReplicas\":[[1,2],[8,3],[16,4],[32,5]]}}", - "--logtostderr=true", - "--v=2" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/cluster-proportional-autoscaler-amd64:1.1.2-r2", - "imagePullPolicy": "IfNotPresent", - "name": "autoscaler", - "resources": { - "requests": { - "cpu": "20m", - "memory": "10Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-autoscaler-token-zkxt8", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-2", - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-dns-autoscaler", - "serviceAccountName": "kube-dns-autoscaler", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-dns-autoscaler-token-zkxt8", - "secret": { - "defaultMode": 420, - "secretName": "kube-dns-autoscaler-token-zkxt8" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:07Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:44Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:06Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://43f5fde3ce0f375a40c08de56087fc3b53f6269b239a3e6383d2082779504b96", - "image": "aksrepos.azurecr.io/mirror/cluster-proportional-autoscaler-amd64:1.1.2-r2", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/cluster-proportional-autoscaler-amd64@sha256:ccd2b031b116750091443930a8e6d0f785cfde38f137969e472b2ac850aeddfb", - "lastState": {}, - "name": "autoscaler", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:40Z" - } - } - } - ], - "hostIP": "10.240.0.7", - "phase": "Running", - "podIP": "10.244.12.118", - "qosClass": "Burstable", - "startTime": "2019-07-09T02:38:07Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "prometheus.io/port": "10055", - "prometheus.io/scrape": "true" - }, - "creationTimestamp": "2019-07-09T02:38:06Z", - "generateName": "kube-dns-v20-55cb5d96f7-", - "labels": { - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "pod-template-hash": "1176185293", - "version": "v20" - }, - "name": "kube-dns-v20-55cb5d96f7-lmrpl", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "kube-dns-v20-55cb5d96f7", - "uid": "71892fd6-a1f2-11e9-9bc6-127bb0ec03b8" - } - ], - "resourceVersion": "15144030", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-dns-v20-55cb5d96f7-lmrpl", - "uid": "952488f3-a1f2-11e9-8b08-d602e29755d5" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - }, - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "k8s-app", - "operator": "In", - "values": [ - "kube-dns" - ] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - }, - "weight": 100 - } - ] - } - }, - "containers": [ - { - "args": [ - "--kubecfg-file=/config/kubeconfig", - "--config-dir=/kube-dns-config", - "--domain=cluster.local.", - "--dns-port=10053", - "--v=2" - ], - "env": [ - { - "name": "PROMETHEUS_PORT", - "value": "10055" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthcheck/kubedns", - "port": 10054, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "kubedns", - "ports": [ - { - "containerPort": 10053, - "name": "dns-local", - "protocol": "UDP" - }, - { - "containerPort": 10053, - "name": "dns-tcp-local", - "protocol": "TCP" - }, - { - "containerPort": 10055, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readiness", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "170Mi" - }, - "requests": { - "cpu": "100m", - "memory": "70Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/kube-dns-config", - "name": "kube-dns-config" - }, - { - "mountPath": "/config", - "name": "kubedns-kubecfg", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - }, - { - "args": [ - "-v=2", - "-logtostderr", - "-configDir=/kube-dns-config", - "-restartDnsmasq=true", - "--", - "-k", - "--cache-size=1000", - "--no-negcache", - "--no-resolv", - "--server=127.0.0.1#10053", - "--server=/cluster.local/127.0.0.1#10053", - "--server=/in-addr.arpa/127.0.0.1#10053", - "--server=/ip6.arpa/127.0.0.1#10053", - "--log-facility=-" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10", - "imagePullPolicy": "IfNotPresent", - "name": "dnsmasq", - "ports": [ - { - "containerPort": 53, - "name": "dns", - "protocol": "UDP" - }, - { - "containerPort": 53, - "name": "dns-tcp", - "protocol": "TCP" - } - ], - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/kube-dns-config", - "name": "kube-dns-config" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - }, - { - "args": [ - "--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1 \u003e/dev/null || exit 1; done", - "--url=/healthz-dnsmasq", - "--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1:10053 \u003e/dev/null || exit 1; done", - "--url=/healthz-kubedns", - "--port=8080", - "--quiet" - ], - "env": [ - { - "name": "PROBE_DOMAINS", - "value": "bing.com kubernetes.default.svc.cluster.local" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthz-dnsmasq", - "port": 8080, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "healthz", - "ports": [ - { - "containerPort": 8080, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "50Mi" - }, - "requests": { - "cpu": "10m", - "memory": "50Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - }, - { - "args": [ - "--v=2", - "--logtostderr", - "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV", - "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/metrics", - "port": 10054, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "sidecar", - "ports": [ - { - "containerPort": 10054, - "name": "metrics", - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "20Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-1", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-dns", - "serviceAccountName": "kube-dns", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "kube-dns", - "optional": true - }, - "name": "kube-dns-config" - }, - { - "configMap": { - "defaultMode": 420, - "name": "kubedns-kubecfg" - }, - "name": "kubedns-kubecfg" - }, - { - "name": "kube-dns-token-ghgtl", - "secret": { - "defaultMode": 420, - "secretName": "kube-dns-token-ghgtl" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:09Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:50Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:06Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://8aa7d794d423f29469d8a35cc295bfaf2434a26756d7063fb19e06ce838aa5d9", - "image": "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64@sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8", - "lastState": {}, - "name": "dnsmasq", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:23Z" - } - } - }, - { - "containerID": "docker://7ee72258ca97555017c3096c3c125935b22e1735dafd494bec7f5480a408314a", - "image": "aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/exechealthz-amd64@sha256:34722333f0cd0b891b61c9e0efa31913f22157e341a3aabb79967305d4e78260", - "lastState": {}, - "name": "healthz", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:25Z" - } - } - }, - { - "containerID": "docker://bf6c7e823d08306e6ba13353ae89319080990a5d302b1d7370e76acd34c34a52", - "image": "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64@sha256:618a82fa66cf0c75e4753369a6999032372be7308866fc9afb381789b1e5ad52", - "lastState": {}, - "name": "kubedns", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:19Z" - } - } - }, - { - "containerID": "docker://2e4faf4da65a23316dc7065e3de27bf1ebd9ac2a8f07b9053de5ab63ab4c2d7e", - "image": "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64@sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4", - "lastState": {}, - "name": "sidecar", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:28Z" - } - } - } - ], - "hostIP": "10.240.0.5", - "phase": "Running", - "podIP": "10.244.0.192", - "qosClass": "Burstable", - "startTime": "2019-07-09T02:38:09Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "prometheus.io/port": "10055", - "prometheus.io/scrape": "true" - }, - "creationTimestamp": "2019-07-09T02:38:06Z", - "generateName": "kube-dns-v20-55cb5d96f7-", - "labels": { - "k8s-app": "kube-dns", - "kubernetes.io/cluster-service": "true", - "pod-template-hash": "1176185293", - "version": "v20" - }, - "name": "kube-dns-v20-55cb5d96f7-pl7sh", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "kube-dns-v20-55cb5d96f7", - "uid": "71892fd6-a1f2-11e9-9bc6-127bb0ec03b8" - } - ], - "resourceVersion": "15144050", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-dns-v20-55cb5d96f7-pl7sh", - "uid": "95046bc6-a1f2-11e9-8b08-d602e29755d5" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - }, - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "podAffinityTerm": { - "labelSelector": { - "matchExpressions": [ - { - "key": "k8s-app", - "operator": "In", - "values": [ - "kube-dns" - ] - } - ] - }, - "topologyKey": "kubernetes.io/hostname" - }, - "weight": 100 - } - ] - } - }, - "containers": [ - { - "args": [ - "--kubecfg-file=/config/kubeconfig", - "--config-dir=/kube-dns-config", - "--domain=cluster.local.", - "--dns-port=10053", - "--v=2" - ], - "env": [ - { - "name": "PROMETHEUS_PORT", - "value": "10055" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthcheck/kubedns", - "port": 10054, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "kubedns", - "ports": [ - { - "containerPort": 10053, - "name": "dns-local", - "protocol": "UDP" - }, - { - "containerPort": 10053, - "name": "dns-tcp-local", - "protocol": "TCP" - }, - { - "containerPort": 10055, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readiness", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "170Mi" - }, - "requests": { - "cpu": "100m", - "memory": "70Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/kube-dns-config", - "name": "kube-dns-config" - }, - { - "mountPath": "/config", - "name": "kubedns-kubecfg", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - }, - { - "args": [ - "-v=2", - "-logtostderr", - "-configDir=/kube-dns-config", - "-restartDnsmasq=true", - "--", - "-k", - "--cache-size=1000", - "--no-negcache", - "--no-resolv", - "--server=127.0.0.1#10053", - "--server=/cluster.local/127.0.0.1#10053", - "--server=/in-addr.arpa/127.0.0.1#10053", - "--server=/ip6.arpa/127.0.0.1#10053", - "--log-facility=-" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10", - "imagePullPolicy": "IfNotPresent", - "name": "dnsmasq", - "ports": [ - { - "containerPort": 53, - "name": "dns", - "protocol": "UDP" - }, - { - "containerPort": 53, - "name": "dns-tcp", - "protocol": "TCP" - } - ], - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/kube-dns-config", - "name": "kube-dns-config" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - }, - { - "args": [ - "--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1 \u003e/dev/null || exit 1; done", - "--url=/healthz-dnsmasq", - "--cmd=for d in $PROBE_DOMAINS; do nslookup $d 127.0.0.1:10053 \u003e/dev/null || exit 1; done", - "--url=/healthz-kubedns", - "--port=8080", - "--quiet" - ], - "env": [ - { - "name": "PROBE_DOMAINS", - "value": "bing.com kubernetes.default.svc.cluster.local" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 5, - "httpGet": { - "path": "/healthz-dnsmasq", - "port": 8080, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "healthz", - "ports": [ - { - "containerPort": 8080, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "50Mi" - }, - "requests": { - "cpu": "10m", - "memory": "50Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - }, - { - "args": [ - "--v=2", - "--logtostderr", - "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV", - "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/metrics", - "port": 10054, - "scheme": "HTTP" - }, - "initialDelaySeconds": 60, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "name": "sidecar", - "ports": [ - { - "containerPort": 10054, - "name": "metrics", - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "20Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-dns-token-ghgtl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-2", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-dns", - "serviceAccountName": "kube-dns", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "kube-dns", - "optional": true - }, - "name": "kube-dns-config" - }, - { - "configMap": { - "defaultMode": 420, - "name": "kubedns-kubecfg" - }, - "name": "kubedns-kubecfg" - }, - { - "name": "kube-dns-token-ghgtl", - "secret": { - "defaultMode": 420, - "secretName": "kube-dns-token-ghgtl" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:10Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:39:14Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:06Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://c16dce3b5c1f06c6fbfdf52edb98f9916740c0f652dc72b2fe0f9f0cc5c4c4de", - "image": "aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64:1.14.10", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/k8s-dns-dnsmasq-nanny-amd64@sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8", - "lastState": {}, - "name": "dnsmasq", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:51Z" - } - } - }, - { - "containerID": "docker://410ceb88fcbc2c3cdf19ffc5ce88adb0ba933bbc3cf446a90e669a978a7d933c", - "image": "aksrepos.azurecr.io/mirror/exechealthz-amd64:1.2", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/exechealthz-amd64@sha256:34722333f0cd0b891b61c9e0efa31913f22157e341a3aabb79967305d4e78260", - "lastState": {}, - "name": "healthz", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:58Z" - } - } - }, - { - "containerID": "docker://694f575606b51234a98b3e22d2afd04f3fa11c30b6090a901e64922eeb9fba95", - "image": "aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64:1.14.13", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/k8s-dns-kube-dns-amd64@sha256:618a82fa66cf0c75e4753369a6999032372be7308866fc9afb381789b1e5ad52", - "lastState": {}, - "name": "kubedns", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:39Z" - } - } - }, - { - "containerID": "docker://d7865fb7465b2f9cd218cdf6694018aee55260966f2bf51e6b628a86c6b9041f", - "image": "aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64:1.14.10", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/k8s-dns-sidecar-amd64@sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4", - "lastState": {}, - "name": "sidecar", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:39:04Z" - } - } - } - ], - "hostIP": "10.240.0.7", - "phase": "Running", - "podIP": "10.244.12.117", - "qosClass": "Burstable", - "startTime": "2019-07-09T02:38:10Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "aks.microsoft.com/release-time": "seconds:1566580134 nanos:758740921 ", - "remediator.aks.microsoft.com/kube-proxy-restart": "24" - }, - "creationTimestamp": "2019-08-23T17:13:13Z", - "generateName": "kube-proxy-", - "labels": { - "component": "kube-proxy", - "controller-revision-hash": "3559350992", - "pod-template-generation": "141", - "tier": "node" - }, - "name": "kube-proxy-ct2tl", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-proxy", - "uid": "45640bf6-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "19049034", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-proxy-ct2tl", - "uid": "49e373c8-c5c9-11e9-8736-86290fd7dd1f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/hyperkube", - "proxy", - "--kubeconfig=/var/lib/kubelet/kubeconfig", - "--cluster-cidr=10.244.0.0/16", - "--feature-gates=ExperimentalCriticalPodAnnotation=true", - "--v=3" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imagePullPolicy": "IfNotPresent", - "name": "kube-proxy", - "resources": { - "requests": { - "cpu": "100m" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/lib/kubelet", - "name": "kubeconfig", - "readOnly": true - }, - { - "mountPath": "/etc/kubernetes/certs", - "name": "certificates", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-proxy-token-f5vbg", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-0", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-proxy", - "serviceAccountName": "kube-proxy", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/var/lib/kubelet", - "type": "" - }, - "name": "kubeconfig" - }, - { - "hostPath": { - "path": "/etc/kubernetes/certs", - "type": "" - }, - "name": "certificates" - }, - { - "name": "kube-proxy-token-f5vbg", - "secret": { - "defaultMode": 420, - "secretName": "kube-proxy-token-f5vbg" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:13:13Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:13:23Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:13:13Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://ef115b31792ece39d1526075f9f3763f8cbf526814624795a05786d83367427e", - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "lastState": {}, - "name": "kube-proxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T17:13:22Z" - } - } - } - ], - "hostIP": "10.240.0.4", - "phase": "Running", - "podIP": "10.240.0.4", - "qosClass": "Burstable", - "startTime": "2019-08-23T17:13:13Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "aks.microsoft.com/release-time": "seconds:1566580134 nanos:758740921 ", - "remediator.aks.microsoft.com/kube-proxy-restart": "24" - }, - "creationTimestamp": "2019-08-23T17:10:52Z", - "generateName": "kube-proxy-", - "labels": { - "component": "kube-proxy", - "controller-revision-hash": "3559350992", - "pod-template-generation": "141", - "tier": "node" - }, - "name": "kube-proxy-d59xd", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-proxy", - "uid": "45640bf6-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "19048698", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-proxy-d59xd", - "uid": "f65e6a62-c5c8-11e9-8736-86290fd7dd1f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/hyperkube", - "proxy", - "--kubeconfig=/var/lib/kubelet/kubeconfig", - "--cluster-cidr=10.244.0.0/16", - "--feature-gates=ExperimentalCriticalPodAnnotation=true", - "--v=3" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imagePullPolicy": "IfNotPresent", - "name": "kube-proxy", - "resources": { - "requests": { - "cpu": "100m" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/lib/kubelet", - "name": "kubeconfig", - "readOnly": true - }, - { - "mountPath": "/etc/kubernetes/certs", - "name": "certificates", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-proxy-token-f5vbg", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-1", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-proxy", - "serviceAccountName": "kube-proxy", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/var/lib/kubelet", - "type": "" - }, - "name": "kubeconfig" - }, - { - "hostPath": { - "path": "/etc/kubernetes/certs", - "type": "" - }, - "name": "certificates" - }, - { - "name": "kube-proxy-token-f5vbg", - "secret": { - "defaultMode": 420, - "secretName": "kube-proxy-token-f5vbg" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:10:52Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:11:05Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:10:52Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://c4e9d0e372116b9cab048f7bb381e93b423dac2285da75f66664a473fcc043b3", - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "lastState": {}, - "name": "kube-proxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T17:11:04Z" - } - } - } - ], - "hostIP": "10.240.0.5", - "phase": "Running", - "podIP": "10.240.0.5", - "qosClass": "Burstable", - "startTime": "2019-08-23T17:10:52Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "aks.microsoft.com/release-time": "seconds:1566580134 nanos:758740921 ", - "remediator.aks.microsoft.com/kube-proxy-restart": "24" - }, - "creationTimestamp": "2019-08-23T17:12:23Z", - "generateName": "kube-proxy-", - "labels": { - "component": "kube-proxy", - "controller-revision-hash": "3559350992", - "pod-template-generation": "141", - "tier": "node" - }, - "name": "kube-proxy-kpm8j", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-proxy", - "uid": "45640bf6-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "19048942", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-proxy-kpm8j", - "uid": "2c3de48d-c5c9-11e9-8736-86290fd7dd1f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/hyperkube", - "proxy", - "--kubeconfig=/var/lib/kubelet/kubeconfig", - "--cluster-cidr=10.244.0.0/16", - "--feature-gates=ExperimentalCriticalPodAnnotation=true", - "--v=3" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imagePullPolicy": "IfNotPresent", - "name": "kube-proxy", - "resources": { - "requests": { - "cpu": "100m" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/lib/kubelet", - "name": "kubeconfig", - "readOnly": true - }, - { - "mountPath": "/etc/kubernetes/certs", - "name": "certificates", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-proxy-token-f5vbg", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-2", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-proxy", - "serviceAccountName": "kube-proxy", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/var/lib/kubelet", - "type": "" - }, - "name": "kubeconfig" - }, - { - "hostPath": { - "path": "/etc/kubernetes/certs", - "type": "" - }, - "name": "certificates" - }, - { - "name": "kube-proxy-token-f5vbg", - "secret": { - "defaultMode": 420, - "secretName": "kube-proxy-token-f5vbg" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:12:24Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:12:34Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:12:24Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://51067a965113e6d285a676e0d1e212ffbb60046aab6c4702f5554617415b2031", - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "lastState": {}, - "name": "kube-proxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T17:12:33Z" - } - } - } - ], - "hostIP": "10.240.0.7", - "phase": "Running", - "podIP": "10.240.0.7", - "qosClass": "Burstable", - "startTime": "2019-08-23T17:12:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "aks.microsoft.com/release-time": "seconds:1566580134 nanos:758740921 ", - "remediator.aks.microsoft.com/kube-proxy-restart": "24" - }, - "creationTimestamp": "2019-08-23T17:11:38Z", - "generateName": "kube-proxy-", - "labels": { - "component": "kube-proxy", - "controller-revision-hash": "3559350992", - "pod-template-generation": "141", - "tier": "node" - }, - "name": "kube-proxy-skzg4", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-proxy", - "uid": "45640bf6-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "19048774", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-proxy-skzg4", - "uid": "114f7246-c5c9-11e9-8736-86290fd7dd1f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/hyperkube", - "proxy", - "--kubeconfig=/var/lib/kubelet/kubeconfig", - "--cluster-cidr=10.244.0.0/16", - "--feature-gates=ExperimentalCriticalPodAnnotation=true", - "--v=3" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imagePullPolicy": "IfNotPresent", - "name": "kube-proxy", - "resources": { - "requests": { - "cpu": "100m" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/lib/kubelet", - "name": "kubeconfig", - "readOnly": true - }, - { - "mountPath": "/etc/kubernetes/certs", - "name": "certificates", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-proxy-token-f5vbg", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-3", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-proxy", - "serviceAccountName": "kube-proxy", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/var/lib/kubelet", - "type": "" - }, - "name": "kubeconfig" - }, - { - "hostPath": { - "path": "/etc/kubernetes/certs", - "type": "" - }, - "name": "certificates" - }, - { - "name": "kube-proxy-token-f5vbg", - "secret": { - "defaultMode": 420, - "secretName": "kube-proxy-token-f5vbg" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:11:38Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:11:42Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T17:11:38Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://a3172e9191547b0ea3eb7db629cd4bba2240f5c9d0186ea37be49d9877034541", - "image": "aksrepos.azurecr.io/mirror/hyperkube-amd64:v1.11.8", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/hyperkube-amd64@sha256:1447d5b491fcee503c9f8fb712e1593dc3772c7e661251f54c297477cc716913", - "lastState": {}, - "name": "kube-proxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T17:11:41Z" - } - } - } - ], - "hostIP": "10.240.0.6", - "phase": "Running", - "podIP": "10.240.0.6", - "qosClass": "Burstable", - "startTime": "2019-08-23T17:11:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-07-15T20:54:26Z", - "generateName": "kube-svc-redirect-", - "labels": { - "component": "kube-svc-redirect", - "controller-revision-hash": "1216437240", - "pod-template-generation": "9", - "tier": "node" - }, - "name": "kube-svc-redirect-czm8d", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-svc-redirect", - "uid": "45a5fc62-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "15831523", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-svc-redirect-czm8d", - "uid": "bb3d3ef2-a742-11e9-a38a-22d1c75c4357" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "KUBERNETES_SVC_IP", - "value": "10.0.0.1" - }, - { - "name": "KUBE_SVC_REDIRECTOR_PROXY_IP", - "value": "127.0.0.1:14612" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2", - "imagePullPolicy": "IfNotPresent", - "name": "redirector", - "resources": { - "requests": { - "cpu": "5m", - "memory": "2Mi" - } - }, - "securityContext": { - "capabilities": { - "add": [ - "NET_ADMIN" - ] - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - }, - { - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "azureproxy", - "ports": [ - { - "containerPort": 14612, - "hostPort": 14612, - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "5m", - "memory": "32Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/nginx/nginx.conf", - "name": "azureproxy-nginx", - "readOnly": true, - "subPath": "nginx.conf" - }, - { - "mountPath": "/etc/nginx/conf.d", - "name": "azureproxy-configs", - "readOnly": true - }, - { - "mountPath": "/etc/nginx/certs", - "name": "azureproxy-certs", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-0", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-svc-redirector", - "serviceAccountName": "kube-svc-redirector", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-nginx" - }, - "name": "azureproxy-nginx" - }, - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-config" - }, - "name": "azureproxy-configs" - }, - { - "name": "azureproxy-certs", - "secret": { - "defaultMode": 420, - "secretName": "azureproxy-certs" - } - }, - { - "name": "kube-svc-redirector-token-ngjg2", - "secret": { - "defaultMode": 420, - "secretName": "kube-svc-redirector-token-ngjg2" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:54:26Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:55:03Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:54:26Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://942d4ddc66e488245fa77cf331a38de7df760d5d5d96b344f5bfbc84adbab861", - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/nginx@sha256:91d22184f3f9b1be658c2cc2c12d324de7ff12c8b9c9a597905457b4d93b069d", - "lastState": {}, - "name": "azureproxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-15T20:55:02Z" - } - } - }, - { - "containerID": "docker://71d6f73215c0994fa2f7b340732d5e4453a86ece31dcf5278fb2abc32e3e4de2", - "image": "aksrepos.azurecr.io/mirror/kube-svc-redirect:v1.0.2", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "lastState": {}, - "name": "redirector", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-15T20:54:36Z" - } - } - } - ], - "hostIP": "10.240.0.4", - "phase": "Running", - "podIP": "10.240.0.4", - "qosClass": "Burstable", - "startTime": "2019-07-15T20:54:26Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-08-07T18:57:56Z", - "generateName": "kube-svc-redirect-", - "labels": { - "component": "kube-svc-redirect", - "controller-revision-hash": "1216437240", - "pod-template-generation": "9", - "tier": "node" - }, - "name": "kube-svc-redirect-mqk98", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-svc-redirect", - "uid": "45a5fc62-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "16965477", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-svc-redirect-mqk98", - "uid": "44a61692-b945-11e9-a1b6-127094e7fd94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "KUBERNETES_SVC_IP", - "value": "10.0.0.1" - }, - { - "name": "KUBE_SVC_REDIRECTOR_PROXY_IP", - "value": "127.0.0.1:14612" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2", - "imagePullPolicy": "IfNotPresent", - "name": "redirector", - "resources": { - "requests": { - "cpu": "5m", - "memory": "2Mi" - } - }, - "securityContext": { - "capabilities": { - "add": [ - "NET_ADMIN" - ] - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - }, - { - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "azureproxy", - "ports": [ - { - "containerPort": 14612, - "hostPort": 14612, - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "5m", - "memory": "32Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/nginx/nginx.conf", - "name": "azureproxy-nginx", - "readOnly": true, - "subPath": "nginx.conf" - }, - { - "mountPath": "/etc/nginx/conf.d", - "name": "azureproxy-configs", - "readOnly": true - }, - { - "mountPath": "/etc/nginx/certs", - "name": "azureproxy-certs", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-3", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-svc-redirector", - "serviceAccountName": "kube-svc-redirector", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-nginx" - }, - "name": "azureproxy-nginx" - }, - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-config" - }, - "name": "azureproxy-configs" - }, - { - "name": "azureproxy-certs", - "secret": { - "defaultMode": 420, - "secretName": "azureproxy-certs" - } - }, - { - "name": "kube-svc-redirector-token-ngjg2", - "secret": { - "defaultMode": 420, - "secretName": "kube-svc-redirector-token-ngjg2" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-07T18:57:58Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-07T18:58:09Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-07T18:57:58Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://5f47547dc8e4fceb8e2a6e01cee5612b49e2dc2d5682b6a58f648d8223b3a6b0", - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/nginx@sha256:91d22184f3f9b1be658c2cc2c12d324de7ff12c8b9c9a597905457b4d93b069d", - "lastState": {}, - "name": "azureproxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-07T18:58:09Z" - } - } - }, - { - "containerID": "docker://5da4e17288399f8e2d4998e5c06159d0d2d39690e89195c5381ab7e3c91aaf99", - "image": "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2", - "imageID": "docker-pullable://aksrepos.azurecr.io/prod/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "lastState": {}, - "name": "redirector", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-07T18:58:08Z" - } - } - } - ], - "hostIP": "10.240.0.6", - "phase": "Running", - "podIP": "10.240.0.6", - "qosClass": "Burstable", - "startTime": "2019-08-07T18:57:58Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-07-15T20:55:38Z", - "generateName": "kube-svc-redirect-", - "labels": { - "component": "kube-svc-redirect", - "controller-revision-hash": "1216437240", - "pod-template-generation": "9", - "tier": "node" - }, - "name": "kube-svc-redirect-qf4tl", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-svc-redirect", - "uid": "45a5fc62-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "15144014", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-svc-redirect-qf4tl", - "uid": "e690309f-a742-11e9-a38a-22d1c75c4357" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "KUBERNETES_SVC_IP", - "value": "10.0.0.1" - }, - { - "name": "KUBE_SVC_REDIRECTOR_PROXY_IP", - "value": "127.0.0.1:14612" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2", - "imagePullPolicy": "IfNotPresent", - "name": "redirector", - "resources": { - "requests": { - "cpu": "5m", - "memory": "2Mi" - } - }, - "securityContext": { - "capabilities": { - "add": [ - "NET_ADMIN" - ] - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - }, - { - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "azureproxy", - "ports": [ - { - "containerPort": 14612, - "hostPort": 14612, - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "5m", - "memory": "32Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/nginx/nginx.conf", - "name": "azureproxy-nginx", - "readOnly": true, - "subPath": "nginx.conf" - }, - { - "mountPath": "/etc/nginx/conf.d", - "name": "azureproxy-configs", - "readOnly": true - }, - { - "mountPath": "/etc/nginx/certs", - "name": "azureproxy-certs", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-1", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-svc-redirector", - "serviceAccountName": "kube-svc-redirector", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-nginx" - }, - "name": "azureproxy-nginx" - }, - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-config" - }, - "name": "azureproxy-configs" - }, - { - "name": "azureproxy-certs", - "secret": { - "defaultMode": 420, - "secretName": "azureproxy-certs" - } - }, - { - "name": "kube-svc-redirector-token-ngjg2", - "secret": { - "defaultMode": 420, - "secretName": "kube-svc-redirector-token-ngjg2" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:55:38Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:55:47Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:55:38Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://a0fa774ceba9ae78cf75ffb96a0d8f3ca4d48e5d9d17218957b07e8b1e7e2862", - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/nginx@sha256:91d22184f3f9b1be658c2cc2c12d324de7ff12c8b9c9a597905457b4d93b069d", - "lastState": {}, - "name": "azureproxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-15T20:55:46Z" - } - } - }, - { - "containerID": "docker://7f281954c57ff6529aaeea2e79dc45a8abeabd4b360c2bbea5c0830ddac4f093", - "image": "aksrepos.azurecr.io/mirror/kube-svc-redirect:v1.0.2", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "lastState": {}, - "name": "redirector", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-15T20:55:44Z" - } - } - } - ], - "hostIP": "10.240.0.5", - "phase": "Running", - "podIP": "10.240.0.5", - "qosClass": "Burstable", - "startTime": "2019-07-15T20:55:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-07-15T20:56:33Z", - "generateName": "kube-svc-redirect-", - "labels": { - "component": "kube-svc-redirect", - "controller-revision-hash": "1216437240", - "pod-template-generation": "9", - "tier": "node" - }, - "name": "kube-svc-redirect-rtw2t", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "kube-svc-redirect", - "uid": "45a5fc62-44e5-11e9-9920-423525a6b683" - } - ], - "resourceVersion": "15144039", - "selfLink": "/api/v1/namespaces/kube-system/pods/kube-svc-redirect-rtw2t", - "uid": "06fef5f6-a743-11e9-a38a-22d1c75c4357" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "KUBERNETES_SVC_IP", - "value": "10.0.0.1" - }, - { - "name": "KUBE_SVC_REDIRECTOR_PROXY_IP", - "value": "127.0.0.1:14612" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/prod/kube-svc-redirect:v1.0.2", - "imagePullPolicy": "IfNotPresent", - "name": "redirector", - "resources": { - "requests": { - "cpu": "5m", - "memory": "2Mi" - } - }, - "securityContext": { - "capabilities": { - "add": [ - "NET_ADMIN" - ] - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - }, - { - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "azureproxy", - "ports": [ - { - "containerPort": 14612, - "hostPort": 14612, - "protocol": "TCP" - } - ], - "resources": { - "requests": { - "cpu": "5m", - "memory": "32Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/nginx/nginx.conf", - "name": "azureproxy-nginx", - "readOnly": true, - "subPath": "nginx.conf" - }, - { - "mountPath": "/etc/nginx/conf.d", - "name": "azureproxy-configs", - "readOnly": true - }, - { - "mountPath": "/etc/nginx/certs", - "name": "azureproxy-certs", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-svc-redirector-token-ngjg2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-2", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-svc-redirector", - "serviceAccountName": "kube-svc-redirector", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-nginx" - }, - "name": "azureproxy-nginx" - }, - { - "configMap": { - "defaultMode": 420, - "name": "azureproxy-config" - }, - "name": "azureproxy-configs" - }, - { - "name": "azureproxy-certs", - "secret": { - "defaultMode": 420, - "secretName": "azureproxy-certs" - } - }, - { - "name": "kube-svc-redirector-token-ngjg2", - "secret": { - "defaultMode": 420, - "secretName": "kube-svc-redirector-token-ngjg2" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:56:33Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:56:49Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-15T20:56:33Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://aaea93b1e6a0c55e9ac0c002ffa6fdfb99e98b2f1a38c474cc2b9b65e947b6d9", - "image": "aksrepos.azurecr.io/mirror/nginx:1.13.12-alpine", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/nginx@sha256:91d22184f3f9b1be658c2cc2c12d324de7ff12c8b9c9a597905457b4d93b069d", - "lastState": {}, - "name": "azureproxy", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-15T20:56:48Z" - } - } - }, - { - "containerID": "docker://c03c8b9e99095205945e15bef5f60c0501c8a0a77186afc1fcc8eb0804274e78", - "image": "aksrepos.azurecr.io/mirror/kube-svc-redirect:v1.0.2", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/kube-svc-redirect@sha256:a448687b78d24dae388bd3d54591c179c891fa078404752bc9c9dfdaecdc02ef", - "lastState": {}, - "name": "redirector", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-15T20:56:43Z" - } - } - } - ], - "hostIP": "10.240.0.7", - "phase": "Running", - "podIP": "10.240.0.7", - "qosClass": "Burstable", - "startTime": "2019-07-15T20:56:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-07-09T02:38:07Z", - "generateName": "kubernetes-dashboard-6dcdfcd68b-", - "labels": { - "k8s-app": "kubernetes-dashboard", - "kubernetes.io/cluster-service": "true", - "pod-template-hash": "2878978246" - }, - "name": "kubernetes-dashboard-6dcdfcd68b-nfqbf", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "kubernetes-dashboard-6dcdfcd68b", - "uid": "71ff2821-a1f2-11e9-9bc6-127bb0ec03b8" - } - ], - "resourceVersion": "15831517", - "selfLink": "/api/v1/namespaces/kube-system/pods/kubernetes-dashboard-6dcdfcd68b-nfqbf", - "uid": "9583b2ab-a1f2-11e9-8b08-d602e29755d5" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64:v1.10.1", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/", - "port": 9090, - "scheme": "HTTP" - }, - "initialDelaySeconds": 30, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 30 - }, - "name": "main", - "ports": [ - { - "containerPort": 9090, - "name": "http", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "100m", - "memory": "500Mi" - }, - "requests": { - "cpu": "100m", - "memory": "50Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kubernetes-dashboard-token-w4t8s", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-0", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kubernetes-dashboard", - "serviceAccountName": "kubernetes-dashboard", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kubernetes-dashboard-token-w4t8s", - "secret": { - "defaultMode": 420, - "secretName": "kubernetes-dashboard-token-w4t8s" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:14Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:39:08Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:07Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://2b042ce7bdf3d03cb606317b19ee797cbf7b99c65076a67001064bccb313b3cb", - "image": "aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64:v1.10.1", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/kubernetes-dashboard-amd64@sha256:0ae6b69432e78069c5ce2bcde0fe409c5c4d6f0f4d9cd50a17974fea38898747", - "lastState": {}, - "name": "main", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:39:07Z" - } - } - } - ], - "hostIP": "10.240.0.4", - "phase": "Running", - "podIP": "10.244.1.197", - "qosClass": "Burstable", - "startTime": "2019-07-09T02:38:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-07-09T02:38:06Z", - "generateName": "metrics-server-76cd9fb66-", - "labels": { - "k8s-app": "metrics-server", - "pod-template-hash": "327859622" - }, - "name": "metrics-server-76cd9fb66-h2q55", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "metrics-server-76cd9fb66", - "uid": "71c837df-a1f2-11e9-9bc6-127bb0ec03b8" - } - ], - "resourceVersion": "15144037", - "selfLink": "/api/v1/namespaces/kube-system/pods/metrics-server-76cd9fb66-h2q55", - "uid": "9543dbb7-a1f2-11e9-8b08-d602e29755d5" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "command": [ - "/metrics-server", - "--source=kubernetes.summary_api:''" - ], - "env": [ - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/mirror/metrics-server-amd64:v0.2.1", - "imagePullPolicy": "IfNotPresent", - "name": "metrics-server", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "metrics-server-token-qtdgm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-1", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "metrics-server", - "serviceAccountName": "metrics-server", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "metrics-server-token-qtdgm", - "secret": { - "defaultMode": 420, - "secretName": "metrics-server-token-qtdgm" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:09Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:20Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-07-09T02:38:07Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://f60ef82657e5ccdfb611a4f3381848dff77a01bddf95c431e4b7a2bf6f4b8087", - "image": "aksrepos.azurecr.io/mirror/metrics-server-amd64:v0.2.1", - "imageID": "docker-pullable://aksrepos.azurecr.io/mirror/metrics-server-amd64@sha256:220c0ed3451cb95e4b2f72dd5dc8d9d39d9f529722e5b29d8286373ce27b117e", - "lastState": {}, - "name": "metrics-server", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-07-09T02:38:18Z" - } - } - } - ], - "hostIP": "10.240.0.5", - "phase": "Running", - "podIP": "10.244.0.193", - "qosClass": "BestEffort", - "startTime": "2019-07-09T02:38:09Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "agentVersion": "1.10.0.1", - "dockerProviderVersion": "6.0.0-0", - "schema-versions": "v1" - }, - "creationTimestamp": "2019-08-23T19:53:57Z", - "generateName": "omsagent-", - "labels": { - "controller-revision-hash": "868116844", - "dsName": "omsagent-ds", - "pod-template-generation": "9" - }, - "name": "omsagent-25pks", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "omsagent", - "uid": "e2f8c552-c2d2-11e9-8736-86290fd7dd1f" - } - ], - "resourceVersion": "19063729", - "selfLink": "/api/v1/namespaces/kube-system/pods/omsagent-25pks", - "uid": "be78d7f6-c5df-11e9-8736-86290fd7dd1f" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "AKS_RESOURCE_ID", - "value": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test" - }, - { - "name": "AKS_REGION", - "value": "eastus" - }, - { - "name": "CONTROLLER_TYPE", - "value": "DaemonSet" - }, - { - "name": "NODE_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/bin/bash", - "-c", - "/opt/livenessprobe.sh" - ] - }, - "failureThreshold": 3, - "initialDelaySeconds": 60, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "omsagent", - "ports": [ - { - "containerPort": 25225, - "protocol": "TCP" - }, - { - "containerPort": 25224, - "protocol": "UDP" - } - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "600Mi" - }, - "requests": { - "cpu": "75m", - "memory": "225Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/hostfs", - "name": "host-root", - "readOnly": true - }, - { - "mountPath": "/var/run/host", - "name": "docker-sock" - }, - { - "mountPath": "/var/log", - "name": "host-log" - }, - { - "mountPath": "/var/lib/docker/containers", - "name": "containerlog-path" - }, - { - "mountPath": "/etc/kubernetes/host", - "name": "azure-json-path" - }, - { - "mountPath": "/etc/omsagent-secret", - "name": "omsagent-secret" - }, - { - "mountPath": "/etc/config/settings", - "name": "settings-vol-config", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "omsagent-token-fjmqb", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "nodeName": "aks-nodepool1-19574989-2", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "omsagent", - "serviceAccountName": "omsagent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "node-role.kubernetes.io/master", - "operator": "Equal", - "value": "true" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/", - "type": "" - }, - "name": "host-root" - }, - { - "hostPath": { - "path": "/var/run", - "type": "" - }, - "name": "docker-sock" - }, - { - "hostPath": { - "path": "/etc/hostname", - "type": "" - }, - "name": "container-hostname" - }, - { - "hostPath": { - "path": "/var/log", - "type": "" - }, - "name": "host-log" - }, - { - "hostPath": { - "path": "/var/lib/docker/containers", - "type": "" - }, - "name": "containerlog-path" - }, - { - "hostPath": { - "path": "/etc/kubernetes", - "type": "" - }, - "name": "azure-json-path" - }, - { - "name": "omsagent-secret", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-secret" - } - }, - { - "configMap": { - "defaultMode": 420, - "name": "container-azm-ms-agentconfig", - "optional": true - }, - "name": "settings-vol-config" - }, - { - "name": "omsagent-token-fjmqb", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-token-fjmqb" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:53:57Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:54:44Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:53:57Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://acd5cedc2c5874122047c47bb1398f35a7c0297292fc4a0e01345123c233d19a", - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imageID": "docker-pullable://mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "lastState": {}, - "name": "omsagent", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T19:54:43Z" - } - } - } - ], - "hostIP": "10.240.0.7", - "phase": "Running", - "podIP": "10.244.12.169", - "qosClass": "Burstable", - "startTime": "2019-08-23T19:53:57Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "agentVersion": "1.10.0.1", - "dockerProviderVersion": "6.0.0-0", - "schema-versions": "v1" - }, - "creationTimestamp": "2019-08-23T19:51:35Z", - "generateName": "omsagent-", - "labels": { - "controller-revision-hash": "868116844", - "dsName": "omsagent-ds", - "pod-template-generation": "9" - }, - "name": "omsagent-4tncr", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "omsagent", - "uid": "e2f8c552-c2d2-11e9-8736-86290fd7dd1f" - } - ], - "resourceVersion": "19063468", - "selfLink": "/api/v1/namespaces/kube-system/pods/omsagent-4tncr", - "uid": "69e68b21-c5df-11e9-8736-86290fd7dd1f" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "AKS_RESOURCE_ID", - "value": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test" - }, - { - "name": "AKS_REGION", - "value": "eastus" - }, - { - "name": "CONTROLLER_TYPE", - "value": "DaemonSet" - }, - { - "name": "NODE_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/bin/bash", - "-c", - "/opt/livenessprobe.sh" - ] - }, - "failureThreshold": 3, - "initialDelaySeconds": 60, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "omsagent", - "ports": [ - { - "containerPort": 25225, - "protocol": "TCP" - }, - { - "containerPort": 25224, - "protocol": "UDP" - } - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "600Mi" - }, - "requests": { - "cpu": "75m", - "memory": "225Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/hostfs", - "name": "host-root", - "readOnly": true - }, - { - "mountPath": "/var/run/host", - "name": "docker-sock" - }, - { - "mountPath": "/var/log", - "name": "host-log" - }, - { - "mountPath": "/var/lib/docker/containers", - "name": "containerlog-path" - }, - { - "mountPath": "/etc/kubernetes/host", - "name": "azure-json-path" - }, - { - "mountPath": "/etc/omsagent-secret", - "name": "omsagent-secret" - }, - { - "mountPath": "/etc/config/settings", - "name": "settings-vol-config", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "omsagent-token-fjmqb", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "nodeName": "aks-nodepool1-19574989-1", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "omsagent", - "serviceAccountName": "omsagent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "node-role.kubernetes.io/master", - "operator": "Equal", - "value": "true" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/", - "type": "" - }, - "name": "host-root" - }, - { - "hostPath": { - "path": "/var/run", - "type": "" - }, - "name": "docker-sock" - }, - { - "hostPath": { - "path": "/etc/hostname", - "type": "" - }, - "name": "container-hostname" - }, - { - "hostPath": { - "path": "/var/log", - "type": "" - }, - "name": "host-log" - }, - { - "hostPath": { - "path": "/var/lib/docker/containers", - "type": "" - }, - "name": "containerlog-path" - }, - { - "hostPath": { - "path": "/etc/kubernetes", - "type": "" - }, - "name": "azure-json-path" - }, - { - "name": "omsagent-secret", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-secret" - } - }, - { - "configMap": { - "defaultMode": 420, - "name": "container-azm-ms-agentconfig", - "optional": true - }, - "name": "settings-vol-config" - }, - { - "name": "omsagent-token-fjmqb", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-token-fjmqb" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:51:35Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:52:28Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:51:35Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://7803b80452aa34460c848d9c1ca65d6bd925665cf78faaa8dbc122482f93c744", - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imageID": "docker-pullable://mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "lastState": {}, - "name": "omsagent", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T19:52:27Z" - } - } - } - ], - "hostIP": "10.240.0.5", - "phase": "Running", - "podIP": "10.244.0.251", - "qosClass": "Burstable", - "startTime": "2019-08-23T19:51:35Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "agentVersion": "1.10.0.1", - "dockerProviderVersion": "6.0.0-0", - "schema-versions": "v1" - }, - "creationTimestamp": "2019-08-23T19:53:36Z", - "generateName": "omsagent-", - "labels": { - "controller-revision-hash": "868116844", - "dsName": "omsagent-ds", - "pod-template-generation": "9" - }, - "name": "omsagent-h44fk", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "omsagent", - "uid": "e2f8c552-c2d2-11e9-8736-86290fd7dd1f" - } - ], - "resourceVersion": "19063631", - "selfLink": "/api/v1/namespaces/kube-system/pods/omsagent-h44fk", - "uid": "b1e04e1c-c5df-11e9-8736-86290fd7dd1f" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "AKS_RESOURCE_ID", - "value": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test" - }, - { - "name": "AKS_REGION", - "value": "eastus" - }, - { - "name": "CONTROLLER_TYPE", - "value": "DaemonSet" - }, - { - "name": "NODE_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/bin/bash", - "-c", - "/opt/livenessprobe.sh" - ] - }, - "failureThreshold": 3, - "initialDelaySeconds": 60, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "omsagent", - "ports": [ - { - "containerPort": 25225, - "protocol": "TCP" - }, - { - "containerPort": 25224, - "protocol": "UDP" - } - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "600Mi" - }, - "requests": { - "cpu": "75m", - "memory": "225Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/hostfs", - "name": "host-root", - "readOnly": true - }, - { - "mountPath": "/var/run/host", - "name": "docker-sock" - }, - { - "mountPath": "/var/log", - "name": "host-log" - }, - { - "mountPath": "/var/lib/docker/containers", - "name": "containerlog-path" - }, - { - "mountPath": "/etc/kubernetes/host", - "name": "azure-json-path" - }, - { - "mountPath": "/etc/omsagent-secret", - "name": "omsagent-secret" - }, - { - "mountPath": "/etc/config/settings", - "name": "settings-vol-config", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "omsagent-token-fjmqb", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "nodeName": "aks-nodepool1-19574989-0", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "omsagent", - "serviceAccountName": "omsagent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "node-role.kubernetes.io/master", - "operator": "Equal", - "value": "true" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/", - "type": "" - }, - "name": "host-root" - }, - { - "hostPath": { - "path": "/var/run", - "type": "" - }, - "name": "docker-sock" - }, - { - "hostPath": { - "path": "/etc/hostname", - "type": "" - }, - "name": "container-hostname" - }, - { - "hostPath": { - "path": "/var/log", - "type": "" - }, - "name": "host-log" - }, - { - "hostPath": { - "path": "/var/lib/docker/containers", - "type": "" - }, - "name": "containerlog-path" - }, - { - "hostPath": { - "path": "/etc/kubernetes", - "type": "" - }, - "name": "azure-json-path" - }, - { - "name": "omsagent-secret", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-secret" - } - }, - { - "configMap": { - "defaultMode": 420, - "name": "container-azm-ms-agentconfig", - "optional": true - }, - "name": "settings-vol-config" - }, - { - "name": "omsagent-token-fjmqb", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-token-fjmqb" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:53:36Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:53:51Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:53:36Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://4b71a82e472a8e5d0bc4ef9b9b5d2ccf25741b31269480a77e29424ebe87757c", - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imageID": "docker-pullable://mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "lastState": {}, - "name": "omsagent", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T19:53:49Z" - } - } - } - ], - "hostIP": "10.240.0.4", - "phase": "Running", - "podIP": "10.244.1.35", - "qosClass": "Burstable", - "startTime": "2019-08-23T19:53:36Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "agentVersion": "1.10.0.1", - "dockerProviderVersion": "6.0.0-0", - "schema-versions": "v1" - }, - "creationTimestamp": "2019-08-23T19:51:28Z", - "generateName": "omsagent-rs-5bb85d7468-", - "labels": { - "pod-template-hash": "1664183024", - "rsName": "omsagent-rs" - }, - "name": "omsagent-rs-5bb85d7468-dnxpw", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "omsagent-rs-5bb85d7468", - "uid": "659ec974-c5df-11e9-8736-86290fd7dd1f" - } - ], - "resourceVersion": "19063495", - "selfLink": "/api/v1/namespaces/kube-system/pods/omsagent-rs-5bb85d7468-dnxpw", - "uid": "65a6f978-c5df-11e9-8736-86290fd7dd1f" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "AKS_RESOURCE_ID", - "value": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test" - }, - { - "name": "AKS_REGION", - "value": "eastus" - }, - { - "name": "CONTROLLER_TYPE", - "value": "ReplicaSet" - }, - { - "name": "NODE_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/bin/bash", - "-c", - "/opt/livenessprobe.sh" - ] - }, - "failureThreshold": 3, - "initialDelaySeconds": 60, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "omsagent", - "ports": [ - { - "containerPort": 25225, - "protocol": "TCP" - }, - { - "containerPort": 25224, - "protocol": "UDP" - }, - { - "containerPort": 25227, - "name": "in-rs-tcp", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "500Mi" - }, - "requests": { - "cpu": "110m", - "memory": "250Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/host", - "name": "docker-sock" - }, - { - "mountPath": "/var/log", - "name": "host-log" - }, - { - "mountPath": "/var/lib/docker/containers", - "name": "containerlog-path" - }, - { - "mountPath": "/etc/kubernetes/host", - "name": "azure-json-path" - }, - { - "mountPath": "/etc/omsagent-secret", - "name": "omsagent-secret", - "readOnly": true - }, - { - "mountPath": "/etc/config", - "name": "omsagent-rs-config" - }, - { - "mountPath": "/etc/config/settings", - "name": "settings-vol-config", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "omsagent-token-fjmqb", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "nodeName": "aks-nodepool1-19574989-0", - "nodeSelector": { - "beta.kubernetes.io/os": "linux", - "kubernetes.io/role": "agent" - }, - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "omsagent", - "serviceAccountName": "omsagent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "hostPath": { - "path": "/var/run", - "type": "" - }, - "name": "docker-sock" - }, - { - "hostPath": { - "path": "/etc/hostname", - "type": "" - }, - "name": "container-hostname" - }, - { - "hostPath": { - "path": "/var/log", - "type": "" - }, - "name": "host-log" - }, - { - "hostPath": { - "path": "/var/lib/docker/containers", - "type": "" - }, - "name": "containerlog-path" - }, - { - "hostPath": { - "path": "/etc/kubernetes", - "type": "" - }, - "name": "azure-json-path" - }, - { - "name": "omsagent-secret", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-secret" - } - }, - { - "configMap": { - "defaultMode": 420, - "name": "omsagent-rs-config" - }, - "name": "omsagent-rs-config" - }, - { - "configMap": { - "defaultMode": 420, - "name": "container-azm-ms-agentconfig", - "optional": true - }, - "name": "settings-vol-config" - }, - { - "name": "omsagent-token-fjmqb", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-token-fjmqb" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:51:28Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:52:37Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:51:28Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://7e080036bc213a7dadd95b1d8439e06a1b62822219642a83cab059dc4292b0e5", - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imageID": "docker-pullable://mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "lastState": {}, - "name": "omsagent", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T19:52:37Z" - } - } - } - ], - "hostIP": "10.240.0.4", - "phase": "Running", - "podIP": "10.244.1.34", - "qosClass": "Burstable", - "startTime": "2019-08-23T19:51:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "agentVersion": "1.10.0.1", - "dockerProviderVersion": "6.0.0-0", - "schema-versions": "v1" - }, - "creationTimestamp": "2019-08-23T19:52:35Z", - "generateName": "omsagent-", - "labels": { - "controller-revision-hash": "868116844", - "dsName": "omsagent-ds", - "pod-template-generation": "9" - }, - "name": "omsagent-sb6xx", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "omsagent", - "uid": "e2f8c552-c2d2-11e9-8736-86290fd7dd1f" - } - ], - "resourceVersion": "19063577", - "selfLink": "/api/v1/namespaces/kube-system/pods/omsagent-sb6xx", - "uid": "8dbd5e8b-c5df-11e9-8736-86290fd7dd1f" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "AKS_RESOURCE_ID", - "value": "/subscriptions/72c8e8ca-dc16-47dc-b65c-6b5875eb600a/resourcegroups/dilipr-health-test/providers/Microsoft.ContainerService/managedClusters/dilipr-health-test" - }, - { - "name": "AKS_REGION", - "value": "eastus" - }, - { - "name": "CONTROLLER_TYPE", - "value": "DaemonSet" - }, - { - "name": "NODE_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/bin/bash", - "-c", - "/opt/livenessprobe.sh" - ] - }, - "failureThreshold": 3, - "initialDelaySeconds": 60, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "omsagent", - "ports": [ - { - "containerPort": 25225, - "protocol": "TCP" - }, - { - "containerPort": 25224, - "protocol": "UDP" - } - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "600Mi" - }, - "requests": { - "cpu": "75m", - "memory": "225Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/hostfs", - "name": "host-root", - "readOnly": true - }, - { - "mountPath": "/var/run/host", - "name": "docker-sock" - }, - { - "mountPath": "/var/log", - "name": "host-log" - }, - { - "mountPath": "/var/lib/docker/containers", - "name": "containerlog-path" - }, - { - "mountPath": "/etc/kubernetes/host", - "name": "azure-json-path" - }, - { - "mountPath": "/etc/omsagent-secret", - "name": "omsagent-secret" - }, - { - "mountPath": "/etc/config/settings", - "name": "settings-vol-config", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "omsagent-token-fjmqb", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "nodeName": "aks-nodepool1-19574989-3", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "omsagent", - "serviceAccountName": "omsagent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "node-role.kubernetes.io/master", - "operator": "Equal", - "value": "true" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/", - "type": "" - }, - "name": "host-root" - }, - { - "hostPath": { - "path": "/var/run", - "type": "" - }, - "name": "docker-sock" - }, - { - "hostPath": { - "path": "/etc/hostname", - "type": "" - }, - "name": "container-hostname" - }, - { - "hostPath": { - "path": "/var/log", - "type": "" - }, - "name": "host-log" - }, - { - "hostPath": { - "path": "/var/lib/docker/containers", - "type": "" - }, - "name": "containerlog-path" - }, - { - "hostPath": { - "path": "/etc/kubernetes", - "type": "" - }, - "name": "azure-json-path" - }, - { - "name": "omsagent-secret", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-secret" - } - }, - { - "configMap": { - "defaultMode": 420, - "name": "container-azm-ms-agentconfig", - "optional": true - }, - "name": "settings-vol-config" - }, - { - "name": "omsagent-token-fjmqb", - "secret": { - "defaultMode": 420, - "secretName": "omsagent-token-fjmqb" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:52:35Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:53:25Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-23T19:52:35Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://f4f0cb19e5da394a4332847953c18d9321319f2ef422533b890ab844cb997879", - "image": "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08222019", - "imageID": "docker-pullable://mcr.microsoft.com/azuremonitor/containerinsights/ciprod@sha256:69b420bdb4081293c37e2d0f8ad2e4054bd516f5c08c7512d6b695660a36eccf", - "lastState": {}, - "name": "omsagent", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-23T19:53:24Z" - } - } - } - ], - "hostIP": "10.240.0.6", - "phase": "Running", - "podIP": "10.244.2.62", - "qosClass": "Burstable", - "startTime": "2019-08-23T19:52:35Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2019-08-12T20:28:08Z", - "generateName": "tunnelfront-65c8cfb7cc-", - "labels": { - "component": "tunnel", - "pod-template-hash": "2174796377" - }, - "name": "tunnelfront-65c8cfb7cc-z8srb", - "namespace": "kube-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "tunnelfront-65c8cfb7cc", - "uid": "7013afa3-a742-11e9-a08d-96dd47774ee5" - } - ], - "resourceVersion": "17628809", - "selfLink": "/api/v1/namespaces/kube-system/pods/tunnelfront-65c8cfb7cc-z8srb", - "uid": "b2a0e1b3-bd3f-11e9-b2a7-d61658c73830" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "kubernetes.azure.com/cluster", - "operator": "Exists" - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "OVERRIDE_TUNNEL_SERVER_NAME", - "value": "t_dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "TUNNEL_CLUSTERUSER_NAME", - "value": "28957308" - }, - { - "name": "TUNNELGATEWAY_SERVER_NAME", - "value": "dilipr-hea-dilipr-health-te-72c8e8-0b16acad.tun.eastus.azmk8s.io" - }, - { - "name": "TUNNELGATEWAY_SSH_PORT", - "value": "22" - }, - { - "name": "TUNNELGATEWAY_TLS_PORT", - "value": "443" - }, - { - "name": "KUBE_CONFIG", - "value": "/etc/kubernetes/kubeconfig/kubeconfig" - }, - { - "name": "KUBERNETES_PORT_443_TCP_ADDR", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - }, - { - "name": "KUBERNETES_PORT", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_PORT_443_TCP", - "value": "tcp://dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io:443" - }, - { - "name": "KUBERNETES_SERVICE_HOST", - "value": "dilipr-hea-dilipr-health-te-72c8e8-d3ccfd8f.hcp.eastus.azmk8s.io" - } - ], - "image": "aksrepos.azurecr.io/prod/hcp-tunnel-front:v1.9.2-v4.0.7", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "exec": { - "command": [ - "/lib/tunnel-front/check-tunnel-connection.sh" - ] - }, - "failureThreshold": 12, - "initialDelaySeconds": 10, - "periodSeconds": 60, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "tunnel-front", - "resources": { - "requests": { - "cpu": "10m", - "memory": "64Mi" - } - }, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/etc/kubernetes/kubeconfig", - "name": "kubeconfig", - "readOnly": true - }, - { - "mountPath": "/etc/kubernetes/certs", - "name": "certificates", - "readOnly": true - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "tunnelfront-token-njgvg", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "Default", - "imagePullSecrets": [ - { - "name": "emptyacrsecret" - } - ], - "nodeName": "aks-nodepool1-19574989-3", - "nodeSelector": { - "beta.kubernetes.io/os": "linux" - }, - "priority": 2000001000, - "priorityClassName": "system-node-critical", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "tunnelfront", - "serviceAccountName": "tunnelfront", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "key": "CriticalAddonsOnly", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "configMap": { - "defaultMode": 420, - "name": "tunnelfront-kubecfg", - "optional": true - }, - "name": "kubeconfig" - }, - { - "hostPath": { - "path": "/etc/kubernetes/certs", - "type": "" - }, - "name": "certificates" - }, - { - "name": "tunnelfront-token-njgvg", - "secret": { - "defaultMode": 420, - "secretName": "tunnelfront-token-njgvg" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-12T20:28:08Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-12T20:28:13Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": null, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2019-08-12T20:28:08Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://ac3b7482b15ba1f825e7a9ceef11defaccdc2682b9a20bb7c98bc307a8a34cf6", - "image": "aksrepos.azurecr.io/prod/hcp-tunnel-front:v1.9.2-v4.0.7", - "imageID": "docker-pullable://aksrepos.azurecr.io/prod/hcp-tunnel-front@sha256:68878ee3ea1781b322ea3952c3370e31dd89be8bb0864e2bf27bdba6dc904c41", - "lastState": {}, - "name": "tunnel-front", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2019-08-12T20:28:13Z" - } - } - } - ], - "hostIP": "10.240.0.6", - "phase": "Running", - "podIP": "10.244.2.10", - "qosClass": "Burstable", - "startTime": "2019-08-12T20:28:08Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": "" - } -} diff --git a/test/unit-tests/plugins/health/test_health_model_definition.json b/test/unit-tests/plugins/health/test_health_model_definition.json deleted file mode 100644 index 31d219705..000000000 --- a/test/unit-tests/plugins/health/test_health_model_definition.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "monitor_id": "monitor_id", - "parent_monitor_id": "parent_monitor_id", - "labels": [ - "container.azm.ms/namespace", - "container.azm.ms/workload-name", - "container.azm.ms/workload-kind", - "container.azm.ms/cluster-region", - "container.azm.ms/cluster-subscription-id", - "container.azm.ms/cluster-resource-group", - "container.azm.ms/cluster-name" - ] - }, - { - "monitor_id": "conditional_monitor_id", - "aggregation_algorithm": "worstOf", - "labels": [ - "kubernetes.io/hostname", - "agentpool", - "kubernetes.io/role", - "container.azm.ms/cluster-region", - "container.azm.ms/cluster-subscription-id", - "container.azm.ms/cluster-resource-group", - "container.azm.ms/cluster-name" - ], - "parent_monitor_id": [ - { - "label": "kubernetes.io/role", - "operator": "==", - "value": "master", - "id": "master_node_pool" - }, - { - "label": "kubernetes.io/role", - "operator": "==", - "value": "agent", - "id": "agent_node_pool" - } - ] - } -] \ No newline at end of file diff --git a/test/unit-tests/plugins/health/unit_monitor_spec.rb b/test/unit-tests/plugins/health/unit_monitor_spec.rb deleted file mode 100644 index 530c98290..000000000 --- a/test/unit-tests/plugins/health/unit_monitor_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -require_relative '../../../../source/plugins/ruby/health/unit_monitor' -require_relative '../test_helpers' - -include HealthModel - -describe "UnitMonitor Spec" do - it "is_aggregate_monitor is false for UnitMonitor" do - # Arrange/Act - monitor = UnitMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, {}, {}, {}) - # Assert - assert_equal monitor.is_aggregate_monitor, false - end - - it "get_member_monitors is nil for UnitMonitor" do - # Arrange/Act - monitor = UnitMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, {}, {}, {}) - #Assert - assert_nil monitor.get_member_monitors - end -end \ No newline at end of file diff --git a/test/unit-tests/plugins/health/unit_monitor_test.rb b/test/unit-tests/plugins/health/unit_monitor_test.rb deleted file mode 100644 index d46ae5665..000000000 --- a/test/unit-tests/plugins/health/unit_monitor_test.rb +++ /dev/null @@ -1,16 +0,0 @@ -require_relative '../../../../source/plugins/ruby/health/unit_monitor' -require_relative '../test_helpers' - -class UnitMonitorTest < Minitest::Test - include HealthModel - - def test_is_aggregate_monitor_false - monitor = UnitMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, {}, {}, {}) - assert_equal monitor.is_aggregate_monitor, false - end - - def test_get_member_monitors_nil - monitor = UnitMonitor.new(:monitor_id, :monitor_instance_id, :pass, :time, {}, {}, {}) - assert_nil monitor.get_member_monitors - end -end diff --git a/test/unit-tests/plugins/test_helpers.rb b/test/unit-tests/plugins/test_helpers.rb deleted file mode 100644 index 543f00ac9..000000000 --- a/test/unit-tests/plugins/test_helpers.rb +++ /dev/null @@ -1,3 +0,0 @@ -gem "minitest" -require "minitest/spec" -require 'minitest/autorun' \ No newline at end of file