From 32f7594e3c469395e8c5ff61fbd7c643a6c7b4bb Mon Sep 17 00:00:00 2001 From: Kobynet Date: Mon, 16 Mar 2020 14:20:08 +0200 Subject: [PATCH 01/13] Added user/pass fields to package phase, fixed helm repo add command --- .../kotlin/com/deviceinsight/helm/PackageMojo.kt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt index 5004acd..8c3026b 100644 --- a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt @@ -38,6 +38,11 @@ class PackageMojo : AbstractHelmMojo() { @Parameter(property = "helm.skip", defaultValue = "false") private var skip: Boolean = false + @Parameter(property = "chartRepoUsername", required = false) + private var chartRepoUsername: String? = null + + @Parameter(property = "chartRepoPassword", required = false) + private var chartRepoPassword: String? = null @Throws(MojoExecutionException::class) override fun execute() { @@ -70,9 +75,15 @@ class PackageMojo : AbstractHelmMojo() { if (majorHelmVersion() < 3) { executeCmd("$helm init --client-only") } + + var authParams = "" + if (chartRepoUsername != null && chartRepoPassword != null) { + authParams = "--username ${chartRepoUsername} --password ${chartRepoPassword}" + } + executeCmd("$helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com") - executeCmd("$helm repo add chartRepo $chartRepoUrl") - executeCmd("$helm dependency update", directory = targetHelmDir) + executeCmd("$helm repo add chartRepo $chartRepoUrl $authParams") + executeCmd("$helm dependency update ", directory = targetHelmDir) executeCmd("$helm package ${chartName()} --version $chartVersion") ensureChartFileExists() From 3f7401ea80bac57c10e4355bd77edac3afa72631 Mon Sep 17 00:00:00 2001 From: Kobynet Date: Mon, 16 Mar 2020 15:27:26 +0200 Subject: [PATCH 02/13] Added user/pass to package phase. Added user.pass to helm repo add command --- .../kotlin/com/deviceinsight/helm/PackageMojo.kt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt index 5004acd..0c2df81 100644 --- a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt @@ -38,6 +38,11 @@ class PackageMojo : AbstractHelmMojo() { @Parameter(property = "helm.skip", defaultValue = "false") private var skip: Boolean = false + @Parameter(property = "chartRepoUsername", required = false) + private var chartRepoUsername: String? = null + + @Parameter(property = "chartRepoPassword", required = false) + private var chartRepoPassword: String? = null @Throws(MojoExecutionException::class) override fun execute() { @@ -70,8 +75,14 @@ class PackageMojo : AbstractHelmMojo() { if (majorHelmVersion() < 3) { executeCmd("$helm init --client-only") } + + var authParams = "" + if (chartRepoUsername != null && chartRepoPassword != null) { + authParams = "--username ${chartRepoUsername} --password ${chartRepoPassword}" + } + executeCmd("$helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com") - executeCmd("$helm repo add chartRepo $chartRepoUrl") + executeCmd("$helm repo add chartRepo $chartRepoUrl $authParams") executeCmd("$helm dependency update", directory = targetHelmDir) executeCmd("$helm package ${chartName()} --version $chartVersion") From 85192d9f2f1947c26c9822ca99875cff78c9fa01 Mon Sep 17 00:00:00 2001 From: Koby Date: Sun, 17 Jan 2021 16:07:36 +0200 Subject: [PATCH 03/13] Fix parallel binary download (#7) * Added resolve goal for cases where the download is made in parallel, we can use the resolve goal as separate goal just before running the maven build. This change is backward-compatible. --- .../deviceinsight/helm/AbstractHelmMojo.kt | 111 ------------------ .../kotlin/com/deviceinsight/helm/LintMojo.kt | 6 +- .../com/deviceinsight/helm/PackageMojo.kt | 5 +- .../com/deviceinsight/helm/TemplateMojo.kt | 4 +- 4 files changed, 8 insertions(+), 118 deletions(-) diff --git a/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt b/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt index 29f723f..66f3ef9 100644 --- a/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt @@ -16,56 +16,19 @@ package com.deviceinsight.helm -import com.deviceinsight.helm.util.PlatformDetector -import org.apache.maven.artifact.Artifact -import org.apache.maven.artifact.repository.ArtifactRepository -import org.apache.maven.artifact.resolver.ArtifactResolutionRequest -import org.apache.maven.artifact.resolver.ArtifactResolutionResult import org.apache.maven.plugin.AbstractMojo -import org.apache.maven.plugins.annotations.Component import org.apache.maven.plugins.annotations.Parameter import org.apache.maven.project.MavenProject -import org.apache.maven.repository.RepositorySystem import java.io.File -import java.net.HttpURLConnection -import java.net.URI -import java.net.URL -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption -import java.util.zip.ZipEntry -import java.util.zip.ZipInputStream -import kotlin.system.measureTimeMillis abstract class AbstractHelmMojo : AbstractMojo() { - @Parameter(property = "helmGroupId", defaultValue = "com.deviceinsight.helm") - private lateinit var helmGroupId: String - - @Parameter(property = "helmArtifactId", defaultValue = "helm") - private lateinit var helmArtifactId: String - - @Parameter(property = "helmVersion", required = true) - private lateinit var helmVersion: String - - @Parameter(property = "helmDownloadUrl", defaultValue = "https://get.helm.sh/") - private lateinit var helmDownloadUrl: URI - @Parameter(property = "chartVersion", required = false, defaultValue = "\${project.model.version}") protected lateinit var chartVersion: String @Parameter(defaultValue = "\${project}", readonly = true, required = true) protected lateinit var project: MavenProject - @Parameter(readonly = true, required = true, defaultValue = "\${localRepository}") - private lateinit var localRepository: ArtifactRepository - - @Parameter(readonly = true, required = true, defaultValue = "\${project.remoteArtifactRepositories}") - private lateinit var remoteRepositories: List - - @Component - private lateinit var repositorySystem: RepositorySystem - @Parameter(property = "chartFolder", required = false) private var chartFolder: String? = null @@ -75,31 +38,6 @@ abstract class AbstractHelmMojo : AbstractMojo() { @Parameter(property = "chartName", required = false) private var chartName: String? = null - protected fun resolveHelmBinary(): String { - - val platformIdentifier = PlatformDetector.detectHelmReleasePlatformIdentifier() - val helmArtifact: Artifact = - repositorySystem.createArtifactWithClassifier(helmGroupId, helmArtifactId, helmVersion, "binary", - platformIdentifier) - - val request = ArtifactResolutionRequest() - request.artifact = helmArtifact - request.isResolveTransitively = false - request.localRepository = localRepository - request.remoteRepositories = remoteRepositories - - val resolutionResult: ArtifactResolutionResult = repositorySystem.resolve(request) - - if (!resolutionResult.isSuccess) { - log.info("Artifact not found in remote repositories") - downloadAndInstallHelm(helmArtifact, platformIdentifier) - } - - helmArtifact.file.setExecutable(true) - - return helmArtifact.file.absolutePath - } - protected fun executeCmd(cmd: String, directory: File = target(), redirectOutput: ProcessBuilder.Redirect = ProcessBuilder.Redirect.PIPE) { val proc = ProcessBuilder(cmd.split(" ")) @@ -119,7 +57,6 @@ abstract class AbstractHelmMojo : AbstractMojo() { } } - protected fun majorHelmVersion(): Int = helmVersion.splitToSequence('.').first().toInt() protected fun target() = File(project.build.directory).resolve("helm") @@ -131,54 +68,6 @@ abstract class AbstractHelmMojo : AbstractMojo() { protected fun isChartFolderPresent() = File("${project.basedir}/${chartFolder()}").exists() - private fun downloadAndInstallHelm(artifact: Artifact, platformIdentifier: String) { - - val fileName = "helm-v$helmVersion-$platformIdentifier" - - val targetFile = artifact.file.toPath() - Files.createDirectories(targetFile.parent) - - val url = helmDownloadUrl.resolve("./$fileName.zip").toURL() - - downloadFileAndExtractBinary(url, targetFile) - } - - private fun downloadFileAndExtractBinary(url: URL, destination: Path) { - val httpConnection = url.openConnection() - httpConnection.connect() - if (httpConnection !is HttpURLConnection || httpConnection.responseCode != 200) { - throw RuntimeException("Could not download file from $url") - } - - val sizeInMiB: Double = httpConnection.contentLengthLong / 1024.0 / 1024.0 - log.info("Downloading $url; need to get %.1f MiB...".format(sizeInMiB)) - - val downloadTimeMillis = measureTimeMillis { - httpConnection.inputStream.use { - ZipInputStream(it).use { zip -> - var entry: ZipEntry? = zip.nextEntry - do { - if (entry != null) { - if (isHelmBinary(entry)) { - Files.copy(zip, destination, StandardCopyOption.REPLACE_EXISTING) - zip.closeEntry() - break - } else { - zip.closeEntry() - entry = zip.nextEntry - } - } - } while (entry != null) - } - } - } - - log.info("Download took %.1f seconds".format(downloadTimeMillis / 1000.0)) - } - - private fun isHelmBinary(entry: ZipEntry): Boolean = - !entry.isDirectory && (entry.name.endsWith("helm") || entry.name.endsWith("helm.exe")) - protected fun quoteFilePath(filePath: String): String = if (filePath.contains(Regex("\\s"))) { "\"$filePath\"" diff --git a/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt b/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt index e0f8db0..ef49fe4 100644 --- a/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt @@ -23,8 +23,8 @@ import org.apache.maven.plugins.annotations.Parameter import java.io.File -@Mojo(name = "lint", defaultPhase = LifecyclePhase.PRE_INTEGRATION_TEST) -class LintMojo : AbstractHelmMojo() { +@Mojo(name = "lint", defaultPhase = LifecyclePhase.NONE) +class LintMojo : ResolveHelmMojo() { /** * An optional values.yaml file that is used to run linting, relative to `${project.basedir}`. @@ -53,7 +53,7 @@ class LintMojo : AbstractHelmMojo() { return } - val helm = resolveHelmBinary() + super.execute() val command = mutableListOf(helm, "lint", chartName()) diff --git a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt index 9484461..3311675 100644 --- a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt @@ -26,7 +26,7 @@ import java.io.File * Packages helm charts */ @Mojo(name = "package", defaultPhase = LifecyclePhase.PACKAGE) -class PackageMojo : AbstractHelmMojo() { +class PackageMojo : ResolveHelmMojo() { companion object { private val PLACEHOLDER_REGEX = Regex("""\$\{(.*?)}""") @@ -66,7 +66,7 @@ class PackageMojo : AbstractHelmMojo() { return } - val helm = resolveHelmBinary() + super.execute() val targetHelmDir = File(target(), chartName()) @@ -92,6 +92,7 @@ class PackageMojo : AbstractHelmMojo() { "" } executeCmd("$helm repo add chartRepo $chartRepoUrl$authParams") + } executeCmd("$helm dependency update", directory = targetHelmDir) executeCmd("$helm package ${chartName()} --version $chartVersion") diff --git a/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt b/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt index d9bca3b..ea945cf 100644 --- a/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt @@ -23,7 +23,7 @@ import org.apache.maven.plugins.annotations.Parameter import java.io.File @Mojo(name = "template", defaultPhase = LifecyclePhase.PRE_INTEGRATION_TEST) -class TemplateMojo : AbstractHelmMojo() { +class TemplateMojo : ResolveHelmMojo() { /** * An optional values.yaml file that is used to render the template, relative to `${project.basedir}`. @@ -55,7 +55,7 @@ class TemplateMojo : AbstractHelmMojo() { return } - val helm = resolveHelmBinary() + super.execute() val command = if (valuesFile != null) { val valuesFilePath = project.basedir.resolve(valuesFile!!).absolutePath From 89de6cb3d354b5c9f12266110f7270b70477f402 Mon Sep 17 00:00:00 2001 From: kobynet Date: Sun, 17 Jan 2021 16:23:15 +0200 Subject: [PATCH 04/13] revert wrong commit --- src/main/kotlin/com/deviceinsight/helm/LintMojo.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt b/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt index ef49fe4..8bd22e7 100644 --- a/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt @@ -23,7 +23,7 @@ import org.apache.maven.plugins.annotations.Parameter import java.io.File -@Mojo(name = "lint", defaultPhase = LifecyclePhase.NONE) +@Mojo(name = "lint", defaultPhase = LifecyclePhase.PRE_INTEGRATION_TEST) class LintMojo : ResolveHelmMojo() { /** From 2228355a37becfd4097e260183ef8515c0f966a0 Mon Sep 17 00:00:00 2001 From: kobynet Date: Sun, 17 Jan 2021 17:13:14 +0200 Subject: [PATCH 05/13] commit missed file --- .../com/deviceinsight/helm/ResolveHelmMojo.kt | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt diff --git a/src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt b/src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt new file mode 100644 index 0000000..48021bd --- /dev/null +++ b/src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2018-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.deviceinsight.helm + +import com.deviceinsight.helm.util.PlatformDetector +import org.apache.maven.artifact.Artifact +import org.apache.maven.artifact.repository.ArtifactRepository +import org.apache.maven.artifact.resolver.ArtifactResolutionRequest +import org.apache.maven.artifact.resolver.ArtifactResolutionResult +import org.apache.maven.plugins.annotations.Component +import org.apache.maven.plugins.annotations.LifecyclePhase +import org.apache.maven.plugins.annotations.Mojo +import org.apache.maven.plugins.annotations.Parameter +import org.apache.maven.repository.RepositorySystem +import java.net.HttpURLConnection +import java.net.URI +import java.net.URL +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream +import kotlin.system.measureTimeMillis + +@Mojo(name = "resolve", defaultPhase = LifecyclePhase.NONE) +open class ResolveHelmMojo : AbstractHelmMojo() { + + @Parameter(property = "helmGroupId", defaultValue = "com.deviceinsight.helm") + private lateinit var helmGroupId: String + + @Parameter(property = "helmArtifactId", defaultValue = "helm") + private lateinit var helmArtifactId: String + + @Parameter(property = "helmVersion", required = true) + private lateinit var helmVersion: String + + @Parameter(property = "helmDownloadUrl", defaultValue = "https://get.helm.sh/") + private lateinit var helmDownloadUrl: URI + + @Parameter(readonly = true, required = true, defaultValue = "\${localRepository}") + private lateinit var localRepository: ArtifactRepository + + @Parameter(readonly = true, required = true, defaultValue = "\${project.remoteArtifactRepositories}") + private lateinit var remoteRepositories: List + + @Component + private lateinit var repositorySystem: RepositorySystem + + protected lateinit var helm : String + + protected fun resolveHelmBinary(): String { + + val platformIdentifier = PlatformDetector.detectHelmReleasePlatformIdentifier() + val helmArtifact: Artifact = + repositorySystem.createArtifactWithClassifier(helmGroupId, helmArtifactId, helmVersion, "binary", + platformIdentifier) + + val request = ArtifactResolutionRequest() + request.artifact = helmArtifact + request.isResolveTransitively = false + request.localRepository = localRepository + request.remoteRepositories = remoteRepositories + + val resolutionResult: ArtifactResolutionResult = repositorySystem.resolve(request) + + if (!resolutionResult.isSuccess) { + log.info("Artifact not found in remote repositories") + downloadAndInstallHelm(helmArtifact, platformIdentifier) + } + + helmArtifact.file.setExecutable(true) + + return helmArtifact.file.absolutePath + } + + private fun downloadAndInstallHelm(artifact: Artifact, platformIdentifier: String) { + + val fileName = "helm-v$helmVersion-$platformIdentifier" + + val targetFile = artifact.file.toPath() + Files.createDirectories(targetFile.parent) + + val url = helmDownloadUrl.resolve("./$fileName.zip").toURL() + + downloadFileAndExtractBinary(url, targetFile) + } + + private fun downloadFileAndExtractBinary(url: URL, destination: Path) { + val httpConnection = url.openConnection() + httpConnection.connect() + if (httpConnection !is HttpURLConnection || httpConnection.responseCode != 200) { + throw RuntimeException("Could not download file from $url") + } + + val sizeInMiB: Double = httpConnection.contentLengthLong / 1024.0 / 1024.0 + log.info("Downloading $url; need to get %.1f MiB...".format(sizeInMiB)) + + val downloadTimeMillis = measureTimeMillis { + httpConnection.inputStream.use { + ZipInputStream(it).use { zip -> + var entry: ZipEntry? = zip.nextEntry + do { + if (entry != null) { + if (isHelmBinary(entry)) { + Files.copy(zip, destination, StandardCopyOption.REPLACE_EXISTING) + zip.closeEntry() + break + } else { + zip.closeEntry() + entry = zip.nextEntry + } + } + } while (entry != null) + } + } + } + + log.info("Download took %.1f seconds".format(downloadTimeMillis / 1000.0)) + } + + protected fun majorHelmVersion(): Int = helmVersion.splitToSequence('.').first().toInt() + + + private fun isHelmBinary(entry: ZipEntry): Boolean = + !entry.isDirectory && (entry.name.endsWith("helm") || entry.name.endsWith("helm.exe")) + + override fun execute() { + helm = resolveHelmBinary() + } +} From 2c7b9c9dea0d8b868d1b4a2e1e4f1c8e2cce422a Mon Sep 17 00:00:00 2001 From: Paul Vorbach Date: Mon, 18 Jan 2021 16:56:38 +0100 Subject: [PATCH 06/13] Update for next development version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0d375a8..01d37de 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.deviceinsight.helm helm-maven-plugin - 2.9.0 + 2.10.0-SNAPSHOT maven-plugin Helm Maven Plugin From 260da8c4aac1c978c44474e2f194ab4cb4f81e3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:47:56 +0000 Subject: [PATCH 07/13] Bump assertj-core from 3.18.1 to 3.19.0 Bumps [assertj-core](https://github.com/assertj/assertj-core) from 3.18.1 to 3.19.0. - [Release notes](https://github.com/assertj/assertj-core/releases) - [Commits](https://github.com/assertj/assertj-core/compare/assertj-core-3.18.1...assertj-core-3.19.0) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 01d37de..a619185 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.5.13 5.7.0 - 3.18.1 + 3.19.0 1.4.20 3.8.1 From c2783b6f47b4d5e38283b69dac410cb87017ca21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Feb 2021 05:37:08 +0000 Subject: [PATCH 08/13] Bump junit-jupiter from 5.7.0 to 5.7.1 Bumps [junit-jupiter](https://github.com/junit-team/junit5) from 5.7.0 to 5.7.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.0...r5.7.1) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a619185..6af377b 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 2.12.1 4.5.13 - 5.7.0 + 5.7.1 3.19.0 1.4.20 From fa7ffa68a387a767e85371740838a93385cb98b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Feb 2021 05:37:59 +0000 Subject: [PATCH 09/13] Bump kotlin.version from 1.4.21 to 1.4.30 Bumps `kotlin.version` from 1.4.21 to 1.4.30. Updates `kotlin-stdlib-jdk8` from 1.4.21 to 1.4.30 - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.4.21...v1.4.30) Updates `kotlin-test-junit` from 1.4.21 to 1.4.30 - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.4.21...v1.4.30) Updates `kotlin-maven-plugin` from 1.4.21 to 1.4.30 Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a619185..86e53e9 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ https://github.com/deviceinsight/helm-maven-plugin - 1.4.21 + 1.4.30 1.8 1.8 1.8 From d25c4cf8eeca80e43c54cfd16adcc25d126b603f Mon Sep 17 00:00:00 2001 From: kobynet Date: Thu, 11 Feb 2021 09:24:06 +0100 Subject: [PATCH 10/13] Fix parallel binary download (#7) * Added resolve goal for cases where the download is made in parallel, we can use the resolve goal as separate goal just before running the maven build. This change is backward-compatible. Fixes #7 --- .../deviceinsight/helm/AbstractHelmMojo.kt | 111 -------------- .../kotlin/com/deviceinsight/helm/LintMojo.kt | 4 +- .../com/deviceinsight/helm/PackageMojo.kt | 5 +- .../com/deviceinsight/helm/ResolveHelmMojo.kt | 144 ++++++++++++++++++ .../com/deviceinsight/helm/TemplateMojo.kt | 4 +- 5 files changed, 150 insertions(+), 118 deletions(-) create mode 100644 src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt diff --git a/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt b/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt index 29f723f..66f3ef9 100644 --- a/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/AbstractHelmMojo.kt @@ -16,56 +16,19 @@ package com.deviceinsight.helm -import com.deviceinsight.helm.util.PlatformDetector -import org.apache.maven.artifact.Artifact -import org.apache.maven.artifact.repository.ArtifactRepository -import org.apache.maven.artifact.resolver.ArtifactResolutionRequest -import org.apache.maven.artifact.resolver.ArtifactResolutionResult import org.apache.maven.plugin.AbstractMojo -import org.apache.maven.plugins.annotations.Component import org.apache.maven.plugins.annotations.Parameter import org.apache.maven.project.MavenProject -import org.apache.maven.repository.RepositorySystem import java.io.File -import java.net.HttpURLConnection -import java.net.URI -import java.net.URL -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption -import java.util.zip.ZipEntry -import java.util.zip.ZipInputStream -import kotlin.system.measureTimeMillis abstract class AbstractHelmMojo : AbstractMojo() { - @Parameter(property = "helmGroupId", defaultValue = "com.deviceinsight.helm") - private lateinit var helmGroupId: String - - @Parameter(property = "helmArtifactId", defaultValue = "helm") - private lateinit var helmArtifactId: String - - @Parameter(property = "helmVersion", required = true) - private lateinit var helmVersion: String - - @Parameter(property = "helmDownloadUrl", defaultValue = "https://get.helm.sh/") - private lateinit var helmDownloadUrl: URI - @Parameter(property = "chartVersion", required = false, defaultValue = "\${project.model.version}") protected lateinit var chartVersion: String @Parameter(defaultValue = "\${project}", readonly = true, required = true) protected lateinit var project: MavenProject - @Parameter(readonly = true, required = true, defaultValue = "\${localRepository}") - private lateinit var localRepository: ArtifactRepository - - @Parameter(readonly = true, required = true, defaultValue = "\${project.remoteArtifactRepositories}") - private lateinit var remoteRepositories: List - - @Component - private lateinit var repositorySystem: RepositorySystem - @Parameter(property = "chartFolder", required = false) private var chartFolder: String? = null @@ -75,31 +38,6 @@ abstract class AbstractHelmMojo : AbstractMojo() { @Parameter(property = "chartName", required = false) private var chartName: String? = null - protected fun resolveHelmBinary(): String { - - val platformIdentifier = PlatformDetector.detectHelmReleasePlatformIdentifier() - val helmArtifact: Artifact = - repositorySystem.createArtifactWithClassifier(helmGroupId, helmArtifactId, helmVersion, "binary", - platformIdentifier) - - val request = ArtifactResolutionRequest() - request.artifact = helmArtifact - request.isResolveTransitively = false - request.localRepository = localRepository - request.remoteRepositories = remoteRepositories - - val resolutionResult: ArtifactResolutionResult = repositorySystem.resolve(request) - - if (!resolutionResult.isSuccess) { - log.info("Artifact not found in remote repositories") - downloadAndInstallHelm(helmArtifact, platformIdentifier) - } - - helmArtifact.file.setExecutable(true) - - return helmArtifact.file.absolutePath - } - protected fun executeCmd(cmd: String, directory: File = target(), redirectOutput: ProcessBuilder.Redirect = ProcessBuilder.Redirect.PIPE) { val proc = ProcessBuilder(cmd.split(" ")) @@ -119,7 +57,6 @@ abstract class AbstractHelmMojo : AbstractMojo() { } } - protected fun majorHelmVersion(): Int = helmVersion.splitToSequence('.').first().toInt() protected fun target() = File(project.build.directory).resolve("helm") @@ -131,54 +68,6 @@ abstract class AbstractHelmMojo : AbstractMojo() { protected fun isChartFolderPresent() = File("${project.basedir}/${chartFolder()}").exists() - private fun downloadAndInstallHelm(artifact: Artifact, platformIdentifier: String) { - - val fileName = "helm-v$helmVersion-$platformIdentifier" - - val targetFile = artifact.file.toPath() - Files.createDirectories(targetFile.parent) - - val url = helmDownloadUrl.resolve("./$fileName.zip").toURL() - - downloadFileAndExtractBinary(url, targetFile) - } - - private fun downloadFileAndExtractBinary(url: URL, destination: Path) { - val httpConnection = url.openConnection() - httpConnection.connect() - if (httpConnection !is HttpURLConnection || httpConnection.responseCode != 200) { - throw RuntimeException("Could not download file from $url") - } - - val sizeInMiB: Double = httpConnection.contentLengthLong / 1024.0 / 1024.0 - log.info("Downloading $url; need to get %.1f MiB...".format(sizeInMiB)) - - val downloadTimeMillis = measureTimeMillis { - httpConnection.inputStream.use { - ZipInputStream(it).use { zip -> - var entry: ZipEntry? = zip.nextEntry - do { - if (entry != null) { - if (isHelmBinary(entry)) { - Files.copy(zip, destination, StandardCopyOption.REPLACE_EXISTING) - zip.closeEntry() - break - } else { - zip.closeEntry() - entry = zip.nextEntry - } - } - } while (entry != null) - } - } - } - - log.info("Download took %.1f seconds".format(downloadTimeMillis / 1000.0)) - } - - private fun isHelmBinary(entry: ZipEntry): Boolean = - !entry.isDirectory && (entry.name.endsWith("helm") || entry.name.endsWith("helm.exe")) - protected fun quoteFilePath(filePath: String): String = if (filePath.contains(Regex("\\s"))) { "\"$filePath\"" diff --git a/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt b/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt index e0f8db0..8bd22e7 100644 --- a/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/LintMojo.kt @@ -24,7 +24,7 @@ import java.io.File @Mojo(name = "lint", defaultPhase = LifecyclePhase.PRE_INTEGRATION_TEST) -class LintMojo : AbstractHelmMojo() { +class LintMojo : ResolveHelmMojo() { /** * An optional values.yaml file that is used to run linting, relative to `${project.basedir}`. @@ -53,7 +53,7 @@ class LintMojo : AbstractHelmMojo() { return } - val helm = resolveHelmBinary() + super.execute() val command = mutableListOf(helm, "lint", chartName()) diff --git a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt index 1ed3621..2e98c23 100644 --- a/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/PackageMojo.kt @@ -26,7 +26,7 @@ import java.io.File * Packages helm charts */ @Mojo(name = "package", defaultPhase = LifecyclePhase.PACKAGE) -class PackageMojo : AbstractHelmMojo() { +class PackageMojo : ResolveHelmMojo() { companion object { private val PLACEHOLDER_REGEX = Regex("""\$\{(.*?)}""") @@ -69,7 +69,7 @@ class PackageMojo : AbstractHelmMojo() { return } - val helm = resolveHelmBinary() + super.execute() val targetHelmDir = File(target(), chartName()) @@ -85,7 +85,6 @@ class PackageMojo : AbstractHelmMojo() { if (majorHelmVersion() < 3) { executeCmd("$helm init --client-only --stable-repo-url $stableRepoUrl") } - if (addIncubatorRepo) { executeCmd("$helm repo add incubator $incubatorRepoUrl") } diff --git a/src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt b/src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt new file mode 100644 index 0000000..48021bd --- /dev/null +++ b/src/main/kotlin/com/deviceinsight/helm/ResolveHelmMojo.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2018-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.deviceinsight.helm + +import com.deviceinsight.helm.util.PlatformDetector +import org.apache.maven.artifact.Artifact +import org.apache.maven.artifact.repository.ArtifactRepository +import org.apache.maven.artifact.resolver.ArtifactResolutionRequest +import org.apache.maven.artifact.resolver.ArtifactResolutionResult +import org.apache.maven.plugins.annotations.Component +import org.apache.maven.plugins.annotations.LifecyclePhase +import org.apache.maven.plugins.annotations.Mojo +import org.apache.maven.plugins.annotations.Parameter +import org.apache.maven.repository.RepositorySystem +import java.net.HttpURLConnection +import java.net.URI +import java.net.URL +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream +import kotlin.system.measureTimeMillis + +@Mojo(name = "resolve", defaultPhase = LifecyclePhase.NONE) +open class ResolveHelmMojo : AbstractHelmMojo() { + + @Parameter(property = "helmGroupId", defaultValue = "com.deviceinsight.helm") + private lateinit var helmGroupId: String + + @Parameter(property = "helmArtifactId", defaultValue = "helm") + private lateinit var helmArtifactId: String + + @Parameter(property = "helmVersion", required = true) + private lateinit var helmVersion: String + + @Parameter(property = "helmDownloadUrl", defaultValue = "https://get.helm.sh/") + private lateinit var helmDownloadUrl: URI + + @Parameter(readonly = true, required = true, defaultValue = "\${localRepository}") + private lateinit var localRepository: ArtifactRepository + + @Parameter(readonly = true, required = true, defaultValue = "\${project.remoteArtifactRepositories}") + private lateinit var remoteRepositories: List + + @Component + private lateinit var repositorySystem: RepositorySystem + + protected lateinit var helm : String + + protected fun resolveHelmBinary(): String { + + val platformIdentifier = PlatformDetector.detectHelmReleasePlatformIdentifier() + val helmArtifact: Artifact = + repositorySystem.createArtifactWithClassifier(helmGroupId, helmArtifactId, helmVersion, "binary", + platformIdentifier) + + val request = ArtifactResolutionRequest() + request.artifact = helmArtifact + request.isResolveTransitively = false + request.localRepository = localRepository + request.remoteRepositories = remoteRepositories + + val resolutionResult: ArtifactResolutionResult = repositorySystem.resolve(request) + + if (!resolutionResult.isSuccess) { + log.info("Artifact not found in remote repositories") + downloadAndInstallHelm(helmArtifact, platformIdentifier) + } + + helmArtifact.file.setExecutable(true) + + return helmArtifact.file.absolutePath + } + + private fun downloadAndInstallHelm(artifact: Artifact, platformIdentifier: String) { + + val fileName = "helm-v$helmVersion-$platformIdentifier" + + val targetFile = artifact.file.toPath() + Files.createDirectories(targetFile.parent) + + val url = helmDownloadUrl.resolve("./$fileName.zip").toURL() + + downloadFileAndExtractBinary(url, targetFile) + } + + private fun downloadFileAndExtractBinary(url: URL, destination: Path) { + val httpConnection = url.openConnection() + httpConnection.connect() + if (httpConnection !is HttpURLConnection || httpConnection.responseCode != 200) { + throw RuntimeException("Could not download file from $url") + } + + val sizeInMiB: Double = httpConnection.contentLengthLong / 1024.0 / 1024.0 + log.info("Downloading $url; need to get %.1f MiB...".format(sizeInMiB)) + + val downloadTimeMillis = measureTimeMillis { + httpConnection.inputStream.use { + ZipInputStream(it).use { zip -> + var entry: ZipEntry? = zip.nextEntry + do { + if (entry != null) { + if (isHelmBinary(entry)) { + Files.copy(zip, destination, StandardCopyOption.REPLACE_EXISTING) + zip.closeEntry() + break + } else { + zip.closeEntry() + entry = zip.nextEntry + } + } + } while (entry != null) + } + } + } + + log.info("Download took %.1f seconds".format(downloadTimeMillis / 1000.0)) + } + + protected fun majorHelmVersion(): Int = helmVersion.splitToSequence('.').first().toInt() + + + private fun isHelmBinary(entry: ZipEntry): Boolean = + !entry.isDirectory && (entry.name.endsWith("helm") || entry.name.endsWith("helm.exe")) + + override fun execute() { + helm = resolveHelmBinary() + } +} diff --git a/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt b/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt index d9bca3b..ea945cf 100644 --- a/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt +++ b/src/main/kotlin/com/deviceinsight/helm/TemplateMojo.kt @@ -23,7 +23,7 @@ import org.apache.maven.plugins.annotations.Parameter import java.io.File @Mojo(name = "template", defaultPhase = LifecyclePhase.PRE_INTEGRATION_TEST) -class TemplateMojo : AbstractHelmMojo() { +class TemplateMojo : ResolveHelmMojo() { /** * An optional values.yaml file that is used to render the template, relative to `${project.basedir}`. @@ -55,7 +55,7 @@ class TemplateMojo : AbstractHelmMojo() { return } - val helm = resolveHelmBinary() + super.execute() val command = if (valuesFile != null) { val valuesFilePath = project.basedir.resolve(valuesFile!!).absolutePath From b4ebeb138a8897c45ba0ef53f58f8ea9d9a4bb74 Mon Sep 17 00:00:00 2001 From: Paul Vorbach Date: Thu, 11 Feb 2021 09:26:25 +0100 Subject: [PATCH 11/13] Amend changelog entry for new resolve goal --- CHANGELOG.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index d2ccbfe..5d36648 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,10 @@ == Unreleased +* Add a new goal, `resolve`, for downloading the helm binary as a separate step. This will still be done automatically + by the other goals, but it can be used when running maven builds in parallel, so the helm binary doesn't need to be + downloaded twice and to prevent concurrency issues. + == Version 2.9.0 * Support configuring the default URL of the helm repository for helm 2.x via `stableRepoUrl` From 9d3218c6449e5ef3bfc38afdf8f42984fc4335df Mon Sep 17 00:00:00 2001 From: Paul Vorbach Date: Thu, 11 Feb 2021 09:32:30 +0100 Subject: [PATCH 12/13] Update versions for release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 69acbeb..b1d7cdb 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.deviceinsight.helm helm-maven-plugin - 2.10.0-SNAPSHOT + 2.10.0 maven-plugin Helm Maven Plugin From f27225ac7d4f31b6fd280d3302cec5db9d0a554b Mon Sep 17 00:00:00 2001 From: Paul Vorbach Date: Thu, 11 Feb 2021 09:33:57 +0100 Subject: [PATCH 13/13] Update version in readme and changelog --- CHANGELOG.adoc | 2 ++ README.adoc | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 5d36648..b85e93f 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,8 @@ == Unreleased +== Version 2.10.0 + * Add a new goal, `resolve`, for downloading the helm binary as a separate step. This will still be done automatically by the other goals, but it can be used when running maven builds in parallel, so the helm binary doesn't need to be downloaded twice and to prevent concurrency issues. diff --git a/README.adoc b/README.adoc index 543fb2b..6e1aacf 100644 --- a/README.adoc +++ b/README.adoc @@ -20,11 +20,11 @@ Add the following to your `pom.xml` com.deviceinsight.helm helm-maven-plugin - 2.9.0 + 2.10.0 my-chart https://charts.helm.sh/stable - 3.4.2 + 3.5.2 true src/test/helm/my-chart/values.yaml @@ -143,11 +143,11 @@ To use the `deployAtEnd` functionality it's mandatory to put the Helm Maven Plug com.deviceinsight.helm helm-maven-plugin - 2.9.0 + 2.10.0 my-chart https://charts.helm.sh/stable - 3.4.2 + 3.5.2 true src/test/helm/my-chart/values.yaml true @@ -177,7 +177,7 @@ Problem:: The following error message is a common source of trouble, lately: ... -[ERROR] Failed to execute goal com.deviceinsight.helm:helm-maven-plugin:2.9.0:package (default) on project my-project: Error creating helm chart: When executing '/home/user/.m2/repository/com/deviceinsight/helm/helm/2.16.2/helm-2.16.2-linux-amd64.binary init --client-only' got result code '1' -> [Help 1] +[ERROR] Failed to execute goal com.deviceinsight.helm:helm-maven-plugin:2.10.0:package (default) on project my-project: Error creating helm chart: When executing '/home/user/.m2/repository/com/deviceinsight/helm/helm/2.16.2/helm-2.16.2-linux-amd64.binary init --client-only' got result code '1' -> [Help 1] ---- Solution:: This is likely due to an old version of helm itself. Make sure to configure `` to a version >= 3.4.0 or, if you are still using Helm 2, a version >= 2.17.0 (https://github.com/helm/charts#%EF%B8%8F-deprecation-and-archive-notice[background information]). . {blank}