diff --git a/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs b/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs index dc63ad3c89..22ab36b48f 100644 --- a/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs +++ b/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs @@ -41,6 +41,7 @@ public static IServiceCollection AddBuildScriptGeneratorServices(this IServiceCo services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHttpClient("general", httpClient => { // NOTE: Setting user agent is required to avoid receiving 403 Forbidden response. diff --git a/src/BuildScriptGenerator/Contracts/IMcrSdkProvider.cs b/src/BuildScriptGenerator/Contracts/IMcrSdkProvider.cs new file mode 100644 index 0000000000..e6fcb22f6e --- /dev/null +++ b/src/BuildScriptGenerator/Contracts/IMcrSdkProvider.cs @@ -0,0 +1,25 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + /// + /// Interface for pulling SDKs from MCR/ACR container images. + /// + public interface IMcrSdkProvider + { + /// + /// Pulls the SDK tarball from an MCR/ACR container image and places it + /// in the external SDKs storage directory so existing installation logic can use it. + /// + /// The platform name (e.g. "nodejs", "python", "dotnet", "php"). + /// The platform version (e.g. "18.20.8"). + /// The debian flavor (e.g. "bookworm"). + /// True if the SDK was pulled and cached successfully. + Task PullSdkAsync(string platformName, string version, string debianFlavor); + } +} diff --git a/src/BuildScriptGenerator/DefaultPlatformsInformationProvider.cs b/src/BuildScriptGenerator/DefaultPlatformsInformationProvider.cs index 8c11d5634e..533e227e41 100644 --- a/src/BuildScriptGenerator/DefaultPlatformsInformationProvider.cs +++ b/src/BuildScriptGenerator/DefaultPlatformsInformationProvider.cs @@ -52,6 +52,11 @@ public IEnumerable GetPlatformsInfo(RepositoryContext context) this.outputWriter.WriteLine("External SDK provider is enabled."); } + if (this.commonOptions.EnableMcrSdkProvider) + { + this.outputWriter.WriteLine("MCR SDK provider is enabled."); + } + foreach (var platform in this.platforms) { // Check if a platform is enabled or not diff --git a/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs b/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs index afeb21c9f3..3a0b83988b 100644 --- a/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs +++ b/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs @@ -35,6 +35,7 @@ internal class DotNetCorePlatform : IProgrammingPlatform private readonly DotNetCorePlatformInstaller platformInstaller; private readonly GlobalJsonSdkResolver globalJsonSdkResolver; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IMcrSdkProvider mcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -47,6 +48,7 @@ internal class DotNetCorePlatform : IProgrammingPlatform /// The options if .NET platform. /// The . /// The . + /// The . public DotNetCorePlatform( IDotNetCoreVersionProvider versionProvider, ILogger logger, @@ -56,6 +58,7 @@ public DotNetCorePlatform( DotNetCorePlatformInstaller platformInstaller, GlobalJsonSdkResolver globalJsonSdkResolver, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) { this.versionProvider = versionProvider; @@ -66,6 +69,7 @@ public DotNetCorePlatform( this.platformInstaller = platformInstaller; this.globalJsonSdkResolver = globalJsonSdkResolver; this.externalSdkProvider = externalSdkProvider; + this.mcrSdkProvider = mcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -244,30 +248,18 @@ public string GetInstallerScriptSnippet( } else { - if (this.commonOptions.EnableExternalSdkProvider) + // Try external SDK provider first (blob storage via socket) + bool sdkFetched = this.TryFetchSdkFromExternalProvider(this.Name, dotNetCorePlatformDetectorResult.SdkVersion); + + // Try MCR SDK provider (container image pull) + if (!sdkFetched) + { + sdkFetched = this.TryPullSdkFromMcr(this.Name, dotNetCorePlatformDetectorResult.SdkVersion); + } + + if (sdkFetched) { - this.logger.LogDebug("DotNetCore SDK version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", dotNetCorePlatformDetectorResult.SdkVersion); - - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, dotNetCorePlatformDetectorResult.SdkVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("DotNetCore SDK version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("DotNetCore SDK version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching DotNetCore SDK version version {version} using external SDK provider.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion); - } + installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion, skipSdkBinaryDownload: true); } else { @@ -477,5 +469,90 @@ private bool TryGetExplicitVersion(out string explicitVersion) return false; } + + /// + /// Tries to fetch the SDK from the external SDK provider (blob storage via Unix socket) if enabled. + /// + /// True if the SDK was successfully fetched from the external provider. + private bool TryFetchSdkFromExternalProvider(string platformName, string version) + { + if (!this.commonOptions.EnableExternalSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "External SDK provider is enabled so trying to fetch SDK using it.", + platformName, + version); + + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion(platformName, version, this.commonOptions.DebianFlavor); + var success = this.externalSdkProvider.RequestBlobAsync(platformName, blobName).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using external SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using external SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using external SDK provider.", platformName, version); + } + + return false; + } + + /// + /// Tries to pull the SDK from MCR container image if enabled. + /// + /// True if the SDK was successfully fetched from MCR. + private bool TryPullSdkFromMcr(string platformName, string version) + { + if (!this.commonOptions.EnableMcrSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "MCR SDK provider is enabled so trying to fetch SDK from container image.", + platformName, + version); + + try + { + var success = this.mcrSdkProvider.PullSdkAsync(platformName, version, this.commonOptions.DebianFlavor).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using MCR SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using MCR SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using MCR SDK provider.", platformName, version); + } + + return false; + } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/McrSdkProvider.cs b/src/BuildScriptGenerator/McrSdkProvider.cs new file mode 100644 index 0000000000..7af93ed75c --- /dev/null +++ b/src/BuildScriptGenerator/McrSdkProvider.cs @@ -0,0 +1,235 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + /// + /// Pulls SDKs from MCR/ACR container images and caches the tarball locally + /// so existing platform installation logic can use it. + /// + /// + /// Image convention: {baseUrl}/{platformName}:{version}-{debianFlavor} + /// The image contains a single tarball at /sdk/{platformName}-{debianFlavor}-{version}.tar.gz + /// which is copied to the ExternalSdksStorageDir used by the existing installer scripts. + /// + public class McrSdkProvider : IMcrSdkProvider + { + private const string DefaultMcrSdkImageBaseUrl = "mcr.microsoft.com/oryx/sdks"; + private const int ProcessTimeoutSeconds = 300; + private readonly ILogger logger; + private readonly IStandardOutputWriter outputWriter; + private readonly BuildScriptGeneratorOptions commonOptions; + + public McrSdkProvider( + IStandardOutputWriter outputWriter, + ILogger logger, + IOptions commonOptions) + { + this.logger = logger; + this.outputWriter = outputWriter; + this.commonOptions = commonOptions.Value; + } + + /// + public async Task PullSdkAsync(string platformName, string version, string debianFlavor) + { + var baseUrl = string.IsNullOrEmpty(this.commonOptions.McrSdkImageBaseUrl) + ? DefaultMcrSdkImageBaseUrl + : this.commonOptions.McrSdkImageBaseUrl; + + var imageTag = $"{baseUrl}/{platformName}:{version}-{debianFlavor}"; + var blobName = BlobNameHelper.GetBlobNameForVersion(platformName, version, debianFlavor); + var containerSdkPath = $"/sdk/{blobName}"; + var localDir = Path.Combine(ExternalSdkProvider.ExternalSdksStorageDir, platformName); + var localPath = Path.Combine(localDir, blobName); + + this.logger.LogInformation( + "MCR SDK provider: pulling image {imageTag} for platform {platformName} version {version}", + imageTag, + platformName, + version); + this.outputWriter.WriteLine($"MCR SDK provider: pulling image {imageTag}"); + + string containerId = null; + try + { + // 1. docker pull + var pullSuccess = await this.RunDockerCommandAsync($"pull {imageTag}"); + if (!pullSuccess) + { + this.logger.LogError("MCR SDK provider: failed to pull image {imageTag}", imageTag); + this.outputWriter.WriteLine($"MCR SDK provider: failed to pull image {imageTag}"); + return false; + } + + // 2. docker create (returns container ID) + containerId = await this.RunDockerCommandForOutputAsync($"create {imageTag} /bin/true"); + if (string.IsNullOrEmpty(containerId)) + { + this.logger.LogError("MCR SDK provider: failed to create container from {imageTag}", imageTag); + this.outputWriter.WriteLine($"MCR SDK provider: failed to create container from {imageTag}"); + return false; + } + + containerId = containerId.Trim(); + + // 3. Ensure target directory exists + Directory.CreateDirectory(localDir); + + // 4. docker cp + var cpSuccess = await this.RunDockerCommandAsync($"cp {containerId}:{containerSdkPath} {localPath}"); + if (!cpSuccess) + { + this.logger.LogError( + "MCR SDK provider: failed to copy SDK from container {containerId}:{containerSdkPath} to {localPath}", + containerId, + containerSdkPath, + localPath); + this.outputWriter.WriteLine($"MCR SDK provider: failed to copy SDK from container"); + return false; + } + + if (!File.Exists(localPath)) + { + this.logger.LogError("MCR SDK provider: expected file {localPath} not found after docker cp", localPath); + return false; + } + + this.logger.LogInformation( + "MCR SDK provider: successfully cached SDK at {localPath}", + localPath); + this.outputWriter.WriteLine($"MCR SDK provider: SDK cached at {localPath}"); + return true; + } + catch (Exception ex) + { + this.logger.LogError(ex, "MCR SDK provider: error pulling SDK for {platformName} {version}", platformName, version); + this.outputWriter.WriteLine($"MCR SDK provider: error - {ex.Message}"); + return false; + } + finally + { + // 5. cleanup: docker rm + if (!string.IsNullOrEmpty(containerId)) + { + try + { + await this.RunDockerCommandAsync($"rm -f {containerId}"); + } + catch (Exception ex) + { + this.logger.LogWarning(ex, "MCR SDK provider: failed to remove container {containerId}", containerId); + } + } + } + } + + private async Task RunDockerCommandAsync(string arguments) + { + var psi = new ProcessStartInfo + { + FileName = "docker", + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + using var process = Process.Start(psi); + if (process == null) + { + this.logger.LogError("MCR SDK provider: failed to start docker process with args: {args}", arguments); + return false; + } + + var completed = await Task.Run(() => process.WaitForExit(ProcessTimeoutSeconds * 1000)); + if (!completed) + { + this.logger.LogError("MCR SDK provider: docker command timed out: docker {args}", arguments); + try + { + process.Kill(); + } + catch + { + // Best effort kill + } + + return false; + } + + if (process.ExitCode != 0) + { + var stderr = await process.StandardError.ReadToEndAsync(); + this.logger.LogError( + "MCR SDK provider: docker {args} failed with exit code {exitCode}. stderr: {stderr}", + arguments, + process.ExitCode, + stderr); + return false; + } + + return true; + } + + private async Task RunDockerCommandForOutputAsync(string arguments) + { + var psi = new ProcessStartInfo + { + FileName = "docker", + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + using var process = Process.Start(psi); + if (process == null) + { + this.logger.LogError("MCR SDK provider: failed to start docker process with args: {args}", arguments); + return null; + } + + var output = await process.StandardOutput.ReadToEndAsync(); + var completed = await Task.Run(() => process.WaitForExit(ProcessTimeoutSeconds * 1000)); + if (!completed) + { + this.logger.LogError("MCR SDK provider: docker command timed out: docker {args}", arguments); + try + { + process.Kill(); + } + catch + { + // Best effort kill + } + + return null; + } + + if (process.ExitCode != 0) + { + var stderr = await process.StandardError.ReadToEndAsync(); + this.logger.LogError( + "MCR SDK provider: docker {args} failed with exit code {exitCode}. stderr: {stderr}", + arguments, + process.ExitCode, + stderr); + return null; + } + + return output; + } + } +} diff --git a/src/BuildScriptGenerator/Node/NodePlatform.cs b/src/BuildScriptGenerator/Node/NodePlatform.cs index 87cbed0817..6fd52705ce 100644 --- a/src/BuildScriptGenerator/Node/NodePlatform.cs +++ b/src/BuildScriptGenerator/Node/NodePlatform.cs @@ -86,6 +86,7 @@ internal class NodePlatform : IProgrammingPlatform private readonly IEnvironment environment; private readonly NodePlatformInstaller platformInstaller; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IMcrSdkProvider mcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -99,6 +100,7 @@ internal class NodePlatform : IProgrammingPlatform /// The environment of Node.js platform. /// The . /// The . + /// The . public NodePlatform( IOptions commonOptions, IOptions nodeScriptGeneratorOptions, @@ -108,6 +110,7 @@ public NodePlatform( IEnvironment environment, NodePlatformInstaller nodePlatformInstaller, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) { this.commonOptions = commonOptions.Value; @@ -118,6 +121,7 @@ public NodePlatform( this.environment = environment; this.platformInstaller = nodePlatformInstaller; this.externalSdkProvider = externalSdkProvider; + this.mcrSdkProvider = mcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -506,42 +510,18 @@ public string GetInstallerScriptSnippet( } else { - if (this.commonOptions.EnableExternalSdkProvider) + // Try external SDK provider first (blob storage via socket) + bool sdkFetched = this.TryFetchSdkFromExternalProvider(this.Name, detectorResult.PlatformVersion); + + // Try MCR SDK provider (container image pull) + if (!sdkFetched) { - this.logger.LogDebug( - "Node version {version} is not installed. " + - "External SDK provider is enabled so trying to fetch SDK using it.", - detectorResult.PlatformVersion); + sdkFetched = this.TryPullSdkFromMcr(this.Name, detectorResult.PlatformVersion); + } - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, detectorResult.PlatformVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug( - "Node version {version} is fetched successfully using external SDK provider. " + - "So generating an installation script snippet which skips platform binary download.", - detectorResult.PlatformVersion); - - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug( - "Node version {version} is not fetched successfully using external SDK provider. " + - "So generating an installation script snippet for it.", - detectorResult.PlatformVersion); - - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet( - detectorResult.PlatformVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching Node.js version {version} using external SDK provider.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } + if (sdkFetched) + { + installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion, skipSdkBinaryDownload: true); } else { @@ -763,5 +743,90 @@ private string GetVersionUsingHierarchicalRules(string detectedVersion) var versionInfo = this.nodeVersionProvider.GetVersionInfo(); return versionInfo.DefaultVersion; } + + /// + /// Tries to fetch the SDK from the external SDK provider (blob storage via Unix socket) if enabled. + /// + /// True if the SDK was successfully fetched from the external provider. + private bool TryFetchSdkFromExternalProvider(string platformName, string version) + { + if (!this.commonOptions.EnableExternalSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "External SDK provider is enabled so trying to fetch SDK using it.", + platformName, + version); + + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion(platformName, version, this.commonOptions.DebianFlavor); + var success = this.externalSdkProvider.RequestBlobAsync(platformName, blobName).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using external SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using external SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using external SDK provider.", platformName, version); + } + + return false; + } + + /// + /// Tries to pull the SDK from MCR container image if enabled. + /// + /// True if the SDK was successfully fetched from MCR. + private bool TryPullSdkFromMcr(string platformName, string version) + { + if (!this.commonOptions.EnableMcrSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "MCR SDK provider is enabled so trying to fetch SDK from container image.", + platformName, + version); + + try + { + var success = this.mcrSdkProvider.PullSdkAsync(platformName, version, this.commonOptions.DebianFlavor).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using MCR SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using MCR SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using MCR SDK provider.", platformName, version); + } + + return false; + } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs b/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs index e0c51dd5fd..caf38a5ae9 100644 --- a/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs +++ b/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs @@ -43,6 +43,10 @@ public class BuildScriptGeneratorOptions public bool EnableExternalSdkProvider { get; set; } + public bool EnableMcrSdkProvider { get; set; } + + public string McrSdkImageBaseUrl { get; set; } + public string DynamicInstallRootDir { get; set; } public bool EnableCheckers { get; set; } diff --git a/src/BuildScriptGenerator/Php/PhpPlatform.cs b/src/BuildScriptGenerator/Php/PhpPlatform.cs index 2a189a077c..e96e282caf 100644 --- a/src/BuildScriptGenerator/Php/PhpPlatform.cs +++ b/src/BuildScriptGenerator/Php/PhpPlatform.cs @@ -34,6 +34,7 @@ internal class PhpPlatform : IProgrammingPlatform private readonly PhpPlatformInstaller phpInstaller; private readonly PhpComposerInstaller phpComposerInstaller; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IMcrSdkProvider mcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -47,6 +48,7 @@ internal class PhpPlatform : IProgrammingPlatform /// The . /// The . /// The . + /// The . public PhpPlatform( IOptions phpScriptGeneratorOptions, IOptions commonOptions, @@ -57,6 +59,7 @@ public PhpPlatform( PhpPlatformInstaller phpInstaller, PhpComposerInstaller phpComposerInstaller, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) { this.phpScriptGeneratorOptions = phpScriptGeneratorOptions.Value; @@ -68,6 +71,7 @@ public PhpPlatform( this.phpInstaller = phpInstaller; this.phpComposerInstaller = phpComposerInstaller; this.externalSdkProvider = externalSdkProvider; + this.mcrSdkProvider = mcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -301,31 +305,18 @@ private void InstallPhp(string phpVersion, StringBuilder scriptBuilder) } else { - if (this.commonOptions.EnableExternalSdkProvider) + // Try external SDK provider first (blob storage via socket) + bool sdkFetched = this.TryFetchSdkFromExternalProvider("php", phpVersion); + + // Try MCR SDK provider (container image pull) + if (!sdkFetched) { - this.logger.LogDebug("Php version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", phpVersion); + sdkFetched = this.TryPullSdkFromMcr("php", phpVersion); + } - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion("php", phpVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("Php version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", phpVersion); - - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("Php version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", phpVersion); - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching php version {version} using external SDK provider.", phpVersion); - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion); - } + if (sdkFetched) + { + script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion, skipSdkBinaryDownload: true); } else { @@ -353,31 +344,18 @@ private void InstallPhpComposer(string phpComposerVersion, StringBuilder scriptB } else { - if (this.commonOptions.EnableExternalSdkProvider) + // Try external SDK provider first (blob storage via socket) + bool sdkFetched = this.TryFetchSdkFromExternalProvider("php-composer", phpComposerVersion); + + // Try MCR SDK provider (container image pull) + if (!sdkFetched) { - this.logger.LogDebug("Php Composer version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", phpComposerVersion); + sdkFetched = this.TryPullSdkFromMcr("php-composer", phpComposerVersion); + } - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion("php-composer", phpComposerVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync("php-composer", blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("Php composer version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", phpComposerVersion); - - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("Php comose version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", phpComposerVersion); - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching php composer version {version} using external SDK provider.", phpComposerVersion); - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion); - } + if (sdkFetched) + { + script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion, skipSdkBinaryDownload: true); } else { @@ -486,5 +464,90 @@ private string GetMaxSatisfyingPhpVersionAndVerify(string version) return maxSatisfyingVersion; } + + /// + /// Tries to fetch the SDK from the external SDK provider (blob storage via Unix socket) if enabled. + /// + /// True if the SDK was successfully fetched from the external provider. + private bool TryFetchSdkFromExternalProvider(string platformName, string version) + { + if (!this.commonOptions.EnableExternalSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "External SDK provider is enabled so trying to fetch SDK using it.", + platformName, + version); + + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion(platformName, version, this.commonOptions.DebianFlavor); + var success = this.externalSdkProvider.RequestBlobAsync(platformName, blobName).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using external SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using external SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using external SDK provider.", platformName, version); + } + + return false; + } + + /// + /// Tries to pull the SDK from MCR container image if enabled. + /// + /// True if the SDK was successfully fetched from MCR. + private bool TryPullSdkFromMcr(string platformName, string version) + { + if (!this.commonOptions.EnableMcrSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "MCR SDK provider is enabled so trying to fetch SDK from container image.", + platformName, + version); + + try + { + var success = this.mcrSdkProvider.PullSdkAsync(platformName, version, this.commonOptions.DebianFlavor).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using MCR SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using MCR SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using MCR SDK provider.", platformName, version); + } + + return false; + } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/Python/PythonPlatform.cs b/src/BuildScriptGenerator/Python/PythonPlatform.cs index 071b9fe8cd..56e20514f8 100644 --- a/src/BuildScriptGenerator/Python/PythonPlatform.cs +++ b/src/BuildScriptGenerator/Python/PythonPlatform.cs @@ -89,6 +89,7 @@ internal class PythonPlatform : IProgrammingPlatform private readonly IPythonPlatformDetector detector; private readonly PythonPlatformInstaller platformInstaller; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IMcrSdkProvider mcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -100,6 +101,7 @@ internal class PythonPlatform : IProgrammingPlatform /// The logger of Python platform. /// The detector of Python platform. /// The . + /// The . public PythonPlatform( IOptions commonOptions, IOptions pythonScriptGeneratorOptions, @@ -108,6 +110,7 @@ public PythonPlatform( IPythonPlatformDetector detector, PythonPlatformInstaller platformInstaller, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) { this.commonOptions = commonOptions.Value; @@ -117,6 +120,7 @@ public PythonPlatform( this.detector = detector; this.platformInstaller = platformInstaller; this.externalSdkProvider = externalSdkProvider; + this.mcrSdkProvider = mcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -395,30 +399,18 @@ public string GetInstallerScriptSnippet( } else { - if (this.commonOptions.EnableExternalSdkProvider) + // Try external SDK provider first (blob storage via socket) + bool sdkFetched = this.TryFetchSdkFromExternalProvider(this.Name, detectorResult.PlatformVersion); + + // Try MCR SDK provider (container image pull) + if (!sdkFetched) + { + sdkFetched = this.TryPullSdkFromMcr(this.Name, detectorResult.PlatformVersion); + } + + if (sdkFetched) { - this.logger.LogDebug("Python version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", detectorResult.PlatformVersion); - - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, detectorResult.PlatformVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("Python version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("Python version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching python version {version} using external SDK provider.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } + installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion, skipSdkBinaryDownload: true); } else { @@ -733,5 +725,90 @@ private string GetVersionUsingHierarchicalRules(string detectedVersion) var versionInfo = this.versionProvider.GetVersionInfo(); return versionInfo.DefaultVersion; } + + /// + /// Tries to fetch the SDK from the external SDK provider (blob storage via Unix socket) if enabled. + /// + /// True if the SDK was successfully fetched from the external provider. + private bool TryFetchSdkFromExternalProvider(string platformName, string version) + { + if (!this.commonOptions.EnableExternalSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "External SDK provider is enabled so trying to fetch SDK using it.", + platformName, + version); + + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion(platformName, version, this.commonOptions.DebianFlavor); + var success = this.externalSdkProvider.RequestBlobAsync(platformName, blobName).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using external SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using external SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using external SDK provider.", platformName, version); + } + + return false; + } + + /// + /// Tries to pull the SDK from MCR container image if enabled. + /// + /// True if the SDK was successfully fetched from MCR. + private bool TryPullSdkFromMcr(string platformName, string version) + { + if (!this.commonOptions.EnableMcrSdkProvider) + { + return false; + } + + this.logger.LogDebug( + "{platform} version {version} is not installed. " + + "MCR SDK provider is enabled so trying to fetch SDK from container image.", + platformName, + version); + + try + { + var success = this.mcrSdkProvider.PullSdkAsync(platformName, version, this.commonOptions.DebianFlavor).Result; + if (success) + { + this.logger.LogDebug( + "{platform} version {version} fetched successfully using MCR SDK provider.", + platformName, + version); + return true; + } + + this.logger.LogDebug( + "{platform} version {version} could not be fetched using MCR SDK provider. Falling through to next provider.", + platformName, + version); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching {platform} version {version} using MCR SDK provider.", platformName, version); + } + + return false; + } } } \ No newline at end of file diff --git a/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs b/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs index f3313c7c7e..47c15bf110 100644 --- a/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs +++ b/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs @@ -62,6 +62,8 @@ public void Configure(BuildScriptGeneratorLib.BuildScriptGeneratorOptions option // Dynamic install options.EnableDynamicInstall = this.GetBooleanValue(SettingsKeys.EnableDynamicInstall); options.EnableExternalSdkProvider = this.GetBooleanValue(SettingsKeys.EnableExternalSdkProvider); + options.EnableMcrSdkProvider = this.GetBooleanValue(SettingsKeys.EnableMcrSdkProvider); + options.McrSdkImageBaseUrl = this.GetStringValue(SettingsKeys.McrSdkImageBaseUrl); var dynamicInstallRootDir = this.GetStringValue(SettingsKeys.DynamicInstallRootDir); diff --git a/src/BuildScriptGeneratorCli/SettingsKeys.cs b/src/BuildScriptGeneratorCli/SettingsKeys.cs index b478745127..fd538c43d6 100644 --- a/src/BuildScriptGeneratorCli/SettingsKeys.cs +++ b/src/BuildScriptGeneratorCli/SettingsKeys.cs @@ -18,6 +18,8 @@ public static class SettingsKeys public const string CompressDestinationDir = "COMPRESS_DESTINATION_DIR"; public const string EnableDynamicInstall = "ENABLE_DYNAMIC_INSTALL"; public const string EnableExternalSdkProvider = "ORYX_ENABLE_EXTERNAL_SDK_PROVIDER"; + public const string EnableMcrSdkProvider = "ORYX_ENABLE_MCR_SDK_PROVIDER"; + public const string McrSdkImageBaseUrl = "ORYX_MCR_SDK_IMAGE_BASE_URL"; public const string DisableCheckers = "DISABLE_CHECKERS"; public const string DisableDotNetCoreBuild = "DISABLE_DOTNETCORE_BUILD"; public const string DisableGolangBuild = "DISABLE_GOLANG_BUILD"; diff --git a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs index 3621abb421..af42f99d15 100644 --- a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs +++ b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs @@ -98,7 +98,8 @@ private DotNetCorePlatform CreateDotNetCorePlatform( detector, DotNetCoreInstaller, globalJsonSdkResolver, - externalSdkProvider, + externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -113,6 +114,7 @@ public TestDotNetCorePlatform( DotNetCorePlatformInstaller DotNetCoreInstaller, GlobalJsonSdkResolver globalJsonSdkResolver, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) : base( DotNetCoreVersionProvider, @@ -123,6 +125,7 @@ public TestDotNetCorePlatform( DotNetCoreInstaller, globalJsonSdkResolver, externalSdkProvider, + mcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs index 328819275a..65f4986862 100644 --- a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs +++ b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs @@ -166,6 +166,7 @@ private DotNetCorePlatform CreatePlatform( installer, globalJsonSdkResolver, externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -179,6 +180,7 @@ public TestDotNetCorePlatform( DotNetCorePlatformInstaller platformInstaller, GlobalJsonSdkResolver globalJsonSdkResolver, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) : base( versionProvider, @@ -189,6 +191,7 @@ public TestDotNetCorePlatform( platformInstaller, globalJsonSdkResolver, externalSdkProvider, + mcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs b/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs index 347d020f85..537af9913b 100644 --- a/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs +++ b/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs @@ -984,6 +984,7 @@ private static IProgrammingPlatform GetNodePlatform( new TestEnvironment(), new NodePlatformInstaller(Options.Create(commonOptions), NullLoggerFactory.Instance), externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } diff --git a/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs b/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs index c28e3d0ce9..c4a83d60f3 100644 --- a/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs +++ b/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs @@ -1025,6 +1025,7 @@ private TestNodePlatform CreateNodePlatform( environment, platformInstaller, externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -1048,7 +1049,8 @@ private TestNodePlatform CreateNodePlatform( detector, environment, platformInstaller, - externalSdkProvider, + externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -1077,6 +1079,7 @@ private TestNodePlatform CreateNodePlatform( environment, installer, externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -1101,6 +1104,7 @@ public TestNodePlatform( IEnvironment environment, NodePlatformInstaller nodePlatformInstaller, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) : base( cliOptions, @@ -1111,6 +1115,7 @@ public TestNodePlatform( environment, nodePlatformInstaller, externalSdkProvider, + mcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs b/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs index 57f06f0c92..5cc3d51f39 100644 --- a/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs +++ b/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs @@ -591,6 +591,7 @@ private PhpPlatform CreatePhpPlatform( phpInstaller, phpComposerInstaller, externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -616,6 +617,7 @@ public TestPhpPlatform( PhpPlatformInstaller phpInstaller, PhpComposerInstaller phpComposerInstaller, IExternalSdkProvider externalSdkProvider, + IMcrSdkProvider mcrSdkProvider, TelemetryClient telemetryClient) : base( phpScriptGeneratorOptions, @@ -627,6 +629,7 @@ public TestPhpPlatform( phpInstaller, phpComposerInstaller, externalSdkProvider, + mcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs b/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs index 958a40ea0a..7e7c5be065 100644 --- a/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs +++ b/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs @@ -146,6 +146,7 @@ private IProgrammingPlatform GetScriptGenerator( phpInstaller: null, phpComposerInstaller: null, externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } diff --git a/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs b/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs index f068c1a5ba..b354be52f4 100644 --- a/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs +++ b/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs @@ -433,6 +433,7 @@ private PythonPlatform CreatePlatform( detector: null, platformInstaller, externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -460,6 +461,7 @@ private PythonPlatform CreatePlatform( detector, new PythonPlatformInstaller(Options.Create(commonOptions), NullLoggerFactory.Instance), externalSdkProvider, + new TestMcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } diff --git a/tests/Oryx.Tests.Common/TestMcrSdkProvider.cs b/tests/Oryx.Tests.Common/TestMcrSdkProvider.cs new file mode 100644 index 0000000000..657e5e916e --- /dev/null +++ b/tests/Oryx.Tests.Common/TestMcrSdkProvider.cs @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Microsoft.Oryx.BuildScriptGenerator; + +namespace Microsoft.Oryx.Tests.Common +{ + public class TestMcrSdkProvider : IMcrSdkProvider + { + public Task PullSdkAsync(string platformName, string version, string debianFlavor) + { + return Task.FromResult(false); + } + } +}