diff --git a/src/BuildScriptGenerator.Common/SdkStorageConstants.cs b/src/BuildScriptGenerator.Common/SdkStorageConstants.cs index 92c1ba0920..61ea1a8e7a 100644 --- a/src/BuildScriptGenerator.Common/SdkStorageConstants.cs +++ b/src/BuildScriptGenerator.Common/SdkStorageConstants.cs @@ -21,5 +21,18 @@ public static class SdkStorageConstants public const string DotnetRuntimeVersionMetadataName = "Dotnet_runtime_version"; public const string LegacyDotnetRuntimeVersionMetadataName = "Runtime_version"; public const string OsTypeMetadataName = "Os_type"; + + // ACR-based SDK distribution constants + public const string EnableAcrSdkProviderKey = "ORYX_ENABLE_ACR_SDK_PROVIDER"; + public const string AcrSdkRegistryUrlKeyName = "ORYX_ACR_SDK_REGISTRY_URL"; + public const string DefaultAcrSdkRegistryUrl = "https://oryxacr.azurecr.io"; + public const string AcrSdkRepositoryPrefix = "sdks"; + public const string AcrDefaultVersionTag = "default"; + public const string AcrCatalogTag = "catalog"; + public const string AcrVersionLabelName = "org.oryx.version"; + public const string AcrPlatformLabelName = "org.oryx.platform"; + public const string AcrOsFlavorLabelName = "org.oryx.os-flavor"; + public const string AcrDotnetRuntimeVersionLabelName = "org.oryx.dotnet-runtime-version"; + public const string AcrDotnetSdkVersionLabelName = "org.oryx.dotnet-sdk-version"; } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/AcrVersionProviderBase.cs b/src/BuildScriptGenerator/AcrVersionProviderBase.cs new file mode 100644 index 0000000000..ec046b4fb1 --- /dev/null +++ b/src/BuildScriptGenerator/AcrVersionProviderBase.cs @@ -0,0 +1,105 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Oryx.BuildScriptGenerator.Common; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + /// + /// Base class for ACR-based SDK version providers. Parallel to + /// but discovers versions via OCI Distribution API (tag listing + image config labels) instead of + /// Azure Blob Storage listing with XML metadata. + /// + public class AcrVersionProviderBase + { + private readonly ILogger logger; + private readonly string debianFlavor; + + public AcrVersionProviderBase( + IOptions commonOptions, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + { + var options = commonOptions.Value; + this.logger = loggerFactory.CreateLogger(this.GetType()); + this.debianFlavor = options.DebianFlavor; + + var registryUrl = string.IsNullOrEmpty(options.OryxAcrSdkRegistryUrl) + ? SdkStorageConstants.DefaultAcrSdkRegistryUrl + : options.OryxAcrSdkRegistryUrl; + + this.OciClient = new OciRegistryClient(registryUrl, httpClientFactory, loggerFactory); + } + + protected OciRegistryClient OciClient { get; } + + /// + /// Lists available versions for a platform from ACR tags. + /// Tags are in the format "{osFlavor}-{version}" (e.g. "bookworm-20.19.3"). + /// Tags ending with "-default" or "-catalog" are excluded. + /// + protected PlatformVersionInfo GetAvailableVersionsFromAcr(string platformName) + { + var repository = $"{SdkStorageConstants.AcrSdkRepositoryPrefix}/{platformName}"; + + this.logger.LogDebug("Getting available versions for {platformName} from ACR repository {repository}.", platformName, repository); + + var allTags = this.GetTags(repository); + var supportedVersions = this.FilterVersionTags(allTags); + var defaultVersion = this.GetDefaultVersion(repository); + + this.logger.LogDebug( + "Found {count} versions for {platformName} on ACR (default: {default}).", + supportedVersions.Count, + platformName, + defaultVersion ?? "none"); + + return PlatformVersionInfo.CreateAvailableOnAcr(supportedVersions, defaultVersion); + } + + private List GetTags(string repository) + { + try + { + return this.OciClient.GetAllTagsAsync(repository).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Failed to get tags from ACR for {repository}.", repository); + throw; + } + } + + private List FilterVersionTags(List allTags) + { + var prefix = $"{this.debianFlavor}-"; + return allTags + .Where(t => t.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + && !t.EndsWith($"-{SdkStorageConstants.AcrDefaultVersionTag}", StringComparison.OrdinalIgnoreCase) + && !t.EndsWith($"-{SdkStorageConstants.AcrCatalogTag}", StringComparison.OrdinalIgnoreCase)) + .Select(t => t.Substring(prefix.Length)) + .ToList(); + } + + private string GetDefaultVersion(string repository) + { + try + { + return this.OciClient.GetDefaultVersionAsync(repository, this.debianFlavor).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + this.logger.LogWarning(ex, "Failed to get default version from ACR for {repository}.", repository); + return null; + } + } + } +} diff --git a/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs b/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs index dc63ad3c89..9d825a03de 100644 --- a/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs +++ b/src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs @@ -41,6 +41,7 @@ public static IServiceCollection AddBuildScriptGeneratorServices(this IServiceCo services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHttpClient("general", httpClient => { // NOTE: Setting user agent is required to avoid receiving 403 Forbidden response. diff --git a/src/BuildScriptGenerator/Contracts/IExternalAcrSdkProvider.cs b/src/BuildScriptGenerator/Contracts/IExternalAcrSdkProvider.cs new file mode 100644 index 0000000000..b66f2404f1 --- /dev/null +++ b/src/BuildScriptGenerator/Contracts/IExternalAcrSdkProvider.cs @@ -0,0 +1,31 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + /// + /// Interface for external ACR-based SDK provider that communicates with LWASv2 + /// to request SDK downloads from Azure Container Registry (WAWS Images ACR). + /// This is the ACR equivalent of which uses blob storage. + /// + /// + /// Gated by the ORYX_ENABLE_ACR_SDK_PROVIDER feature flag. + /// When enabled and LWASv2 is available, this provider tells LWASv2 to pull + /// the SDK OCI image from the WAWS Images ACR and extract the SDK tarball to disk. + /// + public interface IExternalAcrSdkProvider + { + /// + /// Requests LWASv2 to pull an SDK image from the WAWS Images ACR and extract it to the local cache. + /// + /// The platform name (e.g., "nodejs", "python", "dotnet", "php"). + /// The SDK version (e.g., "20.19.3"). + /// The Debian flavor (e.g., "bookworm", "bullseye"). + /// True if the SDK was successfully pulled and extracted by LWASv2. + Task RequestSdkFromAcrAsync(string platformName, string version, string debianFlavor); + } +} diff --git a/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs b/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs index afeb21c9f3..4b733ff5a9 100644 --- a/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs +++ b/src/BuildScriptGenerator/DotNetCore/DotnetCorePlatform.cs @@ -35,6 +35,7 @@ internal class DotNetCorePlatform : IProgrammingPlatform private readonly DotNetCorePlatformInstaller platformInstaller; private readonly GlobalJsonSdkResolver globalJsonSdkResolver; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IExternalAcrSdkProvider externalAcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -56,6 +57,7 @@ public DotNetCorePlatform( DotNetCorePlatformInstaller platformInstaller, GlobalJsonSdkResolver globalJsonSdkResolver, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) { this.versionProvider = versionProvider; @@ -66,6 +68,7 @@ public DotNetCorePlatform( this.platformInstaller = platformInstaller; this.globalJsonSdkResolver = globalJsonSdkResolver; this.externalSdkProvider = externalSdkProvider; + this.externalAcrSdkProvider = externalAcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -233,55 +236,38 @@ public string GetInstallerScriptSnippet( $"'{typeof(DotNetCorePlatformDetectorResult)}' but got '{detectorResult.GetType()}'."); } - string installationScriptSnippet = null; - if (this.commonOptions.EnableDynamicInstall) + if (!this.commonOptions.EnableDynamicInstall) { - this.logger.LogDebug("Dynamic install is enabled."); + this.logger.LogDebug("Dynamic install is not enabled."); + return null; + } - if (this.platformInstaller.IsVersionAlreadyInstalled(dotNetCorePlatformDetectorResult.SdkVersion)) - { - this.logger.LogDebug("DotNetCore SDK version {globalJsonSdkVersion} is already installed. So skipping installing it again.", dotNetCorePlatformDetectorResult.SdkVersion); - } - else - { - if (this.commonOptions.EnableExternalSdkProvider) - { - this.logger.LogDebug("DotNetCore SDK version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", dotNetCorePlatformDetectorResult.SdkVersion); - - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, dotNetCorePlatformDetectorResult.SdkVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("DotNetCore SDK version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("DotNetCore SDK version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching DotNetCore SDK version version {version} using external SDK provider.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion); - } - } - else - { - this.logger.LogDebug("DotNetCore SDK version {globalJsonSdkVersion} is not installed. So generating an installation script snippet for it.", dotNetCorePlatformDetectorResult.SdkVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(dotNetCorePlatformDetectorResult.SdkVersion); - } - } + this.logger.LogDebug("Dynamic install is enabled."); + + var sdkVersion = dotNetCorePlatformDetectorResult.SdkVersion; + + if (this.platformInstaller.IsVersionAlreadyInstalled(sdkVersion)) + { + this.logger.LogDebug( + "DotNetCore SDK version {globalJsonSdkVersion} is already installed. So skipping installing it again.", + sdkVersion); + return null; } - else + + if (this.commonOptions.EnableExternalSdkProvider) { - this.logger.LogDebug("Dynamic install is not enabled."); + return this.TryInstallFromExternalSdkProvider(sdkVersion); + } + + if (this.commonOptions.EnableAcrSdkProvider) + { + return this.TryInstallFromAcrSdkProvider(sdkVersion); } - return installationScriptSnippet; + this.logger.LogDebug( + "DotNetCore SDK version {globalJsonSdkVersion} is not installed. So generating an installation script snippet for it.", + sdkVersion); + return this.platformInstaller.GetInstallerScriptSnippet(sdkVersion); } /// @@ -359,6 +345,70 @@ private static void SetStartupFileNameInfoInManifestFile( buildProperties[DotNetCoreManifestFilePropertyKeys.StartupDllFileName] = startupDllFileName; } + private string TryInstallFromAcrSdkProvider(string sdkVersion) + { + this.logger.LogDebug( + "DotNetCore SDK version {version} is not installed. ACR SDK provider is enabled, so trying to fetch SDK using it.", + sdkVersion); + + try + { + if (this.externalAcrSdkProvider.RequestSdkFromAcrAsync( + this.Name, sdkVersion, this.commonOptions.DebianFlavor).Result) + { + this.logger.LogDebug( + "DotNetCore SDK version {version} is fetched successfully using ACR SDK provider. Skipping platform binary download.", + sdkVersion); + return this.platformInstaller.GetInstallerScriptSnippet(sdkVersion, skipSdkBinaryDownload: true); + } + + this.logger.LogDebug( + "DotNetCore SDK version {version} is not fetched via ACR SDK provider. Falling back to CDN download.", + sdkVersion); + } + catch (Exception ex) + { + this.logger.LogError( + ex, + "Error while fetching DotNetCore SDK version {version} using ACR SDK provider. Falling back to CDN download.", + sdkVersion); + } + + return this.platformInstaller.GetInstallerScriptSnippet(sdkVersion); + } + + private string TryInstallFromExternalSdkProvider(string sdkVersion) + { + this.logger.LogDebug( + "DotNetCore SDK version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", + sdkVersion); + + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, sdkVersion, this.commonOptions.DebianFlavor); + if (this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result) + { + this.logger.LogDebug( + "DotNetCore SDK version {version} is fetched successfully using external SDK provider. Skipping platform binary download.", + sdkVersion); + return this.platformInstaller.GetInstallerScriptSnippet(sdkVersion, skipSdkBinaryDownload: true); + } + + this.logger.LogDebug( + "DotNetCore SDK version {version} is not fetched successfully using external SDK provider. Generating installation script snippet.", + sdkVersion); + } + catch (Exception ex) + { + this.logger.LogError( + ex, + "Error while fetching DotNetCore SDK version version {version} using external SDK provider.", + sdkVersion); + } + + return this.platformInstaller.GetInstallerScriptSnippet(sdkVersion); + } + private string GetSdkVersion( RepositoryContext context, string runtimeVersion, diff --git a/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorServiceCollectionExtensions.cs b/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorServiceCollectionExtensions.cs index ecd68fd61c..a687d26dc4 100644 --- a/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorServiceCollectionExtensions.cs +++ b/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorServiceCollectionExtensions.cs @@ -21,6 +21,7 @@ public static IServiceCollection AddDotNetCoreScriptGeneratorServices(this IServ services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); return services; } } diff --git a/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreAcrVersionProvider.cs b/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreAcrVersionProvider.cs new file mode 100644 index 0000000000..adc6e1ec1e --- /dev/null +++ b/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreAcrVersionProvider.cs @@ -0,0 +1,186 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Oryx.BuildScriptGenerator.Common; + +namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore +{ + /// + /// ACR-based version provider for .NET SDKs. + /// Unlike other platforms, .NET requires a runtime→SDK version mapping. + /// This provider uses a catalog tag containing a JSON mapping, and falls back + /// to per-tag config label inspection. + /// + public class DotNetCoreAcrVersionProvider : AcrVersionProviderBase, IDotNetCoreVersionProvider + { + private readonly BuildScriptGeneratorOptions commonOptions; + private readonly ILogger logger; + private Dictionary versionMap; + private string defaultRuntimeVersion; + + public DotNetCoreAcrVersionProvider( + IOptions commonOptions, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + : base(commonOptions, httpClientFactory, loggerFactory) + { + this.commonOptions = commonOptions.Value; + this.logger = loggerFactory.CreateLogger(); + } + + public string GetDefaultRuntimeVersion() + { + this.EnsureVersionInfo(); + return this.defaultRuntimeVersion; + } + + public Dictionary GetSupportedVersions() + { + this.EnsureVersionInfo(); + return this.versionMap; + } + + private void EnsureVersionInfo() + { + if (this.versionMap != null) + { + return; + } + + var repository = $"{SdkStorageConstants.AcrSdkRepositoryPrefix}/{DotNetCoreConstants.PlatformName}"; + var debianFlavor = this.commonOptions.DebianFlavor; + + // Try catalog tag first — single HTTP round-trip for the full runtime→SDK mapping + if (!this.TryGetVersionInfoFromCatalog(repository, debianFlavor)) + { + // Fallback: inspect individual tag configs (more HTTP calls) + this.GetVersionInfoFromTags(repository, debianFlavor); + } + } + + private bool TryGetVersionInfoFromCatalog(string repository, string debianFlavor) + { + try + { + var catalogTag = $"{debianFlavor}-{SdkStorageConstants.AcrCatalogTag}"; + this.logger.LogDebug("Trying .NET catalog tag {tag} from ACR", catalogTag); + + var manifest = this.OciClient.GetManifestAsync(repository, catalogTag).GetAwaiter().GetResult(); + var configDigest = manifest.Config?.Digest; + if (string.IsNullOrEmpty(configDigest)) + { + return false; + } + + var config = this.OciClient.GetImageConfigAsync(repository, configDigest).GetAwaiter().GetResult(); + if (config?.Config?.Labels == null || + !config.Config.Labels.TryGetValue("org.oryx.dotnet-version-map", out var mapJson)) + { + return false; + } + + var catalog = JsonSerializer.Deserialize(mapJson); + if (catalog?.Mappings == null) + { + return false; + } + + this.versionMap = new Dictionary(catalog.Mappings, StringComparer.OrdinalIgnoreCase); + this.defaultRuntimeVersion = catalog.DefaultRuntimeVersion; + this.logger.LogDebug("Got .NET version map from catalog tag with {count} entries.", this.versionMap.Count); + return true; + } + catch (Exception ex) + { + this.logger.LogDebug(ex, "Catalog tag not available, falling back to per-tag inspection."); + return false; + } + } + + private void GetVersionInfoFromTags(string repository, string debianFlavor) + { + this.logger.LogDebug("Getting .NET version info from individual ACR tags."); + + var allTags = this.OciClient.GetAllTagsAsync(repository).GetAwaiter().GetResult(); + + var prefix = $"{debianFlavor}-"; + var versionTags = allTags + .Where(t => t.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + && !t.EndsWith($"-{SdkStorageConstants.AcrDefaultVersionTag}", StringComparison.OrdinalIgnoreCase) + && !t.EndsWith($"-{SdkStorageConstants.AcrCatalogTag}", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + var supportedVersions = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var tag in versionTags) + { + this.TryExtractDotNetVersionFromTag(repository, tag, supportedVersions); + } + + this.versionMap = supportedVersions; + this.defaultRuntimeVersion = this.GetDefaultRuntimeVersionFromAcr(repository, debianFlavor); + } + + private void TryExtractDotNetVersionFromTag( + string repository, + string tag, + Dictionary supportedVersions) + { + try + { + var manifest = this.OciClient.GetManifestAsync(repository, tag).GetAwaiter().GetResult(); + var configDigest = manifest.Config?.Digest; + if (string.IsNullOrEmpty(configDigest)) + { + return; + } + + var config = this.OciClient.GetImageConfigAsync(repository, configDigest).GetAwaiter().GetResult(); + var labels = config?.Config?.Labels; + if (labels == null) + { + return; + } + + if (labels.TryGetValue(SdkStorageConstants.AcrDotnetRuntimeVersionLabelName, out var runtimeVersion) && + labels.TryGetValue(SdkStorageConstants.AcrDotnetSdkVersionLabelName, out var sdkVersion)) + { + supportedVersions[runtimeVersion] = sdkVersion; + } + } + catch (Exception ex) + { + this.logger.LogWarning(ex, "Failed to inspect config for tag {tag}.", tag); + } + } + + private string GetDefaultRuntimeVersionFromAcr(string repository, string debianFlavor) + { + try + { + return this.OciClient.GetDefaultVersionAsync(repository, debianFlavor).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + this.logger.LogWarning(ex, "Failed to get default .NET runtime version from ACR."); + return null; + } + } + + private class DotNetCatalog + { + public Dictionary Mappings { get; set; } + + public string DefaultRuntimeVersion { get; set; } + } + } +} diff --git a/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreVersionProvider.cs b/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreVersionProvider.cs index 28c5236f5b..f5a3f2d953 100644 --- a/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreVersionProvider.cs +++ b/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreVersionProvider.cs @@ -15,6 +15,7 @@ internal class DotNetCoreVersionProvider : IDotNetCoreVersionProvider private readonly DotNetCoreOnDiskVersionProvider onDiskVersionProvider; private readonly DotNetCoreSdkStorageVersionProvider sdkStorageVersionProvider; private readonly DotNetCoreExternalVersionProvider externalVersionProvider; + private readonly DotNetCoreAcrVersionProvider acrVersionProvider; private readonly ILogger logger; private string defaultRuntimeVersion; private Dictionary supportedVersions; @@ -24,12 +25,14 @@ public DotNetCoreVersionProvider( DotNetCoreOnDiskVersionProvider onDiskVersionProvider, DotNetCoreSdkStorageVersionProvider sdkStorageVersionProvider, DotNetCoreExternalVersionProvider externalVersionProvider, + DotNetCoreAcrVersionProvider acrVersionProvider, ILogger logger) { this.cliOptions = cliOptions.Value; this.onDiskVersionProvider = onDiskVersionProvider; this.sdkStorageVersionProvider = sdkStorageVersionProvider; this.externalVersionProvider = externalVersionProvider; + this.acrVersionProvider = acrVersionProvider; this.logger = logger; } @@ -37,29 +40,9 @@ public string GetDefaultRuntimeVersion() { if (string.IsNullOrEmpty(this.defaultRuntimeVersion)) { - if (this.cliOptions.EnableDynamicInstall) - { - if (this.cliOptions.EnableExternalSdkProvider) - { - try - { - this.defaultRuntimeVersion = this.externalVersionProvider.GetDefaultRuntimeVersion(); - } - catch (System.Exception ex) - { - this.logger.LogError($"Failed to get default runtime version from external SDK provider. Falling back to http based sdkStorageVersionProvider. Ex: {ex}"); - this.defaultRuntimeVersion = this.sdkStorageVersionProvider.GetDefaultRuntimeVersion(); - } - } - else - { - this.defaultRuntimeVersion = this.sdkStorageVersionProvider.GetDefaultRuntimeVersion(); - } - } - else - { - this.defaultRuntimeVersion = this.onDiskVersionProvider.GetDefaultRuntimeVersion(); - } + this.defaultRuntimeVersion = this.cliOptions.EnableDynamicInstall + ? this.ResolveDynamicDefaultRuntimeVersion() + : this.onDiskVersionProvider.GetDefaultRuntimeVersion(); } this.logger.LogDebug("Default runtime version is {defaultRuntimeVersion}", this.defaultRuntimeVersion); @@ -71,29 +54,9 @@ public Dictionary GetSupportedVersions() { if (this.supportedVersions == null) { - if (this.cliOptions.EnableDynamicInstall) - { - if (this.cliOptions.EnableExternalSdkProvider) - { - try - { - this.supportedVersions = this.externalVersionProvider.GetSupportedVersions(); - } - catch (System.Exception ex) - { - this.logger.LogError($"Failed to get supported versions from external SDK provider. Falling back to http based sdkStorageVersionProvider. Ex: {ex}"); - this.supportedVersions = this.sdkStorageVersionProvider.GetSupportedVersions(); - } - } - else - { - this.supportedVersions = this.sdkStorageVersionProvider.GetSupportedVersions(); - } - } - else - { - this.supportedVersions = this.onDiskVersionProvider.GetSupportedVersions(); - } + this.supportedVersions = this.cliOptions.EnableDynamicInstall + ? this.ResolveDynamicSupportedVersions() + : this.onDiskVersionProvider.GetSupportedVersions(); // A temporary fix to make building netcoreapp1.0 versions using the 1.1 SDK // This SDK has 2 runtimes: 1.1.13 and 1.0.16 @@ -105,5 +68,67 @@ public Dictionary GetSupportedVersions() return this.supportedVersions; } + + private string ResolveDynamicDefaultRuntimeVersion() + { + if (this.cliOptions.EnableExternalSdkProvider) + { + try + { + return this.externalVersionProvider.GetDefaultRuntimeVersion(); + } + catch (System.Exception ex) + { + this.logger.LogError( + $"Failed to get default runtime version from external SDK provider. Falling back. Ex: {ex}"); + } + } + + if (this.cliOptions.EnableAcrSdkProvider) + { + try + { + return this.acrVersionProvider.GetDefaultRuntimeVersion(); + } + catch (System.Exception ex) + { + this.logger.LogError( + $"Failed to get default runtime version from ACR provider. Falling back to blob storage. Ex: {ex}"); + } + } + + return this.sdkStorageVersionProvider.GetDefaultRuntimeVersion(); + } + + private Dictionary ResolveDynamicSupportedVersions() + { + if (this.cliOptions.EnableExternalSdkProvider) + { + try + { + return this.externalVersionProvider.GetSupportedVersions(); + } + catch (System.Exception ex) + { + this.logger.LogError( + $"Failed to get supported versions from external SDK provider. Falling back. Ex: {ex}"); + } + } + + if (this.cliOptions.EnableAcrSdkProvider) + { + try + { + return this.acrVersionProvider.GetSupportedVersions(); + } + catch (System.Exception ex) + { + this.logger.LogError( + $"Failed to get supported versions from ACR provider. Falling back to blob storage. Ex: {ex}"); + } + } + + return this.sdkStorageVersionProvider.GetSupportedVersions(); + } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/ExternalAcrSdkProvider.cs b/src/BuildScriptGenerator/ExternalAcrSdkProvider.cs new file mode 100644 index 0000000000..461b1fa4da --- /dev/null +++ b/src/BuildScriptGenerator/ExternalAcrSdkProvider.cs @@ -0,0 +1,310 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Oryx.BuildScriptGenerator.Common; +using Microsoft.Oryx.Common.Extensions; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + /// + /// External ACR-based SDK provider that fetches SDK tarballs from OCI images. + /// SDK images are built as FROM scratch; COPY sdk.tar.gz /, so each image + /// contains exactly one layer — the SDK tarball itself. + /// + /// + /// Two pull strategies, tried in order: + /// + /// + /// Unix socket (LWASv2) — available inside App Service. Sends a request to + /// LWASv2's OryxProxy with source=acr; LWASv2 pulls the image and extracts + /// the tarball to /var/OryxSdks. + /// + /// + /// Direct OCI pull — fallback when the socket is unavailable (CLI builds, + /// local dev). Uses to fetch the manifest, extract + /// the single layer digest, download the blob, and verify SHA256. + /// + /// + /// + public class ExternalAcrSdkProvider : IExternalAcrSdkProvider + { + /// + /// The directory where SDKs are cached by the external provider (same as blob-based). + /// + public const string ExternalSdksStorageDir = "/var/OryxSdks"; + + private const string SocketPath = "/var/sockets/oryx-pull-sdk.socket"; + private const int MaxTimeoutForSocketOperationInSeconds = 120; + + private readonly ILogger logger; + private readonly IStandardOutputWriter outputWriter; + private readonly BuildScriptGeneratorOptions options; + private readonly OciRegistryClient ociClient; + + public ExternalAcrSdkProvider( + IStandardOutputWriter outputWriter, + ILogger logger, + IOptions options, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + { + this.logger = logger; + this.outputWriter = outputWriter; + this.options = options.Value; + + var registryUrl = string.IsNullOrEmpty(this.options.OryxAcrSdkRegistryUrl) + ? SdkStorageConstants.DefaultAcrSdkRegistryUrl + : this.options.OryxAcrSdkRegistryUrl; + + this.ociClient = new OciRegistryClient(registryUrl, httpClientFactory, loggerFactory); + } + + /// + public async Task RequestSdkFromAcrAsync(string platformName, string version, string debianFlavor) + { + if (string.IsNullOrEmpty(platformName)) + { + throw new ArgumentException("Platform name cannot be null or empty.", nameof(platformName)); + } + + if (string.IsNullOrEmpty(version)) + { + throw new ArgumentException("Version cannot be null or empty.", nameof(version)); + } + + if (string.IsNullOrEmpty(debianFlavor)) + { + debianFlavor = this.options.DebianFlavor ?? "bookworm"; + } + + var blobName = $"{platformName}-{debianFlavor}-{version}.tar.gz"; + var expectedFilePath = Path.Combine(ExternalSdksStorageDir, platformName, blobName); + + this.logger.LogInformation( + "Requesting SDK from ACR: platform={PlatformName}, version={Version}, debianFlavor={DebianFlavor}", + platformName, + version, + debianFlavor); + this.outputWriter.WriteLine( + $"Requesting SDK from ACR: {platformName} {version} ({debianFlavor})"); + + // Check if the file is already cached locally + if (File.Exists(expectedFilePath)) + { + this.logger.LogInformation( + "SDK already cached locally at {FilePath}, skipping ACR pull.", + expectedFilePath); + this.outputWriter.WriteLine($"SDK already cached locally at {expectedFilePath}"); + return true; + } + + // Strategy 1: Try Unix socket (LWASv2) if the socket exists on disk + if (File.Exists(SocketPath)) + { + var socketResult = await this.TryPullViaSocketAsync(platformName, version, debianFlavor, blobName, expectedFilePath); + if (socketResult) + { + return true; + } + + this.logger.LogWarning( + "LWASv2 socket pull failed for {PlatformName} {Version}. Falling back to direct OCI pull.", + platformName, + version); + } + else + { + this.logger.LogDebug( + "LWASv2 socket not found at {SocketPath}. Using direct OCI pull.", + SocketPath); + } + + // Strategy 2: Direct OCI pull — fetch manifest, get layer digest, download blob + return await this.TryPullDirectFromAcrAsync(platformName, version, debianFlavor, expectedFilePath); + } + + /// + /// Pulls the SDK via LWASv2 Unix socket. + /// + private async Task TryPullViaSocketAsync( + string platformName, + string version, + string debianFlavor, + string blobName, + string expectedFilePath) + { + try + { + var request = new AcrSdkProviderRequest + { + PlatformName = platformName, + BlobName = blobName, + UrlParameters = new Dictionary + { + { "source", "acr" }, + { "version", version }, + { "debianFlavor", debianFlavor }, + }, + }; + + var response = await this.SendSocketRequestAsync(request); + + if (response && File.Exists(expectedFilePath)) + { + this.logger.LogInformation( + "Successfully pulled SDK from ACR via LWASv2: {PlatformName} {Version}", + platformName, + version); + this.outputWriter.WriteLine( + $"Successfully pulled SDK from ACR via LWASv2: {platformName} {version}"); + return true; + } + } + catch (Exception ex) + { + this.logger.LogError( + ex, + "Error requesting SDK from ACR via LWASv2: {PlatformName} {Version}", + platformName, + version); + } + + return false; + } + + /// + /// Pulls the SDK directly from the ACR registry using the OCI Distribution API. + /// SDK images are FROM scratch with a single layer that IS the tarball: + /// + /// FROM scratch + /// COPY sdk.tar.gz / + /// + /// Flow: GET manifest → extract single layer digest → GET blob → verify SHA256 → save to cache. + /// + private async Task TryPullDirectFromAcrAsync( + string platformName, + string version, + string debianFlavor, + string expectedFilePath) + { + try + { + var repository = $"{SdkStorageConstants.AcrSdkRepositoryPrefix}/{platformName}"; + var tag = $"{debianFlavor}-{version}"; + + this.logger.LogInformation( + "Pulling SDK directly from ACR via OCI API: {Repository}:{Tag}", + repository, + tag); + this.outputWriter.WriteLine( + $"Pulling SDK directly from ACR: {repository}:{tag}"); + + var success = await this.ociClient.PullSdkAsync(repository, tag, expectedFilePath); + + if (success) + { + this.logger.LogInformation( + "Successfully pulled SDK directly from ACR: {PlatformName} {Version}, saved to {FilePath}", + platformName, + version, + expectedFilePath); + this.outputWriter.WriteLine( + $"Successfully pulled SDK from ACR: {platformName} {version}"); + return true; + } + + this.logger.LogWarning( + "Direct OCI pull did not succeed for {PlatformName} {Version}.", + platformName, + version); + this.outputWriter.WriteLine( + $"Failed to pull SDK from ACR: {platformName} {version}"); + return false; + } + catch (Exception ex) + { + this.logger.LogError( + ex, + "Error pulling SDK directly from ACR: {PlatformName} {Version}", + platformName, + version); + this.outputWriter.WriteLine( + $"Error pulling SDK from ACR: {platformName} {version}: {ex.Message}"); + return false; + } + } + + private async Task SendSocketRequestAsync(AcrSdkProviderRequest request) + { + using var socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); + try + { + this.logger.LogInformation( + "Sending ACR SDK request to LWASv2: {PlatformName}, {BlobName}", + request.PlatformName, + request.BlobName); + + using (var cts = new CancellationTokenSource( + TimeSpan.FromSeconds(MaxTimeoutForSocketOperationInSeconds))) + { + await socket.ConnectAsync(new UnixDomainSocketEndPoint(SocketPath), cts.Token); + var requestJson = JsonSerializer.Serialize(request) + "$"; + var requestBytes = Encoding.UTF8.GetBytes(requestJson); + + await socket.SendAsync(new ArraySegment(requestBytes), SocketFlags.None, cts.Token); + var buffer = new byte[4096]; + var received = await socket.ReceiveAsync( + new ArraySegment(buffer), SocketFlags.None, cts.Token); + var responseString = Encoding.UTF8.GetString(buffer, 0, received); + + this.logger.LogInformation( + "Received response from LWASv2: {Response}", responseString); + + if (!string.IsNullOrEmpty(responseString) && responseString.EqualsIgnoreCase("Success$")) + { + return true; + } + + this.logger.LogError( + "LWASv2 ACR SDK request unsuccessful. Response: {Response}", + responseString); + } + } + catch (OperationCanceledException) + { + this.outputWriter.WriteLine("The LWASv2 ACR SDK request timed out."); + this.logger.LogError("The LWASv2 ACR SDK request timed out."); + } + catch (Exception ex) + { + this.outputWriter.WriteLine( + $"Error communicating with LWASv2: {ex.Message}"); + this.logger.LogError(ex, "Error communicating with LWASv2."); + } + + return false; + } + + private class AcrSdkProviderRequest + { + public string PlatformName { get; set; } + + public string BlobName { get; set; } + + public IDictionary UrlParameters { get; set; } + } + } +} diff --git a/src/BuildScriptGenerator/Helpers/OciContainerConfig.cs b/src/BuildScriptGenerator/Helpers/OciContainerConfig.cs new file mode 100644 index 0000000000..aac9c89b49 --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/OciContainerConfig.cs @@ -0,0 +1,16 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + public class OciContainerConfig + { + [JsonPropertyName("Labels")] + public Dictionary Labels { get; set; } + } +} diff --git a/src/BuildScriptGenerator/Helpers/OciDescriptor.cs b/src/BuildScriptGenerator/Helpers/OciDescriptor.cs new file mode 100644 index 0000000000..0c291f7b2e --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/OciDescriptor.cs @@ -0,0 +1,21 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Text.Json.Serialization; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + public class OciDescriptor + { + [JsonPropertyName("mediaType")] + public string MediaType { get; set; } + + [JsonPropertyName("digest")] + public string Digest { get; set; } + + [JsonPropertyName("size")] + public long Size { get; set; } + } +} diff --git a/src/BuildScriptGenerator/Helpers/OciImageConfig.cs b/src/BuildScriptGenerator/Helpers/OciImageConfig.cs new file mode 100644 index 0000000000..7938148af2 --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/OciImageConfig.cs @@ -0,0 +1,15 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Text.Json.Serialization; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + public class OciImageConfig + { + [JsonPropertyName("config")] + public OciContainerConfig Config { get; set; } + } +} diff --git a/src/BuildScriptGenerator/Helpers/OciManifest.cs b/src/BuildScriptGenerator/Helpers/OciManifest.cs new file mode 100644 index 0000000000..36312d64cf --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/OciManifest.cs @@ -0,0 +1,25 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + public class OciManifest + { + [JsonPropertyName("schemaVersion")] + public int SchemaVersion { get; set; } + + [JsonPropertyName("mediaType")] + public string MediaType { get; set; } + + [JsonPropertyName("config")] + public OciDescriptor Config { get; set; } + + [JsonPropertyName("layers")] + public List Layers { get; set; } + } +} diff --git a/src/BuildScriptGenerator/Helpers/OciRegistryClient.cs b/src/BuildScriptGenerator/Helpers/OciRegistryClient.cs new file mode 100644 index 0000000000..7d425e26b3 --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/OciRegistryClient.cs @@ -0,0 +1,290 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + /// + /// HTTP client for the OCI Distribution API. Enables Oryx to discover SDK versions + /// and download SDK tarballs from an OCI-compliant container registry (e.g. Azure Container Registry) + /// using only HttpClient — no external tools (docker, crane, oras) required. + /// All SDK images are public, so no authentication is needed. + /// + public class OciRegistryClient + { + private readonly HttpClient httpClient; + private readonly string registryUrl; + private readonly ILogger logger; + + public OciRegistryClient(string registryUrl, IHttpClientFactory httpClientFactory, ILoggerFactory loggerFactory) + { + this.registryUrl = registryUrl.TrimEnd('/'); + this.httpClient = httpClientFactory.CreateClient("general"); + this.logger = loggerFactory.CreateLogger(); + } + + /// + /// Gets the first layer digest from a manifest (SDK images are single-layer FROM scratch images). + /// + public static string GetFirstLayerDigest(OciManifest manifest) + { + return manifest?.Layers?.FirstOrDefault()?.Digest; + } + + /// + /// Lists all tags for a repository, handling Link-header pagination. + /// + public async Task> GetAllTagsAsync(string repository) + { + var allTags = new List(); + var url = $"{this.registryUrl}/v2/{repository}/tags/list"; + + while (!string.IsNullOrEmpty(url)) + { + this.logger.LogDebug("Fetching tags from {url}", url); + using (var response = await this.httpClient.GetAsync(url)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Request to {url} failed with status code {response.StatusCode}"); + } + + var json = await response.Content.ReadAsStringAsync(); + var tagList = JsonSerializer.Deserialize(json); + if (tagList?.Tags != null) + { + allTags.AddRange(tagList.Tags); + } + + // Handle OCI pagination via Link header (RFC 5988) + url = null; + if (response.Headers.TryGetValues("Link", out var linkValues)) + { + var linkHeader = linkValues.FirstOrDefault(); + if (linkHeader != null) + { + var match = Regex.Match(linkHeader, @"<([^>]+)>;\s*rel=""next"""); + if (match.Success) + { + url = match.Groups[1].Value; + if (!url.StartsWith("http")) + { + url = $"{this.registryUrl}{url}"; + } + } + } + } + } + } + + return allTags; + } + + /// + /// Fetches an OCI image manifest for the given repository and tag. + /// Tries OCI manifest format first, falls back to Docker manifest v2. + /// + public async Task GetManifestAsync(string repository, string tag) + { + var url = $"{this.registryUrl}/v2/{repository}/manifests/{tag}"; + using (var request = new HttpRequestMessage(HttpMethod.Get, url)) + { + request.Headers.Add("Accept", "application/vnd.oci.image.manifest.v1+json"); + + using (var response = await this.httpClient.SendAsync(request)) + { + if (!response.IsSuccessStatusCode) + { + // Fall back to Docker manifest v2 + using (var fallbackRequest = new HttpRequestMessage(HttpMethod.Get, url)) + { + fallbackRequest.Headers.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json"); + using (var fallbackResponse = await this.httpClient.SendAsync(fallbackRequest)) + { + if (!fallbackResponse.IsSuccessStatusCode) + { + throw new HttpRequestException($"Fallback request to {url} failed with status code {fallbackResponse.StatusCode}"); + } + + var fallbackJson = await fallbackResponse.Content.ReadAsStringAsync(); + return JsonSerializer.Deserialize(fallbackJson); + } + } + } + + var json = await response.Content.ReadAsStringAsync(); + return JsonSerializer.Deserialize(json); + } + } + } + + /// + /// Fetches the image config blob (contains Labels) for the given repository and digest. + /// + public async Task GetImageConfigAsync(string repository, string configDigest) + { + var url = $"{this.registryUrl}/v2/{repository}/blobs/{configDigest}"; + using (var response = await this.httpClient.GetAsync(url)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Request to {url} failed with status code {response.StatusCode}"); + } + + var json = await response.Content.ReadAsStringAsync(); + return JsonSerializer.Deserialize(json); + } + } + + /// + /// Gets the default version for a platform from the "-default" tag's image config labels. + /// + public async Task GetDefaultVersionAsync(string repository, string osFlavor) + { + var tag = $"{osFlavor}-default"; + this.logger.LogDebug("Fetching default version from {repository}:{tag}", repository, tag); + + try + { + var manifest = await this.GetManifestAsync(repository, tag); + var configDigest = manifest.Config?.Digest; + if (string.IsNullOrEmpty(configDigest)) + { + this.logger.LogWarning("No config digest found in manifest for {repository}:{tag}", repository, tag); + return null; + } + + var config = await this.GetImageConfigAsync(repository, configDigest); + if (config?.Config?.Labels != null && + config.Config.Labels.TryGetValue(Common.SdkStorageConstants.AcrVersionLabelName, out var version)) + { + return version; + } + + this.logger.LogWarning("Version label not found in config for {repository}:{tag}", repository, tag); + return null; + } + catch (Exception ex) + { + this.logger.LogError(ex, "Failed to get default version from {repository}:{tag}", repository, tag); + throw; + } + } + + /// + /// Downloads a layer blob (the SDK tarball) to disk and verifies its SHA256 digest. + /// The digest in the manifest IS the content hash — no separate checksum metadata needed. + /// + public async Task DownloadLayerBlobAsync(string repository, string layerDigest, string outputPath) + { + var url = $"{this.registryUrl}/v2/{repository}/blobs/{layerDigest}"; + this.logger.LogDebug("Downloading layer blob {digest} from {repository}", layerDigest, repository); + + using (var response = await this.httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Request to {url} failed with status code {response.StatusCode}"); + } + + using (var stream = await response.Content.ReadAsStreamAsync()) + using (var fileStream = File.Create(outputPath)) + { + await stream.CopyToAsync(fileStream); + } + } + + // Verify SHA256 digest + var expectedSha = layerDigest.StartsWith("sha256:") + ? layerDigest.Substring("sha256:".Length) + : layerDigest; + + using (var fileStream = File.OpenRead(outputPath)) + using (var sha256 = SHA256.Create()) + { + var hashBytes = sha256.ComputeHash(fileStream); + var actualSha = BitConverter.ToString(hashBytes).Replace("-", string.Empty).ToLowerInvariant(); + + if (!string.Equals(actualSha, expectedSha, StringComparison.OrdinalIgnoreCase)) + { + this.logger.LogError( + "SHA256 digest mismatch for {repository} blob {digest}. Expected: {expected}, Actual: {actual}", + repository, + layerDigest, + expectedSha, + actualSha); + File.Delete(outputPath); + return false; + } + } + + this.logger.LogDebug("Successfully downloaded and verified layer blob {digest}", layerDigest); + return true; + } + + /// + /// Pulls an SDK tarball from an OCI image built with FROM scratch; COPY sdk.tar.gz /. + /// Because the image contains a single layer, that layer IS the SDK tarball. + /// Flow: fetch manifest → extract single layer digest → download blob → verify SHA256. + /// + /// The repository name, e.g. "sdks/python". + /// The image tag, e.g. "bookworm-3.11.0". + /// The full path where the downloaded tarball should be saved. + /// True if the SDK was pulled and verified successfully. + public async Task PullSdkAsync(string repository, string tag, string outputFilePath) + { + this.logger.LogInformation( + "Pulling SDK directly from ACR: {repository}:{tag} -> {outputPath}", + repository, + tag, + outputFilePath); + + // Step 1: Fetch the OCI manifest + var manifest = await this.GetManifestAsync(repository, tag); + if (manifest == null) + { + this.logger.LogError("Failed to get manifest for {repository}:{tag}", repository, tag); + return false; + } + + // Step 2: Get the single layer digest (FROM scratch images have exactly 1 layer) + var layerDigest = GetFirstLayerDigest(manifest); + if (string.IsNullOrEmpty(layerDigest)) + { + this.logger.LogError( + "No layer found in manifest for {repository}:{tag}. Expected a single-layer FROM scratch image.", + repository, + tag); + return false; + } + + this.logger.LogDebug( + "Manifest for {repository}:{tag} has layer digest: {digest}", + repository, + tag, + layerDigest); + + // Ensure the output directory exists + var outputDir = Path.GetDirectoryName(outputFilePath); + if (!string.IsNullOrEmpty(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + + // Step 3: Download the layer blob (this IS the SDK tarball) and verify SHA256 + return await this.DownloadLayerBlobAsync(repository, layerDigest, outputFilePath); + } + } +} diff --git a/src/BuildScriptGenerator/Helpers/OciTagList.cs b/src/BuildScriptGenerator/Helpers/OciTagList.cs new file mode 100644 index 0000000000..261b23565b --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/OciTagList.cs @@ -0,0 +1,19 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + public class OciTagList + { + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("tags")] + public List Tags { get; set; } + } +} diff --git a/src/BuildScriptGenerator/Node/NodePlatform.cs b/src/BuildScriptGenerator/Node/NodePlatform.cs index 87cbed0817..059d5693f1 100644 --- a/src/BuildScriptGenerator/Node/NodePlatform.cs +++ b/src/BuildScriptGenerator/Node/NodePlatform.cs @@ -86,6 +86,7 @@ internal class NodePlatform : IProgrammingPlatform private readonly IEnvironment environment; private readonly NodePlatformInstaller platformInstaller; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IExternalAcrSdkProvider externalAcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -108,6 +109,7 @@ public NodePlatform( IEnvironment environment, NodePlatformInstaller nodePlatformInstaller, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) { this.commonOptions = commonOptions.Value; @@ -118,6 +120,7 @@ public NodePlatform( this.environment = environment; this.platformInstaller = nodePlatformInstaller; this.externalSdkProvider = externalSdkProvider; + this.externalAcrSdkProvider = externalAcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -493,74 +496,38 @@ public string GetInstallerScriptSnippet( BuildScriptGeneratorContext context, PlatformDetectorResult detectorResult) { - string installationScriptSnippet = null; - if (this.commonOptions.EnableDynamicInstall) + if (!this.commonOptions.EnableDynamicInstall) { - this.logger.LogDebug("Dynamic install is enabled."); + this.logger.LogDebug("Dynamic install not enabled."); + return null; + } - if (this.platformInstaller.IsVersionAlreadyInstalled(detectorResult.PlatformVersion)) - { - this.logger.LogDebug( - "Node version {version} is already installed. So skipping installing it again.", - detectorResult.PlatformVersion); - } - else - { - if (this.commonOptions.EnableExternalSdkProvider) - { - this.logger.LogDebug( - "Node version {version} is not installed. " + - "External SDK provider is enabled so trying to fetch SDK using it.", - detectorResult.PlatformVersion); - - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, detectorResult.PlatformVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug( - "Node version {version} is fetched successfully using external SDK provider. " + - "So generating an installation script snippet which skips platform binary download.", - detectorResult.PlatformVersion); - - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug( - "Node version {version} is not fetched successfully using external SDK provider. " + - "So generating an installation script snippet for it.", - detectorResult.PlatformVersion); - - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet( - detectorResult.PlatformVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching Node.js version {version} using external SDK provider.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } - } - else - { - this.logger.LogDebug( - "Node version {version} is not installed. " + - "So generating an installation script snippet for it.", - detectorResult.PlatformVersion); + this.logger.LogDebug("Dynamic install is enabled."); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet( - detectorResult.PlatformVersion); - } - } + var version = detectorResult.PlatformVersion; + + if (this.platformInstaller.IsVersionAlreadyInstalled(version)) + { + this.logger.LogDebug( + "Node version {version} is already installed. So skipping installing it again.", + version); + return null; } - else + + if (this.commonOptions.EnableExternalSdkProvider) { - this.logger.LogDebug("Dynamic install not enabled."); + return this.TryInstallFromExternalSdkProvider(version); } - return installationScriptSnippet; + if (this.commonOptions.EnableAcrSdkProvider) + { + return this.TryInstallFromAcrSdkProvider(version); + } + + this.logger.LogDebug( + "Node version {version} is not installed. So generating an installation script snippet for it.", + version); + return this.platformInstaller.GetInstallerScriptSnippet(version); } /// @@ -717,6 +684,70 @@ private static void GetAppOutputDirPath(dynamic packageJson, Dictionary(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); return services; } } diff --git a/src/BuildScriptGenerator/Node/VersionProviders/NodeAcrVersionProvider.cs b/src/BuildScriptGenerator/Node/VersionProviders/NodeAcrVersionProvider.cs new file mode 100644 index 0000000000..bf9b041c03 --- /dev/null +++ b/src/BuildScriptGenerator/Node/VersionProviders/NodeAcrVersionProvider.cs @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Oryx.BuildScriptGenerator.Node +{ + /// + /// ACR-based version provider for Node.js SDKs. + /// Parallel to but uses OCI Distribution API. + /// + internal class NodeAcrVersionProvider : AcrVersionProviderBase, INodeVersionProvider + { + private PlatformVersionInfo platformVersionInfo; + + public NodeAcrVersionProvider( + IOptions commonOptions, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + : base(commonOptions, httpClientFactory, loggerFactory) + { + } + + public virtual PlatformVersionInfo GetVersionInfo() + { + return this.platformVersionInfo + ??= this.GetAvailableVersionsFromAcr(platformName: "nodejs"); + } + } +} diff --git a/src/BuildScriptGenerator/Node/VersionProviders/NodeVersionProvider.cs b/src/BuildScriptGenerator/Node/VersionProviders/NodeVersionProvider.cs index 1005176d50..e3e8ff415f 100644 --- a/src/BuildScriptGenerator/Node/VersionProviders/NodeVersionProvider.cs +++ b/src/BuildScriptGenerator/Node/VersionProviders/NodeVersionProvider.cs @@ -15,6 +15,7 @@ internal class NodeVersionProvider : INodeVersionProvider private readonly NodeOnDiskVersionProvider onDiskVersionProvider; private readonly NodeSdkStorageVersionProvider sdkStorageVersionProvider; private readonly NodeExternalVersionProvider externalVersionProvider; + private readonly NodeAcrVersionProvider acrVersionProvider; private readonly ILogger logger; private PlatformVersionInfo versionInfo; @@ -23,40 +24,60 @@ public NodeVersionProvider( NodeOnDiskVersionProvider onDiskVersionProvider, NodeSdkStorageVersionProvider sdkStorageVersionProvider, NodeExternalVersionProvider externalVersionProvider, + NodeAcrVersionProvider acrVersionProvider, ILogger logger) { this.options = options.Value; this.onDiskVersionProvider = onDiskVersionProvider; this.sdkStorageVersionProvider = sdkStorageVersionProvider; this.externalVersionProvider = externalVersionProvider; + this.acrVersionProvider = acrVersionProvider; this.logger = logger; } public PlatformVersionInfo GetVersionInfo() { - if (this.versionInfo == null) + if (this.versionInfo != null) { - if (this.options.EnableDynamicInstall) + return this.versionInfo; + } + + this.versionInfo = this.options.EnableDynamicInstall + ? this.ResolveDynamicVersionInfo() + : this.onDiskVersionProvider.GetVersionInfo(); + + return this.versionInfo; + } + + private PlatformVersionInfo ResolveDynamicVersionInfo() + { + if (this.options.EnableExternalSdkProvider) + { + try { - if (this.options.EnableExternalSdkProvider) - { - try - { - return this.externalVersionProvider.GetVersionInfo(); - } - catch (Exception ex) - { - this.logger.LogError($"Failed to get version info from external SDK provider. Falling back to http based sdkStorageVersionProvider. Ex: {ex}"); - } - } - - return this.sdkStorageVersionProvider.GetVersionInfo(); + return this.externalVersionProvider.GetVersionInfo(); } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from external SDK provider. Falling back. Ex: {ex}"); + } + } - this.versionInfo = this.onDiskVersionProvider.GetVersionInfo(); + if (this.options.EnableAcrSdkProvider) + { + try + { + return this.acrVersionProvider.GetVersionInfo(); + } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from ACR provider. Falling back to blob storage. Ex: {ex}"); + } } - return this.versionInfo; + return this.sdkStorageVersionProvider.GetVersionInfo(); } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs b/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs index e0c51dd5fd..6e191b7bcd 100644 --- a/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs +++ b/src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs @@ -98,5 +98,17 @@ public class BuildScriptGeneratorOptions public string ImageType { get; set; } public bool OryxDisablePipUpgrade { get; set; } + + /// + /// Gets or sets a value indicating whether ACR-based SDK provider is enabled. + /// When true, Oryx will discover and download SDKs from an OCI-compliant container registry. + /// + public bool EnableAcrSdkProvider { get; set; } + + /// + /// Gets or sets the base URL of the ACR registry hosting SDK images. + /// e.g. "https://oryxsdks.azurecr.io" + /// + public string OryxAcrSdkRegistryUrl { get; set; } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/Php/PhpPlatform.cs b/src/BuildScriptGenerator/Php/PhpPlatform.cs index 2a189a077c..e3712dea21 100644 --- a/src/BuildScriptGenerator/Php/PhpPlatform.cs +++ b/src/BuildScriptGenerator/Php/PhpPlatform.cs @@ -34,6 +34,7 @@ internal class PhpPlatform : IProgrammingPlatform private readonly PhpPlatformInstaller phpInstaller; private readonly PhpComposerInstaller phpComposerInstaller; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IExternalAcrSdkProvider externalAcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -57,6 +58,7 @@ public PhpPlatform( PhpPlatformInstaller phpInstaller, PhpComposerInstaller phpComposerInstaller, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) { this.phpScriptGeneratorOptions = phpScriptGeneratorOptions.Value; @@ -68,6 +70,7 @@ public PhpPlatform( this.phpInstaller = phpInstaller; this.phpComposerInstaller = phpComposerInstaller; this.externalSdkProvider = externalSdkProvider; + this.externalAcrSdkProvider = externalAcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -226,28 +229,33 @@ public string GetInstallerScriptSnippet( $"'{typeof(PhpPlatformDetectorResult)}' but got '{detectorResult.GetType()}'."); } - if (this.commonOptions.EnableDynamicInstall) + if (!this.commonOptions.EnableDynamicInstall) { - this.logger.LogDebug("Dynamic install is enabled."); + this.logger.LogDebug("Dynamic install not enabled."); + return null; + } - var scriptBuilder = new StringBuilder(); + this.logger.LogDebug("Dynamic install is enabled."); - this.InstallPhp(phpPlatformDetectorResult.PlatformVersion, scriptBuilder); + var scriptBuilder = new StringBuilder(); + if (this.commonOptions.EnableExternalSdkProvider) + { + this.InstallPhp(phpPlatformDetectorResult.PlatformVersion, scriptBuilder); this.InstallPhpComposer(phpPlatformDetectorResult.PhpComposerVersion, scriptBuilder); - - if (scriptBuilder.Length == 0) - { - return null; - } - - return scriptBuilder.ToString(); + } + else if (this.commonOptions.EnableAcrSdkProvider) + { + this.InstallPhpAcr(phpPlatformDetectorResult.PlatformVersion, scriptBuilder); + this.InstallPhpComposerAcr(phpPlatformDetectorResult.PhpComposerVersion, scriptBuilder); } else { - this.logger.LogDebug("Dynamic install not enabled."); - return null; + this.InstallPhp(phpPlatformDetectorResult.PlatformVersion, scriptBuilder); + this.InstallPhpComposer(phpPlatformDetectorResult.PhpComposerVersion, scriptBuilder); } + + return scriptBuilder.Length == 0 ? null : scriptBuilder.ToString(); } /// @@ -293,54 +301,43 @@ public string GetMaxSatisfyingPhpComposerVersionAndVerify(string version) private void InstallPhp(string phpVersion, StringBuilder scriptBuilder) { - string script = null; if (this.phpInstaller.IsVersionAlreadyInstalled(phpVersion)) { this.logger.LogDebug("PHP version {version} is already installed. So skipping installing it again.", phpVersion); return; } - else + + if (this.commonOptions.EnableExternalSdkProvider) { - if (this.commonOptions.EnableExternalSdkProvider) - { - this.logger.LogDebug("Php version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", phpVersion); + this.logger.LogDebug("Php version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", phpVersion); - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion("php", phpVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("Php version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", phpVersion); - - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("Php version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", phpVersion); - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion); - } - } - catch (Exception ex) + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion("php", phpVersion, this.commonOptions.DebianFlavor); + if (this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result) { - this.logger.LogError(ex, "Error while fetching php version {version} using external SDK provider.", phpVersion); - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion); + this.logger.LogDebug("Php version {version} is fetched successfully using external SDK provider. Skipping platform binary download.", phpVersion); + scriptBuilder.AppendLine(this.phpInstaller.GetInstallerScriptSnippet(phpVersion, skipSdkBinaryDownload: true)); + return; } + + this.logger.LogDebug("Php version {version} is not fetched successfully using external SDK provider. Generating installation script snippet.", phpVersion); } - else + catch (Exception ex) { - this.logger.LogDebug("Php version {version} is not installed. So generating an installation script snippet for it.", phpVersion); - script = this.phpInstaller.GetInstallerScriptSnippet(phpVersion); + this.logger.LogError(ex, "Error while fetching php version {version} using external SDK provider.", phpVersion); } - - scriptBuilder.AppendLine(script); } + else + { + this.logger.LogDebug("Php version {version} is not installed. So generating an installation script snippet for it.", phpVersion); + } + + scriptBuilder.AppendLine(this.phpInstaller.GetInstallerScriptSnippet(phpVersion)); } private void InstallPhpComposer(string phpComposerVersion, StringBuilder scriptBuilder) { - // Install PHP Composer - string script = null; if (string.IsNullOrEmpty(phpComposerVersion)) { phpComposerVersion = PhpVersions.ComposerDefaultVersion; @@ -351,42 +348,99 @@ private void InstallPhpComposer(string phpComposerVersion, StringBuilder scriptB this.logger.LogDebug("PHP Composer version {version} is already installed. So skipping installing it again.", phpComposerVersion); return; } - else + + if (this.commonOptions.EnableExternalSdkProvider) { - if (this.commonOptions.EnableExternalSdkProvider) - { - this.logger.LogDebug("Php Composer version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", phpComposerVersion); + this.logger.LogDebug("Php Composer version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", phpComposerVersion); - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion("php-composer", phpComposerVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync("php-composer", blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("Php composer version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", phpComposerVersion); - - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("Php comose version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", phpComposerVersion); - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion); - } - } - catch (Exception ex) + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion("php-composer", phpComposerVersion, this.commonOptions.DebianFlavor); + if (this.externalSdkProvider.RequestBlobAsync("php-composer", blobName).Result) { - this.logger.LogError(ex, "Error while fetching php composer version {version} using external SDK provider.", phpComposerVersion); - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion); + this.logger.LogDebug("Php composer version {version} is fetched successfully using external SDK provider. Skipping platform binary download.", phpComposerVersion); + scriptBuilder.AppendLine(this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion, skipSdkBinaryDownload: true)); + return; } + + this.logger.LogDebug("Php comose version {version} is not fetched successfully using external SDK provider. Generating installation script snippet.", phpComposerVersion); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching php composer version {version} using external SDK provider.", phpComposerVersion); + } + } + else + { + this.logger.LogDebug("Php composer version {version} is not installed. So generating an installation script snippet for it.", phpComposerVersion); + } + + scriptBuilder.AppendLine(this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion)); + } + + private void InstallPhpAcr(string phpVersion, StringBuilder scriptBuilder) + { + if (this.phpInstaller.IsVersionAlreadyInstalled(phpVersion)) + { + this.logger.LogDebug("PHP version {version} is already installed. So skipping installing it again.", phpVersion); + return; + } + + this.logger.LogDebug("PHP version {version} is not installed. ACR SDK provider is enabled, so trying to fetch SDK using it.", phpVersion); + + try + { + if (this.externalAcrSdkProvider.RequestSdkFromAcrAsync( + "php", phpVersion, this.commonOptions.DebianFlavor).Result) + { + this.logger.LogDebug("PHP version {version} is fetched successfully using ACR SDK provider. Skipping platform binary download.", phpVersion); + scriptBuilder.AppendLine(this.phpInstaller.GetInstallerScriptSnippet(phpVersion, skipSdkBinaryDownload: true)); + return; } - else + + this.logger.LogDebug("PHP version {version} is not fetched via ACR SDK provider. Falling back to CDN download.", phpVersion); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching PHP version {version} using ACR SDK provider. Falling back to CDN download.", phpVersion); + } + + scriptBuilder.AppendLine(this.phpInstaller.GetInstallerScriptSnippet(phpVersion)); + } + + private void InstallPhpComposerAcr(string phpComposerVersion, StringBuilder scriptBuilder) + { + if (string.IsNullOrEmpty(phpComposerVersion)) + { + phpComposerVersion = PhpVersions.ComposerDefaultVersion; + } + + if (this.phpComposerInstaller.IsVersionAlreadyInstalled(phpComposerVersion)) + { + this.logger.LogDebug("PHP Composer version {version} is already installed. So skipping installing it again.", phpComposerVersion); + return; + } + + this.logger.LogDebug("PHP Composer version {version} is not installed. ACR SDK provider is enabled, so trying to fetch SDK using it.", phpComposerVersion); + + try + { + if (this.externalAcrSdkProvider.RequestSdkFromAcrAsync( + "php-composer", phpComposerVersion, this.commonOptions.DebianFlavor).Result) { - this.logger.LogDebug("Php composer version {version} is not installed. So generating an installation script snippet for it.", phpComposerVersion); - script = this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion); + this.logger.LogDebug("PHP Composer version {version} is fetched successfully using ACR SDK provider. Skipping platform binary download.", phpComposerVersion); + scriptBuilder.AppendLine(this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion, skipSdkBinaryDownload: true)); + return; } + + this.logger.LogDebug("PHP Composer version {version} is not fetched via ACR SDK provider. Falling back to CDN download.", phpComposerVersion); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error while fetching PHP Composer version {version} using ACR SDK provider. Falling back to CDN download.", phpComposerVersion); } - scriptBuilder.AppendLine(script); + scriptBuilder.AppendLine(this.phpComposerInstaller.GetInstallerScriptSnippet(phpComposerVersion)); } private void ResolveVersionsUsingHierarchicalRules(PhpPlatformDetectorResult detectorResult) diff --git a/src/BuildScriptGenerator/Php/PhpScriptGeneratorServiceCollectionExtensions.cs b/src/BuildScriptGenerator/Php/PhpScriptGeneratorServiceCollectionExtensions.cs index fe96ac23f8..b969f66384 100644 --- a/src/BuildScriptGenerator/Php/PhpScriptGeneratorServiceCollectionExtensions.cs +++ b/src/BuildScriptGenerator/Php/PhpScriptGeneratorServiceCollectionExtensions.cs @@ -24,6 +24,8 @@ public static IServiceCollection AddPhpScriptGeneratorServices(this IServiceColl services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); return services; } } diff --git a/src/BuildScriptGenerator/Php/VersionProviders/PhpAcrVersionProvider.cs b/src/BuildScriptGenerator/Php/VersionProviders/PhpAcrVersionProvider.cs new file mode 100644 index 0000000000..5e1bcc200b --- /dev/null +++ b/src/BuildScriptGenerator/Php/VersionProviders/PhpAcrVersionProvider.cs @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Oryx.BuildScriptGenerator.Php +{ + /// + /// ACR-based version provider for PHP SDKs. + /// Parallel to but uses OCI Distribution API. + /// + internal class PhpAcrVersionProvider : AcrVersionProviderBase, IPhpVersionProvider + { + private PlatformVersionInfo platformVersionInfo; + + public PhpAcrVersionProvider( + IOptions commonOptions, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + : base(commonOptions, httpClientFactory, loggerFactory) + { + } + + public virtual PlatformVersionInfo GetVersionInfo() + { + return this.platformVersionInfo + ??= this.GetAvailableVersionsFromAcr(platformName: ToolNameConstants.PhpName); + } + } +} diff --git a/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerAcrVersionProvider.cs b/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerAcrVersionProvider.cs new file mode 100644 index 0000000000..e23e0f8c96 --- /dev/null +++ b/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerAcrVersionProvider.cs @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Oryx.BuildScriptGenerator.Php +{ + /// + /// ACR-based version provider for PHP Composer SDKs. + /// Parallel to but uses OCI Distribution API. + /// + internal class PhpComposerAcrVersionProvider : AcrVersionProviderBase, IPhpComposerVersionProvider + { + private PlatformVersionInfo platformVersionInfo; + + public PhpComposerAcrVersionProvider( + IOptions commonOptions, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + : base(commonOptions, httpClientFactory, loggerFactory) + { + } + + public virtual PlatformVersionInfo GetVersionInfo() + { + return this.platformVersionInfo + ??= this.GetAvailableVersionsFromAcr(platformName: "php-composer"); + } + } +} diff --git a/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerVersionProvider.cs b/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerVersionProvider.cs index dd2334be23..42cbd3b170 100644 --- a/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerVersionProvider.cs +++ b/src/BuildScriptGenerator/Php/VersionProviders/PhpComposerVersionProvider.cs @@ -15,6 +15,7 @@ internal class PhpComposerVersionProvider : IPhpComposerVersionProvider private readonly PhpComposerOnDiskVersionProvider onDiskVersionProvider; private readonly PhpComposerSdkStorageVersionProvider sdkStorageVersionProvider; private readonly PhpComposerExternalVersionProvider externalVersionProvider; + private readonly PhpComposerAcrVersionProvider acrVersionProvider; private readonly ILogger logger; private PlatformVersionInfo versionInfo; @@ -23,40 +24,60 @@ public PhpComposerVersionProvider( PhpComposerOnDiskVersionProvider onDiskVersionProvider, PhpComposerSdkStorageVersionProvider sdkStorageVersionProvider, PhpComposerExternalVersionProvider externalVersionProvider, + PhpComposerAcrVersionProvider acrVersionProvider, ILogger logger) { this.options = options.Value; this.onDiskVersionProvider = onDiskVersionProvider; this.sdkStorageVersionProvider = sdkStorageVersionProvider; this.externalVersionProvider = externalVersionProvider; + this.acrVersionProvider = acrVersionProvider; this.logger = logger; } public PlatformVersionInfo GetVersionInfo() { - if (this.versionInfo == null) + if (this.versionInfo != null) { - if (this.options.EnableDynamicInstall) + return this.versionInfo; + } + + this.versionInfo = this.options.EnableDynamicInstall + ? this.ResolveDynamicVersionInfo() + : this.onDiskVersionProvider.GetVersionInfo(); + + return this.versionInfo; + } + + private PlatformVersionInfo ResolveDynamicVersionInfo() + { + if (this.options.EnableExternalSdkProvider) + { + try { - if (this.options.EnableExternalSdkProvider) - { - try - { - return this.externalVersionProvider.GetVersionInfo(); - } - catch (Exception ex) - { - this.logger.LogError($"Failed to get version info from external SDK provider. Falling back to http based sdkStorageVersionProvider. Ex: {ex}"); - } - } - - return this.sdkStorageVersionProvider.GetVersionInfo(); + return this.externalVersionProvider.GetVersionInfo(); } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from external SDK provider. Falling back. Ex: {ex}"); + } + } - this.versionInfo = this.onDiskVersionProvider.GetVersionInfo(); + if (this.options.EnableAcrSdkProvider) + { + try + { + return this.acrVersionProvider.GetVersionInfo(); + } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from ACR provider. Falling back to blob storage. Ex: {ex}"); + } } - return this.versionInfo; + return this.sdkStorageVersionProvider.GetVersionInfo(); } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/Php/VersionProviders/PhpVersionProvider.cs b/src/BuildScriptGenerator/Php/VersionProviders/PhpVersionProvider.cs index 991371670d..c4eba3b556 100644 --- a/src/BuildScriptGenerator/Php/VersionProviders/PhpVersionProvider.cs +++ b/src/BuildScriptGenerator/Php/VersionProviders/PhpVersionProvider.cs @@ -15,6 +15,7 @@ internal class PhpVersionProvider : IPhpVersionProvider private readonly PhpOnDiskVersionProvider onDiskVersionProvider; private readonly PhpSdkStorageVersionProvider sdkStorageVersionProvider; private readonly PhpExternalVersionProvider externalVersionProvider; + private readonly PhpAcrVersionProvider acrVersionProvider; private readonly ILogger logger; private PlatformVersionInfo versionInfo; @@ -23,40 +24,60 @@ public PhpVersionProvider( PhpOnDiskVersionProvider onDiskVersionProvider, PhpSdkStorageVersionProvider sdkStorageVersionProvider, PhpExternalVersionProvider externalVersionProvider, + PhpAcrVersionProvider acrVersionProvider, ILogger logger) { this.options = options.Value; this.onDiskVersionProvider = onDiskVersionProvider; this.sdkStorageVersionProvider = sdkStorageVersionProvider; this.externalVersionProvider = externalVersionProvider; + this.acrVersionProvider = acrVersionProvider; this.logger = logger; } public PlatformVersionInfo GetVersionInfo() { - if (this.versionInfo == null) + if (this.versionInfo != null) { - if (this.options.EnableDynamicInstall) + return this.versionInfo; + } + + this.versionInfo = this.options.EnableDynamicInstall + ? this.ResolveDynamicVersionInfo() + : this.onDiskVersionProvider.GetVersionInfo(); + + return this.versionInfo; + } + + private PlatformVersionInfo ResolveDynamicVersionInfo() + { + if (this.options.EnableExternalSdkProvider) + { + try { - if (this.options.EnableExternalSdkProvider) - { - try - { - return this.externalVersionProvider.GetVersionInfo(); - } - catch (Exception ex) - { - this.logger.LogError($"Failed to get version info from external SDK provider. Falling back to http based sdkStorageVersionProvider. Ex: {ex}"); - } - } - - return this.sdkStorageVersionProvider.GetVersionInfo(); + return this.externalVersionProvider.GetVersionInfo(); } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from external SDK provider. Falling back. Ex: {ex}"); + } + } - this.versionInfo = this.onDiskVersionProvider.GetVersionInfo(); + if (this.options.EnableAcrSdkProvider) + { + try + { + return this.acrVersionProvider.GetVersionInfo(); + } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from ACR provider. Falling back to blob storage. Ex: {ex}"); + } } - return this.versionInfo; + return this.sdkStorageVersionProvider.GetVersionInfo(); } } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/PlatformVersionInfo.cs b/src/BuildScriptGenerator/PlatformVersionInfo.cs index fb40ecc3b0..5a1b1700fb 100644 --- a/src/BuildScriptGenerator/PlatformVersionInfo.cs +++ b/src/BuildScriptGenerator/PlatformVersionInfo.cs @@ -49,5 +49,17 @@ public static PlatformVersionInfo CreateAvailableViaExternalProvider( PlatformVersionSourceType = PlatformVersionSourceType.AvailableViaExternalProvider, }; } + + public static PlatformVersionInfo CreateAvailableOnAcr( + IEnumerable supportedVersions, + string defaultVersion) + { + return new PlatformVersionInfo + { + SupportedVersions = supportedVersions, + DefaultVersion = defaultVersion, + PlatformVersionSourceType = PlatformVersionSourceType.AvailableOnAcr, + }; + } } } diff --git a/src/BuildScriptGenerator/PlatformVersionSourceType.cs b/src/BuildScriptGenerator/PlatformVersionSourceType.cs index 8137a275b9..65ac237b04 100644 --- a/src/BuildScriptGenerator/PlatformVersionSourceType.cs +++ b/src/BuildScriptGenerator/PlatformVersionSourceType.cs @@ -10,5 +10,6 @@ public enum PlatformVersionSourceType OnDisk, AvailableOnWeb, AvailableViaExternalProvider, + AvailableOnAcr, } } diff --git a/src/BuildScriptGenerator/Python/PythonPlatform.cs b/src/BuildScriptGenerator/Python/PythonPlatform.cs index 071b9fe8cd..6239d331e0 100644 --- a/src/BuildScriptGenerator/Python/PythonPlatform.cs +++ b/src/BuildScriptGenerator/Python/PythonPlatform.cs @@ -89,6 +89,7 @@ internal class PythonPlatform : IProgrammingPlatform private readonly IPythonPlatformDetector detector; private readonly PythonPlatformInstaller platformInstaller; private readonly IExternalSdkProvider externalSdkProvider; + private readonly IExternalAcrSdkProvider externalAcrSdkProvider; private readonly TelemetryClient telemetryClient; /// @@ -108,6 +109,7 @@ public PythonPlatform( IPythonPlatformDetector detector, PythonPlatformInstaller platformInstaller, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) { this.commonOptions = commonOptions.Value; @@ -117,6 +119,7 @@ public PythonPlatform( this.detector = detector; this.platformInstaller = platformInstaller; this.externalSdkProvider = externalSdkProvider; + this.externalAcrSdkProvider = externalAcrSdkProvider; this.telemetryClient = telemetryClient; } @@ -382,57 +385,38 @@ public string GetInstallerScriptSnippet( return null; } - string installationScriptSnippet = null; - if (this.commonOptions.EnableDynamicInstall) + if (!this.commonOptions.EnableDynamicInstall) { - this.logger.LogDebug("Dynamic install is enabled."); + this.logger.LogDebug("Dynamic install not enabled."); + return null; + } - if (this.platformInstaller.IsVersionAlreadyInstalled(detectorResult.PlatformVersion)) - { - this.logger.LogDebug( - "Python version {version} is already installed. So skipping installing it again.", - detectorResult.PlatformVersion); - } - else - { - if (this.commonOptions.EnableExternalSdkProvider) - { - this.logger.LogDebug("Python version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", detectorResult.PlatformVersion); - - try - { - var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, detectorResult.PlatformVersion, this.commonOptions.DebianFlavor); - var isExternalFetchSuccess = this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result; - if (isExternalFetchSuccess) - { - this.logger.LogDebug("Python version {version} is fetched successfully using external SDK provider. So generating an installation script snippet which skips platform binary download.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion, skipSdkBinaryDownload: true); - } - else - { - this.logger.LogDebug("Python version {version} is not fetched successfully using external SDK provider. So generating an installation script snippet for it.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error while fetching python version {version} using external SDK provider.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } - } - else - { - this.logger.LogDebug("Python version {version} is not installed. So generating an installation script snippet for it.", detectorResult.PlatformVersion); - installationScriptSnippet = this.platformInstaller.GetInstallerScriptSnippet(detectorResult.PlatformVersion); - } - } + this.logger.LogDebug("Dynamic install is enabled."); + + if (this.platformInstaller.IsVersionAlreadyInstalled(detectorResult.PlatformVersion)) + { + this.logger.LogDebug( + "Python version {version} is already installed. So skipping installing it again.", + detectorResult.PlatformVersion); + return null; } - else + + var version = detectorResult.PlatformVersion; + + if (this.commonOptions.EnableExternalSdkProvider) { - this.logger.LogDebug("Dynamic install not enabled."); + return this.TryInstallFromExternalSdkProvider(version); + } + + if (this.commonOptions.EnableAcrSdkProvider) + { + return this.TryInstallFromAcrSdkProvider(version); } - return installationScriptSnippet; + this.logger.LogDebug( + "Python version {version} is not installed. So generating an installation script snippet for it.", + version); + return this.platformInstaller.GetInstallerScriptSnippet(version); } /// @@ -565,6 +549,69 @@ private static bool IsCondaInstalledInImage() return File.Exists(PythonConstants.CondaExecutablePath); } + private string TryInstallFromExternalSdkProvider(string version) + { + this.logger.LogDebug( + "Python version {version} is not installed. External SDK provider is enabled so trying to fetch SDK using it.", + version); + + try + { + var blobName = BlobNameHelper.GetBlobNameForVersion(this.Name, version, this.commonOptions.DebianFlavor); + if (this.externalSdkProvider.RequestBlobAsync(this.Name, blobName).Result) + { + this.logger.LogDebug( + "Python version {version} is fetched successfully using external SDK provider. Skipping platform binary download.", + version); + return this.platformInstaller.GetInstallerScriptSnippet(version, skipSdkBinaryDownload: true); + } + + this.logger.LogDebug( + "Python version {version} is not fetched successfully using external SDK provider. Generating installation script snippet.", + version); + } + catch (Exception ex) + { + this.logger.LogError( + ex, + "Error while fetching python version {version} using external SDK provider.", + version); + } + + return this.platformInstaller.GetInstallerScriptSnippet(version); + } + + private string TryInstallFromAcrSdkProvider(string version) + { + this.logger.LogDebug( + "Python version {version} is not installed. ACR SDK provider is enabled, so trying to fetch SDK using it.", + version); + + try + { + if (this.externalAcrSdkProvider.RequestSdkFromAcrAsync(this.Name, version, this.commonOptions.DebianFlavor).Result) + { + this.logger.LogDebug( + "Python version {version} is fetched successfully using ACR SDK provider. Skipping platform binary download.", + version); + return this.platformInstaller.GetInstallerScriptSnippet(version, skipSdkBinaryDownload: true); + } + + this.logger.LogDebug( + "Python version {version} is not fetched via ACR SDK provider. Falling back to CDN download.", + version); + } + catch (Exception ex) + { + this.logger.LogError( + ex, + "Error while fetching python version {version} using ACR SDK provider. Falling back to CDN download.", + version); + } + + return this.platformInstaller.GetInstallerScriptSnippet(version); + } + private BuildScriptSnippet GetBuildScriptSnippetForConda( BuildScriptGeneratorContext context, PythonPlatformDetectorResult detectorResult) diff --git a/src/BuildScriptGenerator/Python/PythonScriptGeneratorServiceCollectionExtensions.cs b/src/BuildScriptGenerator/Python/PythonScriptGeneratorServiceCollectionExtensions.cs index b767f48663..cff7254538 100644 --- a/src/BuildScriptGenerator/Python/PythonScriptGeneratorServiceCollectionExtensions.cs +++ b/src/BuildScriptGenerator/Python/PythonScriptGeneratorServiceCollectionExtensions.cs @@ -20,6 +20,7 @@ public static IServiceCollection AddPythonScriptGeneratorServices(this IServiceC services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); return services; } } diff --git a/src/BuildScriptGenerator/Python/VersionProviders/PythonAcrVersionProvider.cs b/src/BuildScriptGenerator/Python/VersionProviders/PythonAcrVersionProvider.cs new file mode 100644 index 0000000000..83b28fd6e3 --- /dev/null +++ b/src/BuildScriptGenerator/Python/VersionProviders/PythonAcrVersionProvider.cs @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Oryx.BuildScriptGenerator.Python +{ + /// + /// ACR-based version provider for Python SDKs. + /// Parallel to but uses OCI Distribution API. + /// + internal class PythonAcrVersionProvider : AcrVersionProviderBase, IPythonVersionProvider + { + private PlatformVersionInfo platformVersionInfo; + + public PythonAcrVersionProvider( + IOptions commonOptions, + IHttpClientFactory httpClientFactory, + ILoggerFactory loggerFactory) + : base(commonOptions, httpClientFactory, loggerFactory) + { + } + + public virtual PlatformVersionInfo GetVersionInfo() + { + return this.platformVersionInfo + ??= this.GetAvailableVersionsFromAcr(platformName: ToolNameConstants.PythonName); + } + } +} diff --git a/src/BuildScriptGenerator/Python/VersionProviders/PythonVersionProvider.cs b/src/BuildScriptGenerator/Python/VersionProviders/PythonVersionProvider.cs index b6a268e86e..c8cbb375ce 100644 --- a/src/BuildScriptGenerator/Python/VersionProviders/PythonVersionProvider.cs +++ b/src/BuildScriptGenerator/Python/VersionProviders/PythonVersionProvider.cs @@ -15,6 +15,7 @@ internal class PythonVersionProvider : IPythonVersionProvider private readonly PythonOnDiskVersionProvider onDiskVersionProvider; private readonly PythonSdkStorageVersionProvider sdkStorageVersionProvider; private readonly PythonExternalVersionProvider externalVersionProvider; + private readonly PythonAcrVersionProvider acrVersionProvider; private readonly ILogger logger; private PlatformVersionInfo versionInfo; @@ -23,40 +24,60 @@ public PythonVersionProvider( PythonOnDiskVersionProvider onDiskVersionProvider, PythonSdkStorageVersionProvider sdkStorageVersionProvider, PythonExternalVersionProvider externalVersionProvider, + PythonAcrVersionProvider acrVersionProvider, ILogger logger) { this.options = options.Value; this.onDiskVersionProvider = onDiskVersionProvider; this.sdkStorageVersionProvider = sdkStorageVersionProvider; this.externalVersionProvider = externalVersionProvider; + this.acrVersionProvider = acrVersionProvider; this.logger = logger; } public PlatformVersionInfo GetVersionInfo() { - if (this.versionInfo == null) + if (this.versionInfo != null) { - if (this.options.EnableDynamicInstall) + return this.versionInfo; + } + + this.versionInfo = this.options.EnableDynamicInstall + ? this.ResolveDynamicVersionInfo() + : this.onDiskVersionProvider.GetVersionInfo(); + + return this.versionInfo; + } + + private PlatformVersionInfo ResolveDynamicVersionInfo() + { + if (this.options.EnableExternalSdkProvider) + { + try { - if (this.options.EnableExternalSdkProvider) - { - try - { - return this.externalVersionProvider.GetVersionInfo(); - } - catch (Exception ex) - { - this.logger.LogError($"Failed to get version info from external SDK provider. Falling back to http based sdkStorageVersionProvider. Ex: {ex}"); - } - } - - return this.sdkStorageVersionProvider.GetVersionInfo(); + return this.externalVersionProvider.GetVersionInfo(); } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from external SDK provider. Falling back. Ex: {ex}"); + } + } - this.versionInfo = this.onDiskVersionProvider.GetVersionInfo(); + if (this.options.EnableAcrSdkProvider) + { + try + { + return this.acrVersionProvider.GetVersionInfo(); + } + catch (Exception ex) + { + this.logger.LogError( + $"Failed to get version info from ACR provider. Falling back to blob storage. Ex: {ex}"); + } } - return this.versionInfo; + return this.sdkStorageVersionProvider.GetVersionInfo(); } } } \ No newline at end of file diff --git a/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs b/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs index f3313c7c7e..c46cc174fa 100644 --- a/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs +++ b/src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs @@ -80,6 +80,10 @@ public void Configure(BuildScriptGeneratorLib.BuildScriptGeneratorOptions option options.OsFlavor = this.GetStringValue(SettingsKeys.OsFlavor); options.DebianFlavor = this.GetStringValue(SettingsKeys.DebianFlavor); + + // ACR-based SDK provider + options.EnableAcrSdkProvider = this.GetBooleanValue(SettingsKeys.EnableAcrSdkProvider); + options.OryxAcrSdkRegistryUrl = this.GetStringValue(SettingsKeys.OryxAcrSdkRegistryUrl); } } } diff --git a/src/BuildScriptGeneratorCli/SettingsKeys.cs b/src/BuildScriptGeneratorCli/SettingsKeys.cs index b478745127..b04916f284 100644 --- a/src/BuildScriptGeneratorCli/SettingsKeys.cs +++ b/src/BuildScriptGeneratorCli/SettingsKeys.cs @@ -75,5 +75,7 @@ public static class SettingsKeys public const string DebianFlavor = "DEBIAN_FLAVOR"; public const string CallerId = "CALLER_ID"; public const string OryxDisablePipUpgrade = "ORYX_DISABLE_PIP_UPGRADE"; + public const string EnableAcrSdkProvider = "ORYX_ENABLE_ACR_SDK_PROVIDER"; + public const string OryxAcrSdkRegistryUrl = "ORYX_ACR_SDK_REGISTRY_URL"; } } diff --git a/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go b/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go index 93b1e61f88..40daf390ea 100644 --- a/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go +++ b/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go @@ -18,3 +18,9 @@ const LegacySdkVersionMetadataName string = "Version" const DotnetRuntimeVersionMetadataName string = "Dotnet_runtime_version" const LegacyDotnetRuntimeVersionMetadataName string = "Runtime_version" const OsTypeMetadataName string = "Os_type" + +// ACR-based SDK distribution constants +const EnableAcrSdkProviderKey string = "ORYX_ENABLE_ACR_SDK_PROVIDER" +const AcrSdkRegistryUrlKeyName string = "ORYX_ACR_SDK_REGISTRY_URL" +const DefaultAcrSdkRegistryUrl string = "https://oryxacr.azurecr.io" +const AcrSdkRepositoryPrefix string = "sdks" diff --git a/src/startupscriptgenerator/src/common/setupEnvironment.go b/src/startupscriptgenerator/src/common/setupEnvironment.go index e616ca8298..131da4448f 100644 --- a/src/startupscriptgenerator/src/common/setupEnvironment.go +++ b/src/startupscriptgenerator/src/common/setupEnvironment.go @@ -38,6 +38,12 @@ func GetSetupScript(platformName string, version string, installationDir string) return "" } + // Check if ACR SDK provider is enabled + enableAcrSdkProvider := os.Getenv(consts.EnableAcrSdkProviderKey) + if strings.EqualFold(enableAcrSdkProvider, "true") || enableAcrSdkProvider == "1" { + return GetAcrSetupScript(platformName, version, installationDir, sentinelFilePath) + } + sdkStorageBaseUrl := os.Getenv(consts.SdkStorageBaseUrlKeyName) if sdkStorageBaseUrl == "" { panic("Environment variable " + consts.SdkStorageBaseUrlKeyName + " is required.") @@ -95,3 +101,78 @@ func GetSetupScript(platformName string, version string, installationDir string) scriptBuilder.WriteString("echo\n") return scriptBuilder.String() } + +// GetAcrSetupScript generates a bash script to download an SDK from ACR using the OCI Distribution API. +// The script uses curl to fetch the manifest, extract the layer digest, download the blob, +// verify its SHA256 checksum, and extract the tarball. +func GetAcrSetupScript(platformName string, version string, installationDir string, sentinelFilePath string) string { + acrRegistryUrl := os.Getenv(consts.AcrSdkRegistryUrlKeyName) + if acrRegistryUrl == "" { + acrRegistryUrl = consts.DefaultAcrSdkRegistryUrl + } + + // Remove trailing slash + acrRegistryUrl = strings.TrimRight(acrRegistryUrl, "/") + + debianFlavor := os.Getenv(consts.DebianFlavor) + if debianFlavor == "" { + debianFlavor = "bookworm" + } + + repository := fmt.Sprintf("%s/%s", consts.AcrSdkRepositoryPrefix, platformName) + tag := fmt.Sprintf("%s-%s", debianFlavor, version) + + scriptBuilder := strings.Builder{} + scriptBuilder.WriteString("#!/bin/sh\n") + scriptBuilder.WriteString("set -e\n") + scriptBuilder.WriteString("echo\n") + scriptBuilder.WriteString( + fmt.Sprintf("echo Downloading '%s' version '%s' from ACR to '%s'...\n", platformName, version, installationDir)) + scriptBuilder.WriteString(fmt.Sprintf("mkdir -p %s\n", installationDir)) + scriptBuilder.WriteString(fmt.Sprintf("cd %s\n", installationDir)) + + // Fetch the OCI manifest + manifestUrl := fmt.Sprintf("%s/v2/%s/manifests/%s", acrRegistryUrl, repository, tag) + scriptBuilder.WriteString(fmt.Sprintf( + "echo Fetching OCI manifest from ACR for %s:%s...\n", repository, tag)) + scriptBuilder.WriteString(fmt.Sprintf( + "MANIFEST=$(curl -sSL -H 'Accept: application/vnd.oci.image.manifest.v1+json' '%s')\n", manifestUrl)) + + // Extract the layer digest (first/only layer in a FROM scratch image) + scriptBuilder.WriteString( + "LAYER_DIGEST=$(echo \"$MANIFEST\" | grep -o '\"digest\":\"sha256:[a-f0-9]*\"' | tail -1 | cut -d'\"' -f4)\n") + scriptBuilder.WriteString("if [ -z \"$LAYER_DIGEST\" ]; then\n") + scriptBuilder.WriteString(" echo 'ERROR: Could not extract layer digest from ACR manifest.'\n") + scriptBuilder.WriteString(" exit 1\n") + scriptBuilder.WriteString("fi\n") + scriptBuilder.WriteString("echo \"Layer digest: $LAYER_DIGEST\"\n") + + // Extract expected SHA256 from digest + scriptBuilder.WriteString("EXPECTED_SHA256=$(echo \"$LAYER_DIGEST\" | cut -d':' -f2)\n") + + // Download the layer blob + blobUrl := fmt.Sprintf("%s/v2/%s/blobs/", acrRegistryUrl, repository) + scriptBuilder.WriteString(fmt.Sprintf( + "echo Downloading SDK blob from ACR...\n")) + scriptBuilder.WriteString(fmt.Sprintf( + "curl -sSL '%s'\"$LAYER_DIGEST\" --output sdk.tar.gz\n", blobUrl)) + + // Verify SHA256 checksum + scriptBuilder.WriteString("echo Verifying SHA256 checksum...\n") + scriptBuilder.WriteString("ACTUAL_SHA256=$(sha256sum sdk.tar.gz | cut -d' ' -f1)\n") + scriptBuilder.WriteString("if [ \"$ACTUAL_SHA256\" != \"$EXPECTED_SHA256\" ]; then\n") + scriptBuilder.WriteString(" echo \"ERROR: SHA256 checksum mismatch. Expected: $EXPECTED_SHA256, Got: $ACTUAL_SHA256\"\n") + scriptBuilder.WriteString(" rm -f sdk.tar.gz\n") + scriptBuilder.WriteString(" exit 1\n") + scriptBuilder.WriteString("fi\n") + scriptBuilder.WriteString("echo Checksum verified.\n") + + // Extract and clean up + scriptBuilder.WriteString("echo Extracting contents...\n") + scriptBuilder.WriteString("tar -xzf sdk.tar.gz -C .\n") + scriptBuilder.WriteString("rm -f sdk.tar.gz\n") + scriptBuilder.WriteString(fmt.Sprintf("echo Done. Installed at '%s' (from ACR)\n", installationDir)) + scriptBuilder.WriteString(fmt.Sprintf("echo > %s\n", sentinelFilePath)) + scriptBuilder.WriteString("echo\n") + return scriptBuilder.String() +} diff --git a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs index 3621abb421..06d2e58b74 100644 --- a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs +++ b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCoreBuildScriptGenerationTest.cs @@ -99,6 +99,7 @@ private DotNetCorePlatform CreateDotNetCorePlatform( DotNetCoreInstaller, globalJsonSdkResolver, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -113,6 +114,7 @@ public TestDotNetCorePlatform( DotNetCorePlatformInstaller DotNetCoreInstaller, GlobalJsonSdkResolver globalJsonSdkResolver, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) : base( DotNetCoreVersionProvider, @@ -123,6 +125,7 @@ public TestDotNetCorePlatform( DotNetCoreInstaller, globalJsonSdkResolver, externalSdkProvider, + externalAcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs index 328819275a..9b6a4c3567 100644 --- a/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs +++ b/tests/BuildScriptGenerator.Tests/DotnetCore/DotNetCorePlatformTest.cs @@ -166,6 +166,7 @@ private DotNetCorePlatform CreatePlatform( installer, globalJsonSdkResolver, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -179,6 +180,7 @@ public TestDotNetCorePlatform( DotNetCorePlatformInstaller platformInstaller, GlobalJsonSdkResolver globalJsonSdkResolver, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) : base( versionProvider, @@ -189,6 +191,7 @@ public TestDotNetCorePlatform( platformInstaller, globalJsonSdkResolver, externalSdkProvider, + externalAcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs b/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs index 347d020f85..04e83e45b6 100644 --- a/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs +++ b/tests/BuildScriptGenerator.Tests/Node/NodeBuildScriptGenerationTest.cs @@ -984,6 +984,7 @@ private static IProgrammingPlatform GetNodePlatform( new TestEnvironment(), new NodePlatformInstaller(Options.Create(commonOptions), NullLoggerFactory.Instance), externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } diff --git a/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs b/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs index c28e3d0ce9..d96b754682 100644 --- a/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs +++ b/tests/BuildScriptGenerator.Tests/Node/NodePlatformTest.cs @@ -1025,6 +1025,7 @@ private TestNodePlatform CreateNodePlatform( environment, platformInstaller, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -1049,6 +1050,7 @@ private TestNodePlatform CreateNodePlatform( environment, platformInstaller, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -1077,6 +1079,7 @@ private TestNodePlatform CreateNodePlatform( environment, installer, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -1101,6 +1104,7 @@ public TestNodePlatform( IEnvironment environment, NodePlatformInstaller nodePlatformInstaller, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) : base( cliOptions, @@ -1111,6 +1115,7 @@ public TestNodePlatform( environment, nodePlatformInstaller, externalSdkProvider, + externalAcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/Node/NodeVersionProviderTest.cs b/tests/BuildScriptGenerator.Tests/Node/NodeVersionProviderTest.cs index f3080bc8c8..4484deb14f 100644 --- a/tests/BuildScriptGenerator.Tests/Node/NodeVersionProviderTest.cs +++ b/tests/BuildScriptGenerator.Tests/Node/NodeVersionProviderTest.cs @@ -137,6 +137,7 @@ public override PlatformVersionInfo GetVersionInfo() onDiskProvider, storageProvider, externalProvider, + new NodeAcrVersionProvider(commonOptions, new TestHttpClientFactory(), NullLoggerFactory.Instance), NullLogger.Instance); return (versionProvider, onDiskProvider, storageProvider, externalProvider); } diff --git a/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs b/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs index 57f06f0c92..17163568fc 100644 --- a/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs +++ b/tests/BuildScriptGenerator.Tests/Php/PhpPlatformTest.cs @@ -591,6 +591,7 @@ private PhpPlatform CreatePhpPlatform( phpInstaller, phpComposerInstaller, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -616,6 +617,7 @@ public TestPhpPlatform( PhpPlatformInstaller phpInstaller, PhpComposerInstaller phpComposerInstaller, IExternalSdkProvider externalSdkProvider, + IExternalAcrSdkProvider externalAcrSdkProvider, TelemetryClient telemetryClient) : base( phpScriptGeneratorOptions, @@ -627,6 +629,7 @@ public TestPhpPlatform( phpInstaller, phpComposerInstaller, externalSdkProvider, + externalAcrSdkProvider, telemetryClient) { } diff --git a/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs b/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs index 958a40ea0a..1c2c8ab822 100644 --- a/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs +++ b/tests/BuildScriptGenerator.Tests/Php/PhpScriptGeneratorTest.cs @@ -146,6 +146,7 @@ private IProgrammingPlatform GetScriptGenerator( phpInstaller: null, phpComposerInstaller: null, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } diff --git a/tests/BuildScriptGenerator.Tests/Php/PhpVersionProviderTest.cs b/tests/BuildScriptGenerator.Tests/Php/PhpVersionProviderTest.cs index fe082faa6a..39472085ff 100644 --- a/tests/BuildScriptGenerator.Tests/Php/PhpVersionProviderTest.cs +++ b/tests/BuildScriptGenerator.Tests/Php/PhpVersionProviderTest.cs @@ -140,6 +140,7 @@ public override PlatformVersionInfo GetVersionInfo() onDiskProvider, storageProvider, externalProvider, + new PhpAcrVersionProvider(commonOptions, new TestHttpClientFactory(), NullLoggerFactory.Instance), NullLogger.Instance); return (versionProvider, onDiskProvider, storageProvider, externalProvider); } diff --git a/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs b/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs index f068c1a5ba..41058e6374 100644 --- a/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs +++ b/tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs @@ -433,6 +433,7 @@ private PythonPlatform CreatePlatform( detector: null, platformInstaller, externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } @@ -460,6 +461,7 @@ private PythonPlatform CreatePlatform( detector, new PythonPlatformInstaller(Options.Create(commonOptions), NullLoggerFactory.Instance), externalSdkProvider, + new TestExternalAcrSdkProvider(), TelemetryClientHelper.GetTelemetryClient()); } diff --git a/tests/BuildScriptGenerator.Tests/Python/PythonVersionProviderTest.cs b/tests/BuildScriptGenerator.Tests/Python/PythonVersionProviderTest.cs index 800a2b8b2c..5693db7ecb 100644 --- a/tests/BuildScriptGenerator.Tests/Python/PythonVersionProviderTest.cs +++ b/tests/BuildScriptGenerator.Tests/Python/PythonVersionProviderTest.cs @@ -138,6 +138,7 @@ public override PlatformVersionInfo GetVersionInfo() onDiskProvider, storageProvider, externalProvider, + new PythonAcrVersionProvider(commonOptions, new TestHttpClientFactory(), NullLoggerFactory.Instance), NullLogger.Instance); return (versionProvider, onDiskProvider, storageProvider, externalProvider); } diff --git a/tests/Oryx.Tests.Common/TestExternalAcrSdkProvider.cs b/tests/Oryx.Tests.Common/TestExternalAcrSdkProvider.cs new file mode 100644 index 0000000000..a56624a602 --- /dev/null +++ b/tests/Oryx.Tests.Common/TestExternalAcrSdkProvider.cs @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Microsoft.Oryx.BuildScriptGenerator; + +namespace Microsoft.Oryx.Tests.Common +{ + public class TestExternalAcrSdkProvider : IExternalAcrSdkProvider + { + public Task RequestSdkFromAcrAsync(string platformName, string version, string debianFlavor) + { + return Task.FromResult(false); + } + } +}