From 645b5a6f2dfa56f7a9c230af7452d3cf008efd1e Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:27:37 -0800 Subject: [PATCH 01/14] Add stopwatch and start optimizing --- src/code/InstallHelper.cs | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index d7c3e9cd4..d647d12df 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -6,6 +6,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.IO.Compression; @@ -603,6 +604,7 @@ private List InstallPackages( _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); } + Console.WriteLine($"~~~Finding Dependencies for {parentPkgObj.Name}~~~"); // Get the dependencies from the installed package. if (parentPkgObj.Dependencies.Length > 0) { @@ -738,7 +740,21 @@ private Hashtable BeginPackageInstall( default: // VersionType.NoVersion + + Stopwatch stopwatch = new Stopwatch(); + // Start measuring time for Find + stopwatch.Start(); + responses = currentServer.FindName(pkgNameToInstall, _prerelease, ResourceType.None, out ErrorRecord findNameErrRecord); + + // Stop measuring time + stopwatch.Stop(); + + // Get the elapsed time in milliseconds + long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; + + Console.WriteLine($"Find Name - Elapsed Time: {elapsedMilliseconds} milliseconds"); + if (findNameErrRecord != null) { errRecord = findNameErrRecord; @@ -796,6 +812,10 @@ private Hashtable BeginPackageInstall( // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) if (!_reinstall) { + Stopwatch stopwatch = new Stopwatch(); + // Start measuring time + stopwatch.Start(); + string currPkgNameVersion = $"{pkgToInstall.Name}{pkgToInstall.Version}"; if (_packagesOnMachine.Contains(currPkgNameVersion)) { @@ -806,6 +826,14 @@ private Hashtable BeginPackageInstall( return packagesHash; } + + // Stop measuring time + stopwatch.Stop(); + + // Get the elapsed time in milliseconds + long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; + + Console.WriteLine($"Checking packages already installed on machine - Elapsed Time: {elapsedMilliseconds} milliseconds"); } if (packagesHash.ContainsKey(pkgToInstall.Name)) @@ -851,9 +879,46 @@ private Hashtable BeginPackageInstall( } else { + + // TODO -- optimize + + // 1) Find all dependencies + //pkgToInstall.Dependencies; + var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); + _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); + List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + + foreach (PSResourceInfo pkg in allDependencies) + { + Console.WriteLine($"{pkg.Name}: {pkg.Version}"); + + + } + + + + + // Download the package. string pkgName = pkgToInstall.Name; + + Stopwatch stopwatch = new Stopwatch(); + + // Start measuring time + stopwatch.Start(); + Stream responseStream = currentServer.InstallPackage(pkgName, pkgVersion, _prerelease, out ErrorRecord installNameErrRecord); + + // Stop measuring time + stopwatch.Stop(); + + // Get the elapsed time in milliseconds + long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; + + Console.WriteLine($"Install Package - Elapsed Time: {elapsedMilliseconds} milliseconds"); + + + if (installNameErrRecord != null) { errRecord = installNameErrRecord; From 16abdc915f13b60df801fd4d803036e7e984b2e8 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:28:03 -0800 Subject: [PATCH 02/14] Add more timing, start optimizations --- src/code/InstallPSResource.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/code/InstallPSResource.cs b/src/code/InstallPSResource.cs index a65797268..ea75ddb38 100644 --- a/src/code/InstallPSResource.cs +++ b/src/code/InstallPSResource.cs @@ -6,6 +6,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Management.Automation; @@ -279,6 +280,13 @@ protected override void ProcessRecord() switch (ParameterSetName) { case NameParameterSet: + Stopwatch stopwatch = new Stopwatch(); + + // Start measuring time + stopwatch.Start(); + + + ProcessInstallHelper( pkgNames: Name, pkgVersion: Version, @@ -286,6 +294,16 @@ protected override void ProcessRecord() pkgRepository: Repository, pkgCredential: Credential, reqResourceParams: null); +; + + // Stop measuring time + stopwatch.Stop(); + + // Get the elapsed time in milliseconds + long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; + + Console.WriteLine($"Total Elapsed Time: {elapsedMilliseconds} milliseconds"); + break; case InputObjectParameterSet: From 85f15d83b931f6ecdb2ec88ed7054fa33bc90d29 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Tue, 13 Feb 2024 11:45:01 -0800 Subject: [PATCH 03/14] For each parallel --- src/code/FindHelper.cs | 123 ++++++++++++++++++++++---- src/code/InstallHelper.cs | 163 +++++++++++++---------------------- src/code/V2ServerAPICalls.cs | 8 +- 3 files changed, 171 insertions(+), 123 deletions(-) diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 1d3fce098..97d0d9d98 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -35,6 +35,7 @@ internal class FindHelper private bool _repositoryNameContainsWildcard = true; private NetworkCredential _networkCredential; private Dictionary> _packagesFound; + private HashSet _knownLatestPkgVersion; #endregion @@ -48,6 +49,7 @@ public FindHelper(CancellationToken cancellationToken, PSCmdlet cmdletPassedIn, _cmdletPassedIn = cmdletPassedIn; _networkCredential = networkCredential; _packagesFound = new Dictionary>(StringComparer.OrdinalIgnoreCase); + _knownLatestPkgVersion = new HashSet(StringComparer.OrdinalIgnoreCase); } #endregion @@ -1049,8 +1051,14 @@ internal IEnumerable FindDependencyPackages( { PSResourceInfo depPkg = null; - if (dep.VersionRange.Equals(VersionRange.All)) + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { + if (_knownLatestPkgVersion.Contains(dep.Name)) + { + Console.WriteLine($"{dep.Name} already has the known latest version, so we don't need to find it"); + continue; + } + FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); if (errRecord != null) { @@ -1058,7 +1066,8 @@ internal IEnumerable FindDependencyPackages( { _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); } - else { + else + { _cmdletPassedIn.WriteError(errRecord); } yield return null; @@ -1070,9 +1079,9 @@ internal IEnumerable FindDependencyPackages( { // This scenario may occur when the package version requested is unlisted. _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, this)); yield return null; continue; @@ -1081,9 +1090,73 @@ internal IEnumerable FindDependencyPackages( if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + yield return null; + continue; + } + + depPkg = currentResult.returnedObject; + + if (!_packagesFound.ContainsKey(depPkg.Name)) + { + _knownLatestPkgVersion.Add(depPkg.Name); + foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + { + yield return depRes; + } + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + { + yield return depRes; + } + } + } + } + else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) + { + FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); + if (errRecord != null) + { + if (errRecord.Exception is ResourceNotFoundException) + { + _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); + } + else + { + _cmdletPassedIn.WriteError(errRecord); + } + yield return null; + continue; + } + + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + _cmdletPassedIn.WriteError(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + yield return null; + continue; + } + + if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + _cmdletPassedIn.WriteError(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, this)); yield return null; continue; @@ -1120,7 +1193,8 @@ internal IEnumerable FindDependencyPackages( { _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); } - else { + else + { _cmdletPassedIn.WriteError(errRecord); } yield return null; @@ -1130,9 +1204,9 @@ internal IEnumerable FindDependencyPackages( if (responses.IsFindResultsEmpty()) { _cmdletPassedIn.WriteError(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, + new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), + "FindDepPackagesFindVersionGlobbingFailure", + ErrorCategory.InvalidResult, this)); yield return null; continue; @@ -1143,11 +1217,11 @@ internal IEnumerable FindDependencyPackages( if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, this)); - + yield return null; continue; } @@ -1155,15 +1229,21 @@ internal IEnumerable FindDependencyPackages( // Check to see if version falls within version range PSResourceInfo foundDep = currentResult.returnedObject; string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) { + if (foundDep.IsPrerelease) + { depVersionStr += $"-{foundDep.Prerelease}"; } if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) && dep.VersionRange.Satisfies(depVersion)) { + Console.WriteLine($"**** Dependency package '{foundDep.Name}', version '{foundDep.Version}' saved as new depPkg"); + depPkg = foundDep; + break; } + + Console.WriteLine($"Dependency package '{foundDep.Name}', version '{foundDep.Version}'"); } if (depPkg == null) @@ -1173,16 +1253,23 @@ internal IEnumerable FindDependencyPackages( if (!_packagesFound.ContainsKey(depPkg.Name)) { + Console.WriteLine($"PackagesFound contains {depPkg.Name}"); + foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) { yield return depRes; } } - else { + else + { List pkgVersions = _packagesFound[depPkg.Name] as List; // _packagesFound has depPkg.name in it, but the version is not the same + + Console.WriteLine($" _packagesFound has ${depPkg.Name} in it, but the version '{depPkg.Version}' is not the same '{string.Join(",", pkgVersions)}'"); + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) { + Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) { yield return depRes; diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index d647d12df..059da182a 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -16,6 +16,7 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -571,10 +572,11 @@ private List InstallPackages( currentServer: currentServer, currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, + skipDependencyCheck: skipDependencyCheck, packagesHash: new Hashtable(StringComparer.InvariantCultureIgnoreCase), errRecord: out ErrorRecord errRecord); - // At this point parent package is installed to temp path. + // At this point all packagea are installed to temp path. if (errRecord != null) { if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) @@ -594,69 +596,6 @@ private List InstallPackages( continue; } - Hashtable parentPkgInfo = packagesHash[parentPackage] as Hashtable; - PSResourceInfo parentPkgObj = parentPkgInfo["psResourceInfoPkg"] as PSResourceInfo; - - if (!skipDependencyCheck) - { - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.v3) - { - _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); - } - - Console.WriteLine($"~~~Finding Dependencies for {parentPkgObj.Name}~~~"); - // Get the dependencies from the installed package. - if (parentPkgObj.Dependencies.Length > 0) - { - bool depFindFailed = false; - foreach (PSResourceInfo depPkg in findHelper.FindDependencyPackages(currentServer, currentResponseUtil, parentPkgObj, repository)) - { - if (depPkg == null) - { - depFindFailed = true; - continue; - } - - if (String.Equals(depPkg.Name, parentPkgObj.Name, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - NuGetVersion depVersion = null; - if (depPkg.AdditionalMetadata.ContainsKey("NormalizedVersion")) - { - if (!NuGetVersion.TryParse(depPkg.AdditionalMetadata["NormalizedVersion"] as string, out depVersion)) - { - NuGetVersion.TryParse(depPkg.Version.ToString(), out depVersion); - } - } - - packagesHash = BeginPackageInstall( - searchVersionType: VersionType.SpecificVersion, - specificVersion: depVersion, - versionRange: null, - pkgNameToInstall: depPkg.Name, - repository: repository, - currentServer: currentServer, - currentResponseUtil: currentResponseUtil, - tempInstallPath: tempInstallPath, - packagesHash: packagesHash, - errRecord: out ErrorRecord installPkgErrRecord); - - if (installPkgErrRecord != null) - { - _cmdletPassedIn.WriteError(installPkgErrRecord); - continue; - } - } - - if (depFindFailed) - { - continue; - } - } - } - // If -WhatIf is passed in, early out. if (!_cmdletPassedIn.ShouldProcess("Exit ShouldProcess")) { @@ -706,6 +645,7 @@ private Hashtable BeginPackageInstall( ServerApiCall currentServer, ResponseUtil currentResponseUtil, string tempInstallPath, + bool skipDependencyCheck, Hashtable packagesHash, out ErrorRecord errRecord) { @@ -713,6 +653,7 @@ private Hashtable BeginPackageInstall( FindResults responses = null; errRecord = null; + // Find the parent package that needs to be installed switch (searchVersionType) { case VersionType.VersionRange: @@ -764,6 +705,7 @@ private Hashtable BeginPackageInstall( break; } + // Convert parent package to PSResourceInfo PSResourceInfo pkgToInstall = null; foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) { @@ -809,6 +751,7 @@ private Hashtable BeginPackageInstall( if (pkgVersion == null) { pkgVersion = pkgToInstall.Version.ToString(); } + // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) if (!_reinstall) { @@ -879,61 +822,79 @@ private Hashtable BeginPackageInstall( } else { - - // TODO -- optimize - - // 1) Find all dependencies - //pkgToInstall.Dependencies; - var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); - _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); - List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - - foreach (PSResourceInfo pkg in allDependencies) + // Find all dependencies + string pkgName = pkgToInstall.Name; + if (!skipDependencyCheck) { - Console.WriteLine($"{pkg.Name}: {pkg.Version}"); + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.v3) + { + _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); + } - - } + var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); + _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); + List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + foreach (PSResourceInfo pkg in allDependencies) + { + Console.WriteLine($"{pkg.Name}: {pkg.Version}"); + } + // TODO: ADD FOREACH HERE + Parallel.ForEach(allDependencies, pkgToBeInstalled => + { + // Perform some operation on each item + // Console.WriteLine($"Processing number: {pkgToBeInstalled}, Thread ID: {Task.CurrentId}"); + + Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); - // Download the package. - string pkgName = pkgToInstall.Name; + if (installNameErrRecord != null) + { + // errRecord = installNameErrRecord; - Stopwatch stopwatch = new Stopwatch(); + //write error + //return packagesHash; + } - // Start measuring time - stopwatch.Start(); + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out ErrorRecord tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out ErrorRecord tempInstallErrRecord); - Stream responseStream = currentServer.InstallPackage(pkgName, pkgVersion, _prerelease, out ErrorRecord installNameErrRecord); + if (!installedToTempPathSuccessfully) + { + //return packagesHash; + } - // Stop measuring time - stopwatch.Stop(); + // packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); - // Get the elapsed time in milliseconds - long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; - Console.WriteLine($"Install Package - Elapsed Time: {elapsedMilliseconds} milliseconds"); + }); + } + else { + // TODO: check this version and prerelease combo + Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + errRecord = installNameErrRecord; + return packagesHash; + } - if (installNameErrRecord != null) - { - errRecord = installNameErrRecord; - return packagesHash; - } + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + if (!installedToTempPathSuccessfully) + { + return packagesHash; + } - if (!installedToTempPathSuccessfully) - { - return packagesHash; + packagesHash.Add(pkgToInstall.Name, pkgToInstall.Version); } } + // TODO: figure out packagesHash / UpdatedPackagesHash return updatedPackagesHash; } @@ -1000,7 +961,7 @@ private bool TryInstallToTempPath( out Hashtable updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::TryInstallToTempPath()"); error = null; updatedPackagesHash = packagesHash; try @@ -1100,7 +1061,7 @@ private bool TryInstallToTempPath( { foreach (ErrorRecord parseError in parseScriptFileErrors) { - _cmdletPassedIn.WriteError(parseError); + // _cmdletPassedIn.WriteError(parseError); } error = new ErrorRecord( @@ -1117,7 +1078,7 @@ private bool TryInstallToTempPath( // This package is not a PowerShell package (eg a resource from the NuGet Gallery). installPath = _pathsToInstallPkg.Find(path => path.EndsWith("Modules", StringComparison.InvariantCultureIgnoreCase)); - _cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); + //_cmdletPassedIn.WriteVerbose($"This resource is not a PowerShell package and will be installed to the modules path: {installPath}."); isModule = true; } diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index 0c6311e1c..b7a992df5 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -754,13 +754,13 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) /// private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); + // _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCallForContent()"); errRecord = null; HttpContent content = null; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + //_cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); content = SendV2RequestForContentAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -792,7 +792,7 @@ private HttpContent HttpRequestCallForContent(string requestUrlV2, out ErrorReco if (content == null || string.IsNullOrEmpty(content.ToString())) { - _cmdletPassedIn.WriteDebug("Response is empty"); + //_cmdletPassedIn.WriteDebug("Response is empty"); } return content; @@ -1153,7 +1153,7 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange /// private Stream InstallVersion(string packageName, string version, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::InstallVersion()"); string requestUrlV2; if (_isADORepo) From 9fc852f1bc8f4a4a305432e906447f6c57b1d368 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:17:35 -0800 Subject: [PATCH 04/14] Add TODOs --- src/code/InstallHelper.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index 059da182a..13a7612b8 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -840,14 +840,18 @@ private Hashtable BeginPackageInstall( Console.WriteLine($"{pkg.Name}: {pkg.Version}"); } + // 1) TODO: If installing 5 or less packages, don't use Parallel.ForEach + + // TODO: ADD FOREACH HERE + // 2) TODO: MaxDegreeOfParallelism -- possibly max of about 32 threads (Invoke-Command has default of 32, that's where we got this number from)? + // 3) TODO: Error handling: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-handle-exceptions-in-parallel-loops Parallel.ForEach(allDependencies, pkgToBeInstalled => { // Perform some operation on each item // Console.WriteLine($"Processing number: {pkgToBeInstalled}, Thread ID: {Task.CurrentId}"); - Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) From 00fdc2f6e65fdc6ce07115122ebb69953164d27e Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Wed, 21 Feb 2024 10:42:25 -0800 Subject: [PATCH 05/14] Add 'updatedPackagesHash', max threads --- src/code/InstallHelper.cs | 93 +++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index 13a7612b8..a131a4535 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -833,47 +833,92 @@ private Hashtable BeginPackageInstall( var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); + + // the last package added will be the parent package. List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + PSResourceInfo parentPkg = allDependencies.Last(); + allDependencies.Remove(parentPkg); + // allDependencies contains parent package as well foreach (PSResourceInfo pkg in allDependencies) { Console.WriteLine($"{pkg.Name}: {pkg.Version}"); } - // 1) TODO: If installing 5 or less packages, don't use Parallel.ForEach + // List for keeping track of errors + List errors = new List(); + // If installing 5 or less packages, don't use Parallel.ForEach + if (allDependencies.Count > 5) + { + // Error handling: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-handle-exceptions-in-parallel-loops + // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) + ParallelOptions options = new ParallelOptions + { + MaxDegreeOfParallelism = 32 + }; + _cmdletPassedIn.WriteDebug("**********************************************************Entering parallel foreach."); + Parallel.ForEach(allDependencies, options, pkgToBeInstalled => + { + // Perform some operation on each item + Console.WriteLine($"Processing number: {pkgToBeInstalled}, Thread ID: {Task.CurrentId}"); - // TODO: ADD FOREACH HERE - // 2) TODO: MaxDegreeOfParallelism -- possibly max of about 32 threads (Invoke-Command has default of 32, that's where we got this number from)? - // 3) TODO: Error handling: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-handle-exceptions-in-parallel-loops - Parallel.ForEach(allDependencies, pkgToBeInstalled => - { - // Perform some operation on each item - // Console.WriteLine($"Processing number: {pkgToBeInstalled}, Thread ID: {Task.CurrentId}"); - - Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); + Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) - { - // errRecord = installNameErrRecord; + if (installNameErrRecord != null) + { + errors.Add(installNameErrRecord); - //write error - //return packagesHash; - } + //return packagesHash; + } + + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); + + //return packagesHash; + } - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out ErrorRecord tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out ErrorRecord tempInstallErrRecord); + // packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); + }); - if (!installedToTempPathSuccessfully) + return updatedPackagesHash; + + } + else { + // Install the good, old fashioned way + foreach (var pkgToBeInstalled in allDependencies) { - //return packagesHash; - } + Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); - // packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); + if (installNameErrRecord != null) + { + errRecord = installNameErrRecord; + return packagesHash; + } + + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + errRecord = tempSaveErrRecord ?? tempInstallErrRecord; + return packagesHash; + } + + // no -- packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); + } + + return updatedPackagesHash; + } - }); } else { @@ -894,7 +939,7 @@ private Hashtable BeginPackageInstall( return packagesHash; } - packagesHash.Add(pkgToInstall.Name, pkgToInstall.Version); + //spackagesHash.Add(pkgToInstall.Name, pkgToInstall.Version); } } From e0fc780f61127cc18270f19f1a447c068e26f610 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Wed, 21 Feb 2024 17:29:54 -0800 Subject: [PATCH 06/14] Bug fixes for pkg name/version --- src/code/ACRServerAPICalls.cs | 10 +++--- src/code/InstallHelper.cs | 59 ++++++++++++++++++++++++++++------- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/code/ACRServerAPICalls.cs b/src/code/ACRServerAPICalls.cs index cc91daf69..7ae8c4ed6 100644 --- a/src/code/ACRServerAPICalls.cs +++ b/src/code/ACRServerAPICalls.cs @@ -538,7 +538,7 @@ public override FindResults FindVersionWithTag(string packageName, string versio /// public override Stream InstallPackage(string packageName, string packageVersion, bool includePrerelease, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In ACRServerAPICalls::InstallPackage()"); + //_cmdletPassedIn.WriteDebug("In ACRServerAPICalls::InstallPackage()"); Stream results = new MemoryStream(); if (string.IsNullOrEmpty(packageVersion)) { @@ -583,21 +583,21 @@ private Stream InstallVersion( // Call asynchronous network methods in a try/catch block to handle exceptions. string registry = Repository.Uri.Host; - _cmdletPassedIn.WriteVerbose("Getting acr refresh token"); + //_cmdletPassedIn.WriteVerbose("Getting acr refresh token"); var acrRefreshToken = GetAcrRefreshToken(registry, tenantID, accessToken, out errRecord); if (errRecord != null) { return null; } - _cmdletPassedIn.WriteVerbose("Getting acr access token"); + //_cmdletPassedIn.WriteVerbose("Getting acr access token"); var acrAccessToken = GetAcrAccessToken(registry, acrRefreshToken, out errRecord); if (errRecord != null) { return null; } - _cmdletPassedIn.WriteVerbose($"Getting manifest for {moduleName} - {moduleVersion}"); + //_cmdletPassedIn.WriteVerbose($"Getting manifest for {moduleName} - {moduleVersion}"); var manifest = GetAcrRepositoryManifestAsync(registry, moduleName, moduleVersion, acrAccessToken, out errRecord); if (errRecord != null) { @@ -609,7 +609,7 @@ private Stream InstallVersion( return null; } - _cmdletPassedIn.WriteVerbose($"Downloading blob for {moduleName} - {moduleVersion}"); + //_cmdletPassedIn.WriteVerbose($"Downloading blob for {moduleName} - {moduleVersion}"); // TODO: error handling here? var responseContent = GetAcrBlobAsync(registry, moduleName, digest, acrAccessToken).Result; diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index a131a4535..faafbb022 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -836,8 +836,7 @@ private Hashtable BeginPackageInstall( // the last package added will be the parent package. List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - PSResourceInfo parentPkg = allDependencies.Last(); - allDependencies.Remove(parentPkg); + // allDependencies contains parent package as well foreach (PSResourceInfo pkg in allDependencies) @@ -850,6 +849,10 @@ private Hashtable BeginPackageInstall( // If installing 5 or less packages, don't use Parallel.ForEach if (allDependencies.Count > 5) { + PSResourceInfo parentPkg = allDependencies.Last(); + allDependencies.Remove(parentPkg); + + // Error handling: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-handle-exceptions-in-parallel-loops // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) ParallelOptions options = new ParallelOptions @@ -858,13 +861,15 @@ private Hashtable BeginPackageInstall( }; _cmdletPassedIn.WriteDebug("**********************************************************Entering parallel foreach."); - Parallel.ForEach(allDependencies, options, pkgToBeInstalled => + Parallel.ForEach(allDependencies, options, depPkg => { + var depPkgName = depPkg.Name; + var depPkgVersion = depPkg.Version.ToString(); // Perform some operation on each item - Console.WriteLine($"Processing number: {pkgToBeInstalled}, Thread ID: {Task.CurrentId}"); + Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); - Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); + Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) { @@ -874,8 +879,8 @@ private Hashtable BeginPackageInstall( } ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); if (!installedToTempPathSuccessfully) { @@ -887,6 +892,34 @@ private Hashtable BeginPackageInstall( // packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); }); + + // Install Parent Pkg + Stream responseStream = currentServer.InstallPackage(parentPkg.Name, parentPkg.Version.ToString(), true, out ErrorRecord installNameErrRecord); + + if (installNameErrRecord != null) + { + errors.Add(installNameErrRecord); + + //return packagesHash; + } + + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + + if (!installedToTempPathSuccessfully) + { + errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); + + //return packagesHash; + } + + + foreach (var err in errors) + { + Console.WriteLine(err); + } + return updatedPackagesHash; } @@ -894,7 +927,9 @@ private Hashtable BeginPackageInstall( // Install the good, old fashioned way foreach (var pkgToBeInstalled in allDependencies) { - Stream responseStream = currentServer.InstallPackage(pkgToBeInstalled.Name, pkgToBeInstalled.Version.ToString(), true, out ErrorRecord installNameErrRecord); + var pkgToInstallName = pkgToBeInstalled.Name; + var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); + Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) { @@ -904,8 +939,8 @@ private Hashtable BeginPackageInstall( } ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgName, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); if (!installedToTempPathSuccessfully) { @@ -1187,7 +1222,7 @@ private bool TrySaveNupkgToTempPath( out Hashtable updatedPackagesHash, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::TrySaveNupkgToTempPath()"); error = null; updatedPackagesHash = packagesHash; @@ -1525,7 +1560,7 @@ private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, ou /// private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PSResourceInfo pkg, bool isModule, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::CreateMetadataXMLFile()"); error = null; bool success = true; // Script will have a metadata file similar to: "TestScript_InstalledScriptInfo.xml" From 2713d6fb023b92112cfa9b5bd0ff206f70d8f045 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:38:54 -0800 Subject: [PATCH 07/14] Add runspace --- src/code/InstallHelper.cs | 42 +++++++++++++-------------------------- src/code/Utils.cs | 31 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index faafbb022..f701e08b3 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -855,27 +855,19 @@ private Hashtable BeginPackageInstall( // Error handling: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-handle-exceptions-in-parallel-loops // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) - ParallelOptions options = new ParallelOptions - { - MaxDegreeOfParallelism = 32 - }; - _cmdletPassedIn.WriteDebug("**********************************************************Entering parallel foreach."); - Parallel.ForEach(allDependencies, options, depPkg => + Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, depPkg => { var depPkgName = depPkg.Name; var depPkgVersion = depPkg.Version.ToString(); // Perform some operation on each item Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); - Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) { errors.Add(installNameErrRecord); - - //return packagesHash; } ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; @@ -885,22 +877,26 @@ private Hashtable BeginPackageInstall( if (!installedToTempPathSuccessfully) { errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); - - //return packagesHash; } - - // packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); }); + // Early out if errors collected + if (errors.Count > 0) { + // Write out all errors collected from Parallel.ForEach + foreach (var err in errors) + { + Console.WriteLine(err); + } + + return packagesHash; + } // Install Parent Pkg Stream responseStream = currentServer.InstallPackage(parentPkg.Name, parentPkg.Version.ToString(), true, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) { errors.Add(installNameErrRecord); - - //return packagesHash; + return packagesHash; } ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; @@ -909,15 +905,8 @@ private Hashtable BeginPackageInstall( if (!installedToTempPathSuccessfully) { - errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); - - //return packagesHash; - } - - - foreach (var err in errors) - { - Console.WriteLine(err); + errRecord = tempSaveErrRecord ?? tempInstallErrRecord; + return packagesHash; } return updatedPackagesHash; @@ -934,7 +923,6 @@ private Hashtable BeginPackageInstall( if (installNameErrRecord != null) { errRecord = installNameErrRecord; - return packagesHash; } @@ -947,8 +935,6 @@ private Hashtable BeginPackageInstall( errRecord = tempSaveErrRecord ?? tempInstallErrRecord; return packagesHash; } - - // no -- packagesHash.Add(pkgToBeInstalled.Name, pkgToBeInstalled.Version); } return updatedPackagesHash; diff --git a/src/code/Utils.cs b/src/code/Utils.cs index 815b1182f..cbaea9555 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1203,6 +1203,30 @@ private static bool TryReadPSDataFile( } string contents = System.IO.File.ReadAllText(filePath); + + + + + + // Create a new PowerShell runspace + Runspace runspace = RunspaceFactory.CreateRunspace(); + runspace.Open(); + + // Set the created runspace as the default for the current thread + Runspace.DefaultRunspace = runspace; + using (System.Management.Automation.PowerShell pwsh = System.Management.Automation.PowerShell.Create()) + { + // Use the runspace in the PowerShell pipeline + pwsh.Runspace = runspace; + + // Invoke the pipeline and retrieve the results + Collection result = pwsh.Invoke(); + } + // Close the runspace when done + //runspace.Close(); + + + var scriptBlock = System.Management.Automation.ScriptBlock.Create(contents); // Ensure that the content script block is safe to convert into a PSDataFile Hashtable. @@ -1219,6 +1243,13 @@ private static bool TryReadPSDataFile( result = psObject.BaseObject; } + + + + + + + dataFileInfo = (Hashtable) result; error = null; return true; From 082c1c3ca40de3de29cf0e6548cd00771454ba64 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:28:51 -0800 Subject: [PATCH 08/14] Try adding .GetPowerShell to script block --- src/code/Utils.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/code/Utils.cs b/src/code/Utils.cs index cbaea9555..04f7a2808 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1207,7 +1207,7 @@ private static bool TryReadPSDataFile( - + /* // Create a new PowerShell runspace Runspace runspace = RunspaceFactory.CreateRunspace(); runspace.Open(); @@ -1224,6 +1224,8 @@ private static bool TryReadPSDataFile( } // Close the runspace when done //runspace.Close(); + + */ @@ -1235,7 +1237,14 @@ private static bool TryReadPSDataFile( allowedCommands: allowedCommands, allowedVariables: allowedVariables, allowEnvironmentVariables: allowEnvironmentVariables); - + + var obj = scriptBlock.GetPowerShell(); + if (obj != null) + { + Console.WriteLine("good"); + + } + // Convert contents into PSDataFile Hashtable by executing content as script. object result = scriptBlock.InvokeReturnAsIs(); if (result is PSObject psObject) From c8dec27132bc4138d996c72c0fc3df6f164519ea Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:24:00 -0800 Subject: [PATCH 09/14] Add CLM to runspace --- src/code/InstallHelper.cs | 54 +++++++++++----------- src/code/Utils.cs | 97 ++++++++++++++++++++++----------------- 2 files changed, 81 insertions(+), 70 deletions(-) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index f701e08b3..b2c5353e3 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -439,7 +439,7 @@ private void MoveFilesIntoInstallPath( string moduleManifestVersion, string scriptPath) { - _cmdletPassedIn.WriteDebug("In InstallHelper::MoveFilesIntoInstallPath()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::MoveFilesIntoInstallPath()"); // Creating the proper installation path depending on whether pkg is a module or script var newPathParent = isModule ? Path.Combine(installPath, pkgInfo.Name) : installPath; var finalModuleVersionDir = isModule ? Path.Combine(installPath, pkgInfo.Name, moduleManifestVersion) : installPath; @@ -448,32 +448,32 @@ private void MoveFilesIntoInstallPath( var tempModuleVersionDir = (!isModule || isLocalRepo) ? dirNameVersion : Path.Combine(tempInstallPath, pkgInfo.Name.ToLower(), newVersion); - _cmdletPassedIn.WriteVerbose($"Installation source path is: '{tempModuleVersionDir}'"); - _cmdletPassedIn.WriteVerbose($"Installation destination path is: '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Installation source path is: '{tempModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Installation destination path is: '{finalModuleVersionDir}'"); if (isModule) { // If new path does not exist if (!Directory.Exists(newPathParent)) { - _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); Directory.CreateDirectory(newPathParent); Utils.MoveDirectory(tempModuleVersionDir, finalModuleVersionDir); } else { - _cmdletPassedIn.WriteVerbose($"Temporary module version directory is: '{tempModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Temporary module version directory is: '{tempModuleVersionDir}'"); if (Directory.Exists(finalModuleVersionDir)) { // Delete the directory path before replacing it with the new module. // If deletion fails (usually due to binary file in use), then attempt restore so that the currently // installed module is not corrupted. - _cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure.'{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to delete with restore on failure.'{finalModuleVersionDir}'"); Utils.DeleteDirectoryWithRestore(finalModuleVersionDir); } - _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); + // _cmdletPassedIn.WriteVerbose($"Attempting to move '{tempModuleVersionDir}' to '{finalModuleVersionDir}'"); Utils.MoveDirectory(tempModuleVersionDir, finalModuleVersionDir); } } @@ -499,36 +499,36 @@ private void MoveFilesIntoInstallPath( if (!Directory.Exists(scriptInfoFolderPath)) { - _cmdletPassedIn.WriteVerbose($"Created '{scriptInfoFolderPath}' path for scripts"); + // _cmdletPassedIn.WriteVerbose($"Created '{scriptInfoFolderPath}' path for scripts"); Directory.CreateDirectory(scriptInfoFolderPath); } // Need to delete old xml files because there can only be 1 per script - _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: '{1}'", scriptXmlFilePath, File.Exists(scriptXmlFilePath))); + // _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: '{1}'", scriptXmlFilePath, File.Exists(scriptXmlFilePath))); if (File.Exists(scriptXmlFilePath)) { - _cmdletPassedIn.WriteVerbose("Deleting script metadata XML"); + // _cmdletPassedIn.WriteVerbose("Deleting script metadata XML"); File.Delete(Path.Combine(scriptInfoFolderPath, scriptXML)); } - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML))); Utils.MoveFiles(Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, "InstalledScriptInfos", scriptXML)); // Need to delete old script file, if that exists - _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: ", File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)))); + // _cmdletPassedIn.WriteVerbose(string.Format("Checking if path '{0}' exists: ", File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)))); if (File.Exists(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))) { - _cmdletPassedIn.WriteVerbose("Deleting script file"); + // _cmdletPassedIn.WriteVerbose("Deleting script file"); File.Delete(Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)); } } else { - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, scriptXML))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, scriptXML))); Utils.MoveFiles(Path.Combine(dirNameVersion, scriptXML), Path.Combine(installPath, scriptXML)); } - _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))); + // _cmdletPassedIn.WriteVerbose(string.Format("Moving '{0}' to '{1}'", scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt))); Utils.MoveFiles(scriptPath, Path.Combine(finalModuleVersionDir, pkgInfo.Name + PSScriptFileExt)); } } @@ -1330,7 +1330,7 @@ private bool TryExtractToDirectory(string zipPath, string extractPath, out Error /// private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hashtable packagesHash) { - _cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::TryMoveInstallContent()"); foreach (string pkgName in packagesHash.Keys) { Hashtable pkgInfo = packagesHash[pkgName] as Hashtable; @@ -1355,7 +1355,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash moduleManifestVersion: pkgVersion, scriptPath); - _cmdletPassedIn.WriteVerbose($"Successfully installed package '{pkgName}' to location '{installPath}'"); + //_cmdletPassedIn.WriteVerbose($"Successfully installed package '{pkgName}' to location '{installPath}'"); if (!_savePkg && isScript) { @@ -1374,20 +1374,20 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash if (!String.IsNullOrEmpty(envPATHVarValue) && !envPATHVarValue.Contains(installPath) && !envPATHVarValue.Contains(installPathwithBackSlash)) { - _cmdletPassedIn.WriteWarning(String.Format(ScriptPATHWarning, scope, installPath)); + //_cmdletPassedIn.WriteWarning(String.Format(ScriptPATHWarning, scope, installPath)); } } } catch (Exception e) { - _cmdletPassedIn.WriteError(new ErrorRecord( + /*_cmdletPassedIn.WriteError(new ErrorRecord( new PSInvalidOperationException( message: $"Unable to successfully install package '{pkgName}': '{e.Message}'", innerException: e), "InstallPackageFailed", ErrorCategory.InvalidOperation, _cmdletPassedIn)); - + */ return false; } } @@ -1400,7 +1400,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash /// private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string tempInstallPath, string newVersion, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::CallAcceptLicense()"); error = null; var requireLicenseAcceptance = false; var success = true; @@ -1482,7 +1482,7 @@ private bool CallAcceptLicense(PSResourceInfo p, string moduleManifest, string t /// private bool DetectClobber(string pkgName, Hashtable parsedMetadataHashtable, out ErrorRecord error) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); + //_cmdletPassedIn.WriteDebug("In InstallHelper::DetectClobber()"); error = null; bool foundClobber = false; @@ -1576,7 +1576,7 @@ private bool CreateMetadataXMLFile(string dirNameVersion, string installPath, PS /// private void DeleteExtraneousFiles(string packageName, string dirNameVersion) { - _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); + // _cmdletPassedIn.WriteDebug("In InstallHelper::DeleteExtraneousFiles()"); // Deleting .nupkg SHA file, .nuspec, and .nupkg after unpacking the module // since we download as .zip for HTTP calls, we shouldn't have .nupkg* files // var nupkgSHAToDelete = Path.Combine(dirNameVersion, pkgIdString + ".nupkg.sha512"); @@ -1589,22 +1589,22 @@ private void DeleteExtraneousFiles(string packageName, string dirNameVersion) if (File.Exists(nuspecToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{nuspecToDelete}'"); File.Delete(nuspecToDelete); } if (File.Exists(contentTypesToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{contentTypesToDelete}'"); File.Delete(contentTypesToDelete); } if (Directory.Exists(relsDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{relsDirToDelete}'"); Utils.DeleteDirectory(relsDirToDelete); } if (Directory.Exists(packageDirToDelete)) { - _cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); + //_cmdletPassedIn.WriteDebug($"Deleting '{packageDirToDelete}'"); Utils.DeleteDirectory(packageDirToDelete); } } diff --git a/src/code/Utils.cs b/src/code/Utils.cs index 04f7a2808..928468231 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1195,77 +1195,88 @@ private static bool TryReadPSDataFile( out Hashtable dataFileInfo, out Exception error) { + dataFileInfo = null; + error = null; try { if (filePath is null) { throw new PSArgumentNullException(nameof(filePath)); } - string contents = System.IO.File.ReadAllText(filePath); - - - - /* // Create a new PowerShell runspace Runspace runspace = RunspaceFactory.CreateRunspace(); runspace.Open(); + runspace.SessionStateProxy.LanguageMode = PSLanguageMode.ConstrainedLanguage; // Set the created runspace as the default for the current thread Runspace.DefaultRunspace = runspace; + using (System.Management.Automation.PowerShell pwsh = System.Management.Automation.PowerShell.Create()) { // Use the runspace in the PowerShell pipeline pwsh.Runspace = runspace; // Invoke the pipeline and retrieve the results - Collection result = pwsh.Invoke(); - } - // Close the runspace when done - //runspace.Close(); - - */ - + var cmd = new Command( + command: contents, + isScript: true, + useLocalScope: true); + cmd.MergeMyResults( + myResult: PipelineResultTypes.Error | PipelineResultTypes.Warning | PipelineResultTypes.Verbose | PipelineResultTypes.Debug | PipelineResultTypes.Information, + toResult: PipelineResultTypes.Output); + pwsh.Commands.AddCommand(cmd); - var scriptBlock = System.Management.Automation.ScriptBlock.Create(contents); - // Ensure that the content script block is safe to convert into a PSDataFile Hashtable. - // This will throw for unsafe content. - scriptBlock.CheckRestrictedLanguage( - allowedCommands: allowedCommands, - allowedVariables: allowedVariables, - allowEnvironmentVariables: allowEnvironmentVariables); - - var obj = scriptBlock.GetPowerShell(); - if (obj != null) - { - Console.WriteLine("good"); - - } + try + { + // Invoke the script. + var results = pwsh.Invoke(); - // Convert contents into PSDataFile Hashtable by executing content as script. - object result = scriptBlock.InvokeReturnAsIs(); - if (result is PSObject psObject) - { - result = psObject.BaseObject; + if (results[0] is PSObject pwshObj) + { + switch (pwshObj.BaseObject) + { + case ErrorRecord err: + //_cmdletPassedIn.WriteError(error); + break; + + case WarningRecord warning: + //cmdlet.WriteWarning(warning.Message); + break; + + case VerboseRecord verbose: + //cmdlet.WriteVerbose(verbose.Message); + break; + + case DebugRecord debug: + //cmdlet.WriteDebug(debug.Message); + break; + + case InformationRecord info: + //cmdlet.WriteInformation(info); + break; + + case Hashtable result: + dataFileInfo = result; + return true; + } + } + } + catch (Exception ex) + { + error = ex; + } } - - - - - - - - - dataFileInfo = (Hashtable) result; - error = null; - return true; + // Close the runspace when done + runspace.Close(); + + return false; } catch (Exception ex) { - dataFileInfo = null; error = ex; return false; } From e52bc19504572bb810c9c47552b7a3211acadd8c Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 29 Feb 2024 22:45:35 -0800 Subject: [PATCH 10/14] Add concurrency to finding dependencies --- src/code/FindHelper.cs | 768 ++++++++++++++++++++++++++-------- src/code/InstallHelper.cs | 233 +++++------ src/code/InstallPSResource.cs | 8 +- src/code/Utils.cs | 9 +- src/code/V2ServerAPICalls.cs | 16 +- 5 files changed, 703 insertions(+), 331 deletions(-) diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 97d0d9d98..19ef9fea4 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -5,11 +5,14 @@ using NuGet.Versioning; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Management.Automation; using System.Net; using System.Runtime.ExceptionServices; +using System.Security.Cryptography; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets { @@ -36,6 +39,7 @@ internal class FindHelper private NetworkCredential _networkCredential; private Dictionary> _packagesFound; private HashSet _knownLatestPkgVersion; + List depPkgsFound; #endregion @@ -1039,265 +1043,669 @@ private string FormatPkgVersionString(PSResourceInfo pkg) #region Internal Client Search Methods + internal IEnumerable FindDependencyPackages( ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) + + { + depPkgsFound = new List(); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + + // Console.WriteLine($"returned to parent method "); + foreach (var item in depPkgsFound) + { + // Console.WriteLine(item.Name); + } + foreach (var item in depPkgsFound) + { + if (!_packagesFound.ContainsKey(item.Name)) + { + TryAddToPackagesFound(item); + yield return item; + } + else + { + List pkgVersions = _packagesFound[item.Name]; + // _packagesFound has item.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(item))) + { + TryAddToPackagesFound(item); + + yield return item; + } + } + } + } + + + + internal void FindDependencyPackagesHelper( + ServerApiCall currentServer, + ResponseUtil currentResponseUtil, + PSResourceInfo currentPkg, + PSRepositoryInfo repository) { + List errors = new List(); + if (currentPkg.Dependencies.Length > 0) { - foreach (var dep in currentPkg.Dependencies) + foreach (var depend in currentPkg.Dependencies) { - PSResourceInfo depPkg = null; - if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) + // Console.WriteLine($"Dep Name: {depend.Name}, DepVersion Min: {depend.VersionRange.MinVersion} DepVersion Max: {depend.VersionRange.MaxVersion}"); + } + + // Add parallel.foreach here + // If installing more than 5 packages, do so concurrently + if (currentPkg.Dependencies.Length > 5) + { + + Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, dep => { - if (_knownLatestPkgVersion.Contains(dep.Name)) - { - Console.WriteLine($"{dep.Name} already has the known latest version, so we don't need to find it"); - continue; - } + //// Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); + PSResourceInfo depPkg = null; - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - if (errRecord != null) + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { - if (errRecord.Exception is ResourceNotFoundException) + // // Console.WriteLine($"++++++++++++++DepPkg Name is {dep.Name}"); + // // Console.WriteLine($"+++++++++++++MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); + // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. + if (_knownLatestPkgVersion.Contains(dep.Name)) { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); + // // Console.WriteLine($"{dep.Name} already has the known latest version, so we don't need to find it"); + // "return" } else { - _cmdletPassedIn.WriteError(errRecord); + // // Console.WriteLine("BEfore FindName"); + FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); + // // Console.WriteLine("After FindName"); + + if (errRecord != null) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + } + // "return" + + if (responses != null) + { + // // Console.WriteLine("Before ConvertToPSResourceResult"); + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + // // Console.WriteLine("After FindName"); + + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + _knownLatestPkgVersion.Add(depPkg.Name); + // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 1"); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + // // Console.WriteLine("After recursive FindDependencyPackagesHelper 1"); + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + + } + } + } + } } - yield return null; - continue; } - - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) + else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) { - // This scenario may occur when the package version requested is unlisted. - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + //// Console.WriteLine($"*****************DepPkg Name is {dep.Name}"); + //// Console.WriteLine($"*****************MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + //// Console.WriteLine("Before minVersion FindVersion"); - depPkg = currentResult.returnedObject; + FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); + // // Console.WriteLine("After minVersion FindVersion"); - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - _knownLatestPkgVersion.Add(depPkg.Name); - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + if (errRecord != null) { - yield return depRes; + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + // return null; + } + else + { + // // Console.WriteLine("Before min version ConvertToPSResourceResult"); + + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + // // Console.WriteLine("After min version ConvertToPSResourceResult"); + + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + //return null; + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + //return null; + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + // // Console.WriteLine("Before min version FindDependencyPackagesHelper"); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + // // Console.WriteLine("After min version FindDependencyPackagesHelper"); + + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); + + } + } + } } } else { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + // // Console.WriteLine("Before FindVersionGlobbing"); + + FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); + // // Console.WriteLine("After FindVersionGlobbing"); + + if (errRecord != null) { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else { - yield return depRes; + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } + //return } - } - } - else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) - { - FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) + + if (responses.IsFindResultsEmpty()) { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); + errors.Add(new ErrorRecord( + new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), + "FindDepPackagesFindVersionGlobbingFailure", + ErrorCategory.InvalidResult, + this)); + // return null; } else { - _cmdletPassedIn.WriteError(errRecord); + // // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult"); + + foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) + { + // // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult in for each loop"); + + if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + + //return null; + } + else + { + // Check to see if version falls within version range + PSResourceInfo foundDep = currentResult.returnedObject; + string depVersionStr = $"{foundDep.Version}"; + if (foundDep.IsPrerelease) + { + depVersionStr += $"-{foundDep.Prerelease}"; + } + + if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) + && dep.VersionRange.Satisfies(depVersion)) + { + // // Console.WriteLine($"**** Dependency package '{foundDep.Name}', version '{foundDep.Version}' saved as new depPkg"); + + depPkg = foundDep; + break; + } + + // // Console.WriteLine($"Dependency package '{foundDep.Name}', version '{foundDep.Version}'"); + } + } + + if (depPkg == null) + { + // // Console.WriteLine($"depPkg is null and I don't know what this means"); + } + else + { + if (!depPkgsFound.Contains(depPkg)) + { + // // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + + // // Console.WriteLine($" _packagesFound has ${depPkg.Name} in it, but the version '{depPkg.Version}' is not the same '{string.Join(",", pkgVersions)}'"); + + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + } + } + } } - yield return null; - continue; } - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - yield return null; - continue; - } + }); + + + + + + } + else + { + // Install the good old fashioned way + + // Console.WriteLine($"SHOUL DNOT GET HEREE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - depPkg = currentResult.returnedObject; - if (!_packagesFound.ContainsKey(depPkg.Name)) + foreach (var dep in currentPkg.Dependencies) + { + // Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); + PSResourceInfo depPkg = null; + + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + // Console.WriteLine($"++++++++++++++DepPkg Name is {dep.Name}"); + // Console.WriteLine($"+++++++++++++MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); + // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. + if (_knownLatestPkgVersion.Contains(dep.Name)) { - yield return depRes; + // Console.WriteLine($"{dep.Name} already has the known latest version, so we don't need to find it"); + // "return" } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + else { - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + // Console.WriteLine("BEfore FindName"); + FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); + // Console.WriteLine("After FindName"); + + if (errRecord != null) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + } + // "return" + + if (responses != null) { - yield return depRes; + // Console.WriteLine("Before ConvertToPSResourceResult"); + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + // Console.WriteLine("After FindName"); + + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + _knownLatestPkgVersion.Add(depPkg.Name); + // Console.WriteLine("Before recursive FindDependencyPackagesHelper 1"); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + // Console.WriteLine("After recursive FindDependencyPackagesHelper 1"); + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + + } + } + } } } } - } - else - { - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); - if (errRecord != null) + else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) { - if (errRecord.Exception is ResourceNotFoundException) + // Console.WriteLine($"*****************DepPkg Name is {dep.Name}"); + // Console.WriteLine($"*****************MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); + + // Console.WriteLine("Before minVersion FindVersion"); + + FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); + // Console.WriteLine("After minVersion FindVersion"); + + if (errRecord != null) { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + // return null; } else { - _cmdletPassedIn.WriteError(errRecord); - } - yield return null; - continue; - } + // Console.WriteLine("Before min version ConvertToPSResourceResult"); - if (responses.IsFindResultsEmpty()) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); - yield return null; - continue; - } + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + // Console.WriteLine("After min version ConvertToPSResourceResult"); - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + //return null; + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + //return null; + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + // Console.WriteLine("Before min version FindDependencyPackagesHelper"); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + // Console.WriteLine("After min version FindDependencyPackagesHelper"); + + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); + + } + } + } + } + } + else { - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - _cmdletPassedIn.WriteError(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); + // Console.WriteLine("Before FindVersionGlobbing"); - yield return null; - continue; - } + FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); + // Console.WriteLine("After FindVersionGlobbing"); - // Check to see if version falls within version range - PSResourceInfo foundDep = currentResult.returnedObject; - string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) + if (errRecord != null) { - depVersionStr += $"-{foundDep.Prerelease}"; + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + //return } - if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) - && dep.VersionRange.Satisfies(depVersion)) + if (responses.IsFindResultsEmpty()) { - Console.WriteLine($"**** Dependency package '{foundDep.Name}', version '{foundDep.Version}' saved as new depPkg"); - - depPkg = foundDep; - break; + errors.Add(new ErrorRecord( + new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), + "FindDepPackagesFindVersionGlobbingFailure", + ErrorCategory.InvalidResult, + this)); + // return null; } - - Console.WriteLine($"Dependency package '{foundDep.Name}', version '{foundDep.Version}'"); - } - - if (depPkg == null) - { - continue; - } - - if (!_packagesFound.ContainsKey(depPkg.Name)) - { - Console.WriteLine($"PackagesFound contains {depPkg.Name}"); - - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + else { - yield return depRes; - } - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same + // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult"); - Console.WriteLine($" _packagesFound has ${depPkg.Name} in it, but the version '{depPkg.Version}' is not the same '{string.Join(",", pkgVersions)}'"); + foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) + { + // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult in for each loop"); + + if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + + //return null; + } + else + { + // Check to see if version falls within version range + PSResourceInfo foundDep = currentResult.returnedObject; + string depVersionStr = $"{foundDep.Version}"; + if (foundDep.IsPrerelease) + { + depVersionStr += $"-{foundDep.Prerelease}"; + } + + if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) + && dep.VersionRange.Satisfies(depVersion)) + { + // Console.WriteLine($"**** Dependency package '{foundDep.Name}', version '{foundDep.Version}' saved as new depPkg"); + + depPkg = foundDep; + break; + } + + // Console.WriteLine($"Dependency package '{foundDep.Name}', version '{foundDep.Version}'"); + } + } - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); - foreach (PSResourceInfo depRes in FindDependencyPackages(currentServer, currentResponseUtil, depPkg, repository)) + if (depPkg == null) + { + // Console.WriteLine($"depPkg is null and I don't know what this means"); + } + else { - yield return depRes; + if (!depPkgsFound.Contains(depPkg)) + { + // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); + + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + + // Console.WriteLine($" _packagesFound has ${depPkg.Name} in it, but the version '{depPkg.Version}' is not the same '{string.Join(",", pkgVersions)}'"); + + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + } + } } } } + + } - } - } - if (!_packagesFound.ContainsKey(currentPkg.Name)) - { - TryAddToPackagesFound(currentPkg); - yield return currentPkg; - } - else - { - List pkgVersions = _packagesFound[currentPkg.Name] as List; - // _packagesFound has currentPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(currentPkg))) - { - TryAddToPackagesFound(currentPkg); - yield return currentPkg; + + + } } + // Console.WriteLine($"Returning from helper method"); + // return depPkgsFound; + } #endregion diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index b2c5353e3..acb26ca7d 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; using System.Collections; @@ -576,7 +577,7 @@ private List InstallPackages( packagesHash: new Hashtable(StringComparer.InvariantCultureIgnoreCase), errRecord: out ErrorRecord errRecord); - // At this point all packagea are installed to temp path. + // At this point all packages are installed to temp path. if (errRecord != null) { if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) @@ -681,21 +682,8 @@ private Hashtable BeginPackageInstall( default: // VersionType.NoVersion - - Stopwatch stopwatch = new Stopwatch(); - // Start measuring time for Find - stopwatch.Start(); - responses = currentServer.FindName(pkgNameToInstall, _prerelease, ResourceType.None, out ErrorRecord findNameErrRecord); - // Stop measuring time - stopwatch.Stop(); - - // Get the elapsed time in milliseconds - long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; - - Console.WriteLine($"Find Name - Elapsed Time: {elapsedMilliseconds} milliseconds"); - if (findNameErrRecord != null) { errRecord = findNameErrRecord; @@ -755,10 +743,6 @@ private Hashtable BeginPackageInstall( // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) if (!_reinstall) { - Stopwatch stopwatch = new Stopwatch(); - // Start measuring time - stopwatch.Start(); - string currPkgNameVersion = $"{pkgToInstall.Name}{pkgToInstall.Version}"; if (_packagesOnMachine.Contains(currPkgNameVersion)) { @@ -769,14 +753,6 @@ private Hashtable BeginPackageInstall( return packagesHash; } - - // Stop measuring time - stopwatch.Stop(); - - // Get the elapsed time in milliseconds - long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; - - Console.WriteLine($"Checking packages already installed on machine - Elapsed Time: {elapsedMilliseconds} milliseconds"); } if (packagesHash.ContainsKey(pkgToInstall.Name)) @@ -826,146 +802,143 @@ private Hashtable BeginPackageInstall( string pkgName = pkgToInstall.Name; if (!skipDependencyCheck) { - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.v3) - { - _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); - } - - var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); - _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); + List allDependencies = FindAllDependencies(currentServer, currentResponseUtil, pkgToInstall, repository); - // the last package added will be the parent package. - List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); + return InstallParentAndDependencyPackages(pkgToInstall, allDependencies, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); + } + else { + // TODO: check this version and prerelease combo + Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); - // allDependencies contains parent package as well - foreach (PSResourceInfo pkg in allDependencies) + if (installNameErrRecord != null) { - Console.WriteLine($"{pkg.Name}: {pkg.Version}"); + errRecord = installNameErrRecord; + return packagesHash; } - // List for keeping track of errors - List errors = new List(); - // If installing 5 or less packages, don't use Parallel.ForEach - if (allDependencies.Count > 5) + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + if (!installedToTempPathSuccessfully) { - PSResourceInfo parentPkg = allDependencies.Last(); - allDependencies.Remove(parentPkg); - - - // Error handling: https://learn.microsoft.com/en-us/dotnet/standard/parallel-programming/how-to-handle-exceptions-in-parallel-loops - // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) - _cmdletPassedIn.WriteDebug("**********************************************************Entering parallel foreach."); - Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, depPkg => - { - var depPkgName = depPkg.Name; - var depPkgVersion = depPkg.Version.ToString(); - // Perform some operation on each item - Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); - - Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); - - if (installNameErrRecord != null) - { - errors.Add(installNameErrRecord); - } + return packagesHash; + } + } + } - ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + return updatedPackagesHash; + } - if (!installedToTempPathSuccessfully) - { - errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); - } - }); + private List FindAllDependencies(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo pkgToInstall, PSRepositoryInfo repository) + { + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.v3) + { + _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); + } - // Early out if errors collected - if (errors.Count > 0) { - // Write out all errors collected from Parallel.ForEach - foreach (var err in errors) - { - Console.WriteLine(err); - } + var findHelper = new FindHelper(_cancellationToken, _cmdletPassedIn, _networkCredential); + _cmdletPassedIn.WriteDebug($"Finding dependency packages for '{pkgToInstall.Name}'"); - return packagesHash; - } + // the last package added will be the parent package. + List allDependencies = findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - // Install Parent Pkg - Stream responseStream = currentServer.InstallPackage(parentPkg.Name, parentPkg.Version.ToString(), true, out ErrorRecord installNameErrRecord); - if (installNameErrRecord != null) - { - errors.Add(installNameErrRecord); - return packagesHash; - } - - ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + // allDependencies contains parent package as well + foreach (PSResourceInfo pkg in allDependencies) + { + // Console.WriteLine($"{pkg.Name}: {pkg.Version}"); + } - if (!installedToTempPathSuccessfully) - { - errRecord = tempSaveErrRecord ?? tempInstallErrRecord; - return packagesHash; - } + return allDependencies; + } - return updatedPackagesHash; + private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, List allDependencies, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) + { + List errors = new List(); + // If installing more than 5 packages, do so concurrently + if (allDependencies.Count > 5) + { + // Set the maximum degree of parallelism to 32 (Invoke-Command has default of 32, that's where we got this number from) + Parallel.ForEach(allDependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, depPkg => + { + var depPkgName = depPkg.Name; + var depPkgVersion = depPkg.Version.ToString(); + // Console.WriteLine($"Processing number: {depPkg}, Thread ID: {Task.CurrentId}"); + Stream responseStream = currentServer.InstallPackage(depPkgName, depPkgVersion, true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + errors.Add(installNameErrRecord); } - else { - // Install the good, old fashioned way - foreach (var pkgToBeInstalled in allDependencies) - { - var pkgToInstallName = pkgToBeInstalled.Name; - var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); - Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); - - if (installNameErrRecord != null) - { - errRecord = installNameErrRecord; - return packagesHash; - } - ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, depPkgName, depPkgVersion, depPkg, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); - if (!installedToTempPathSuccessfully) - { - errRecord = tempSaveErrRecord ?? tempInstallErrRecord; - return packagesHash; - } - } + if (!installedToTempPathSuccessfully) + { + errors.Add(tempSaveErrRecord ?? tempInstallErrRecord); + } + }); - return updatedPackagesHash; + if (errors.Count > 0) + { + // Write out all errors collected from Parallel.ForEach + foreach (var err in errors) + { + _cmdletPassedIn.WriteError(err); } + return packagesHash; } - else { - // TODO: check this version and prerelease combo - Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgToInstall.Version.ToString(), true, out ErrorRecord installNameErrRecord); + // Install parent package + Stream responseStream = currentServer.InstallPackage(parentPkg.Name, parentPkg.Version.ToString(), true, out ErrorRecord installNameErrRecord); + if (installNameErrRecord != null) + { + _cmdletPassedIn.WriteError(installNameErrRecord); + return packagesHash; + } + + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, parentPkg.Name, parentPkg.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); + if (!installedToTempPathSuccessfully) + { + _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); + return packagesHash; + } + return updatedPackagesHash; + } + else + { + // Install the good old fashioned way + // Make sure to install dependencies first, then install parent pkg + allDependencies.Add(parentPkg); + foreach (var pkgToBeInstalled in allDependencies) + { + var pkgToInstallName = pkgToBeInstalled.Name; + var pkgToInstallVersion = pkgToBeInstalled.Version.ToString(); + Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); if (installNameErrRecord != null) { - errRecord = installNameErrRecord; + _cmdletPassedIn.WriteError(installNameErrRecord); return packagesHash; } - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgToInstall.Version.ToString(), pkgToInstall, packagesHash, out updatedPackagesHash, out errRecord); + ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; + bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempSaveErrRecord) : + TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, out tempInstallErrRecord); if (!installedToTempPathSuccessfully) { + _cmdletPassedIn.WriteError(tempSaveErrRecord ?? tempInstallErrRecord); return packagesHash; } - - //spackagesHash.Add(pkgToInstall.Name, pkgToInstall.Version); } - } - // TODO: figure out packagesHash / UpdatedPackagesHash - return updatedPackagesHash; + return updatedPackagesHash; + } } /// diff --git a/src/code/InstallPSResource.cs b/src/code/InstallPSResource.cs index ea75ddb38..6199405d5 100644 --- a/src/code/InstallPSResource.cs +++ b/src/code/InstallPSResource.cs @@ -281,12 +281,9 @@ protected override void ProcessRecord() { case NameParameterSet: Stopwatch stopwatch = new Stopwatch(); - // Start measuring time stopwatch.Start(); - - ProcessInstallHelper( pkgNames: Name, pkgVersion: Version, @@ -294,15 +291,12 @@ protected override void ProcessRecord() pkgRepository: Repository, pkgCredential: Credential, reqResourceParams: null); -; // Stop measuring time stopwatch.Stop(); - // Get the elapsed time in milliseconds long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; - - Console.WriteLine($"Total Elapsed Time: {elapsedMilliseconds} milliseconds"); + // Console.WriteLine($"Total Elapsed Time: {elapsedMilliseconds} milliseconds"); break; diff --git a/src/code/Utils.cs b/src/code/Utils.cs index 928468231..06da0b249 100644 --- a/src/code/Utils.cs +++ b/src/code/Utils.cs @@ -1205,7 +1205,8 @@ private static bool TryReadPSDataFile( } string contents = System.IO.File.ReadAllText(filePath); - // Create a new PowerShell runspace + // Parallel.ForEach calls into this method. + // Each thread needs its own runspace created to provide a separate environment for operations to run independently. Runspace runspace = RunspaceFactory.CreateRunspace(); runspace.Open(); runspace.SessionStateProxy.LanguageMode = PSLanguageMode.ConstrainedLanguage; @@ -1215,11 +1216,8 @@ private static bool TryReadPSDataFile( using (System.Management.Automation.PowerShell pwsh = System.Management.Automation.PowerShell.Create()) { - // Use the runspace in the PowerShell pipeline pwsh.Runspace = runspace; - // Invoke the pipeline and retrieve the results - var cmd = new Command( command: contents, isScript: true, @@ -1232,7 +1230,7 @@ private static bool TryReadPSDataFile( try { - // Invoke the script. + // Invoke the pipeline and retrieve the results var results = pwsh.Invoke(); if (results[0] is PSObject pwshObj) @@ -1270,7 +1268,6 @@ private static bool TryReadPSDataFile( error = ex; } } - // Close the runspace when done runspace.Close(); return false; diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index b7a992df5..5bdc03b6f 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -316,7 +316,7 @@ public override FindResults FindCommandOrDscResource(string[] tags, bool include /// public override FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindName()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindName()"); // Make sure to include quotations around the package name var prerelease = includePrerelease ? "IsAbsoluteLatestVersion" : "IsLatestVersion"; @@ -578,7 +578,7 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange /// public override FindResults FindVersion(string packageName, string version, ResourceType type, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersion()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersion()"); // https://www.powershellgallery.com/api/v2/FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion eq '1.1.0' and substringof('PSModule', Tags) eq true // Quotations around package name and version do not matter, same metadata gets returned. // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. @@ -595,7 +595,7 @@ public override FindResults FindVersion(string packageName, string version, Reso } int count = GetCountFromResponse(response, out errRecord); - _cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); + //_cmdletPassedIn.WriteDebug($"Count from response is '{count}'"); if (errRecord != null) { @@ -697,13 +697,13 @@ public override Stream InstallPackage(string packageName, string packageVersion, /// private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::HttpRequestCall()"); errRecord = null; string response = string.Empty; try { - _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); + // _cmdletPassedIn.WriteDebug($"Request url is '{requestUrlV2}'"); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrlV2); response = SendV2RequestAsync(request, _sessionClient).GetAwaiter().GetResult(); @@ -743,7 +743,7 @@ private string HttpRequestCall(string requestUrlV2, out ErrorRecord errRecord) if (string.IsNullOrEmpty(response)) { - _cmdletPassedIn.WriteDebug("Response is empty"); + // _cmdletPassedIn.WriteDebug("Response is empty"); } return response; @@ -1042,7 +1042,7 @@ private string FindNameGlobbingWithTag(string packageName, string[] tags, Resour /// private string FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, int skip, bool getOnlyLatest, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); + //_cmdletPassedIn.WriteDebug("In V2ServerAPICalls::FindVersionGlobbing()"); //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' // NormalizedVersion doesn't include trailing zeroes @@ -1254,7 +1254,7 @@ public int GetCountFromResponse(string httpResponse, out ErrorRecord errRecord) } else { - _cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); + //_cmdletPassedIn.WriteDebug($"Property 'count' and 'd:Id' could not be found in response. This may indicate that the package could not be found"); } } } From c738f900e34b3da124f73095904b209bdd788de2 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:48:18 -0800 Subject: [PATCH 11/14] Add some refactoring --- src/code/ACRServerAPICalls.cs | 7 +- src/code/FindHelper.cs | 845 ++++++++++------------------------ 2 files changed, 248 insertions(+), 604 deletions(-) diff --git a/src/code/ACRServerAPICalls.cs b/src/code/ACRServerAPICalls.cs index 7ae8c4ed6..a46dba181 100644 --- a/src/code/ACRServerAPICalls.cs +++ b/src/code/ACRServerAPICalls.cs @@ -538,7 +538,6 @@ public override FindResults FindVersionWithTag(string packageName, string versio /// public override Stream InstallPackage(string packageName, string packageVersion, bool includePrerelease, out ErrorRecord errRecord) { - //_cmdletPassedIn.WriteDebug("In ACRServerAPICalls::InstallPackage()"); Stream results = new MemoryStream(); if (string.IsNullOrEmpty(packageVersion)) { @@ -555,12 +554,8 @@ public override Stream InstallPackage(string packageName, string packageVersion, return results; } - private Stream InstallVersion( - string moduleName, - string moduleVersion, - out ErrorRecord errRecord) + private Stream InstallVersion(string moduleName, string moduleVersion, out ErrorRecord errRecord) { - errRecord = null; string accessToken = string.Empty; string tenantID = string.Empty; string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 19ef9fea4..620573a5a 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -2,14 +2,13 @@ // Licensed under the MIT License. using Microsoft.PowerShell.PSResourceGet.UtilClasses; +using NuGet.Protocol.Core.Types; using NuGet.Versioning; using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Management.Automation; using System.Net; -using System.Runtime.ExceptionServices; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; @@ -1043,671 +1042,321 @@ private string FormatPkgVersionString(PSResourceInfo pkg) #region Internal Client Search Methods - - internal IEnumerable FindDependencyPackages( - ServerApiCall currentServer, - ResponseUtil currentResponseUtil, - PSResourceInfo currentPkg, - PSRepositoryInfo repository) - + internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { depPkgsFound = new List(); FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); - // Console.WriteLine($"returned to parent method "); - foreach (var item in depPkgsFound) - { - // Console.WriteLine(item.Name); - } - foreach (var item in depPkgsFound) + foreach (var pkg in depPkgsFound) { - if (!_packagesFound.ContainsKey(item.Name)) + if (!_packagesFound.ContainsKey(pkg.Name)) { - TryAddToPackagesFound(item); - yield return item; + TryAddToPackagesFound(pkg); + yield return pkg; } else { - List pkgVersions = _packagesFound[item.Name]; + List pkgVersions = _packagesFound[pkg.Name]; // _packagesFound has item.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(item))) + if (!pkgVersions.Contains(FormatPkgVersionString(pkg))) { - TryAddToPackagesFound(item); + TryAddToPackagesFound(pkg); - yield return item; + yield return pkg; } } } } - - - internal void FindDependencyPackagesHelper( - ServerApiCall currentServer, - ResponseUtil currentResponseUtil, - PSResourceInfo currentPkg, - PSRepositoryInfo repository) + internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { List errors = new List(); - if (currentPkg.Dependencies.Length > 0) { - foreach (var depend in currentPkg.Dependencies) - { - - // Console.WriteLine($"Dep Name: {depend.Name}, DepVersion Min: {depend.VersionRange.MinVersion} DepVersion Max: {depend.VersionRange.MaxVersion}"); - } - - // Add parallel.foreach here // If installing more than 5 packages, do so concurrently if (currentPkg.Dependencies.Length > 5) { - Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = 32 }, dep => { - //// Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); - PSResourceInfo depPkg = null; + // Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); - if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) - { - // // Console.WriteLine($"++++++++++++++DepPkg Name is {dep.Name}"); - // // Console.WriteLine($"+++++++++++++MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); - // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. - if (_knownLatestPkgVersion.Contains(dep.Name)) - { - // // Console.WriteLine($"{dep.Name} already has the known latest version, so we don't need to find it"); - // "return" - } - else - { - // // Console.WriteLine("BEfore FindName"); - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - // // Console.WriteLine("After FindName"); - - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - } - // "return" - - if (responses != null) - { - // // Console.WriteLine("Before ConvertToPSResourceResult"); - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - // // Console.WriteLine("After FindName"); - - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - depPkg = currentResult.returnedObject; - if (!depPkgsFound.Contains(depPkg)) - { - _knownLatestPkgVersion.Add(depPkg.Name); - // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 1"); - - // add pkg then find dependencies - depPkgsFound.Add(depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - // // Console.WriteLine("After recursive FindDependencyPackagesHelper 1"); - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); - - } - } - } - } - } - } - else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) - { - //// Console.WriteLine($"*****************DepPkg Name is {dep.Name}"); - //// Console.WriteLine($"*****************MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); + }); - //// Console.WriteLine("Before minVersion FindVersion"); + // todo: write any errors here? + } + else + { + foreach (var dep in currentPkg.Dependencies) + { + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); + }g - FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); - // // Console.WriteLine("After minVersion FindVersion"); + // todo write out errors here + } + } + } - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - // return null; - } - else - { - // // Console.WriteLine("Before min version ConvertToPSResourceResult"); - - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - // // Console.WriteLine("After min version ConvertToPSResourceResult"); - - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - //return null; - } - else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - //return null; - } - else - { - depPkg = currentResult.returnedObject; - if (!depPkgsFound.Contains(depPkg)) - { - // // Console.WriteLine("Before min version FindDependencyPackagesHelper"); - - // add pkg then find dependencies - depPkgsFound.Add(depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - // // Console.WriteLine("After min version FindDependencyPackagesHelper"); - - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); - - } - } - } - } + + private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) + { + FindDependencyWithNoUpperBound(); + } + else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) + { + FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } + else + { + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { - // // Console.WriteLine("Before FindVersionGlobbing"); - - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); - // // Console.WriteLine("After FindVersionGlobbing"); - - if (errRecord != null) + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - //return - } + // // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); - if (responses.IsFindResultsEmpty()) - { - errors.Add(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); - // return null; - } - else - { - // // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult"); - - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) - { - // // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult in for each loop"); - - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - - //return null; - } - else - { - // Check to see if version falls within version range - PSResourceInfo foundDep = currentResult.returnedObject; - string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) - { - depVersionStr += $"-{foundDep.Prerelease}"; - } - - if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) - && dep.VersionRange.Satisfies(depVersion)) - { - // // Console.WriteLine($"**** Dependency package '{foundDep.Name}', version '{foundDep.Version}' saved as new depPkg"); - - depPkg = foundDep; - break; - } - - // // Console.WriteLine($"Dependency package '{foundDep.Name}', version '{foundDep.Version}'"); - } - } - - if (depPkg == null) - { - // // Console.WriteLine($"depPkg is null and I don't know what this means"); - } - else - { - if (!depPkgsFound.Contains(depPkg)) - { - // // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); - - // add pkg then find dependencies - depPkgsFound.Add(depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - - // // Console.WriteLine($" _packagesFound has ${depPkg.Name} in it, but the version '{depPkg.Version}' is not the same '{string.Join(",", pkgVersions)}'"); - - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - } - } - } } } + } + } + } + else + { + FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); + if (errRecord != null) + { + errors = ProcessErrorRecord(errRecord, errors); + } - - }); - - - - - + if (responses.IsFindResultsEmpty()) + { + errors.Add(new ErrorRecord( + new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), + "FindDepPackagesFindVersionGlobbingFailure", + ErrorCategory.InvalidResult, + this)); } else { - // Install the good old fashioned way - - // Console.WriteLine($"SHOUL DNOT GET HEREE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - - - foreach (var dep in currentPkg.Dependencies) + foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) { - // Console.WriteLine($"FindDependencyPackages Processing number: {dep}, Thread ID: {Task.CurrentId}"); - PSResourceInfo depPkg = null; - - if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) + if (currentResult == null || currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else { - // Console.WriteLine($"++++++++++++++DepPkg Name is {dep.Name}"); - // Console.WriteLine($"+++++++++++++MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); - // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. - if (_knownLatestPkgVersion.Contains(dep.Name)) + // Check to see if version falls within version range + PSResourceInfo foundDep = currentResult.returnedObject; + string depVersionStr = $"{foundDep.Version}"; + if (foundDep.IsPrerelease) { - // Console.WriteLine($"{dep.Name} already has the known latest version, so we don't need to find it"); - // "return" + depVersionStr += $"-{foundDep.Prerelease}"; } - else + + if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) + && dep.VersionRange.Satisfies(depVersion)) { - // Console.WriteLine("BEfore FindName"); - FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); - // Console.WriteLine("After FindName"); - - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - } - // "return" - - if (responses != null) - { - // Console.WriteLine("Before ConvertToPSResourceResult"); - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - // Console.WriteLine("After FindName"); - - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - depPkg = currentResult.returnedObject; - if (!depPkgsFound.Contains(depPkg)) - { - _knownLatestPkgVersion.Add(depPkg.Name); - // Console.WriteLine("Before recursive FindDependencyPackagesHelper 1"); - - // add pkg then find dependencies - depPkgsFound.Add(depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - // Console.WriteLine("After recursive FindDependencyPackagesHelper 1"); - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); - - } - } - } - } + depPkg = foundDep; + break; } } - else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) - { - // Console.WriteLine($"*****************DepPkg Name is {dep.Name}"); - // Console.WriteLine($"*****************MIN VERSION is {dep.VersionRange.MinVersion} MAX VERSION IS : {dep.VersionRange.MaxVersion}"); - - // Console.WriteLine("Before minVersion FindVersion"); + } - FindResults responses = currentServer.FindVersion(dep.Name, dep.VersionRange.MaxVersion.ToString(), _type, out ErrorRecord errRecord); - // Console.WriteLine("After minVersion FindVersion"); + if (depPkg == null) + { + // // Console.WriteLine($"depPkg is null and I don't know what this means"); + } + else + { + if (!depPkgsFound.Contains(depPkg)) + { + // // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - // return null; - } - else - { - // Console.WriteLine("Before min version ConvertToPSResourceResult"); - - PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); - // Console.WriteLine("After min version ConvertToPSResourceResult"); - - if (currentResult == null) - { - // This scenario may occur when the package version requested is unlisted. - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - //return null; - } - else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - //return null; - } - else - { - depPkg = currentResult.returnedObject; - if (!depPkgsFound.Contains(depPkg)) - { - // Console.WriteLine("Before min version FindDependencyPackagesHelper"); - - // add pkg then find dependencies - depPkgsFound.Add(depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - // Console.WriteLine("After min version FindDependencyPackagesHelper"); - - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // Console.WriteLine("Before min version FindDependencyPackagesHelper 2"); - - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - // Console.WriteLine("After min version FindDependencyPackagesHelper 2"); - - } - } - } - } + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } else { - // Console.WriteLine("Before FindVersionGlobbing"); - - FindResults responses = currentServer.FindVersionGlobbing(dep.Name, dep.VersionRange, includePrerelease: true, ResourceType.None, getOnlyLatest: true, out ErrorRecord errRecord); - // Console.WriteLine("After FindVersionGlobbing"); - - if (errRecord != null) - { - if (errRecord.Exception is ResourceNotFoundException) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - else - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - } - //return - } + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same - if (responses.IsFindResultsEmpty()) + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) { - errors.Add(new ErrorRecord( - new InvalidOrEmptyResponse($"Dependency package with name {dep.Name} and version range {dep.VersionRange} could not be found in repository '{repository.Name}"), - "FindDepPackagesFindVersionGlobbingFailure", - ErrorCategory.InvalidResult, - this)); - // return null; - } - else - { - // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult"); - - foreach (PSResourceResult currentResult in currentResponseUtil.ConvertToPSResourceResult(responses)) - { - // Console.WriteLine("Before FindVersionGlobbing ConvertToPSResourceResult in for each loop"); - - if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) - { - errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' and version range '{dep.VersionRange}' could not be found in repository '{repository.Name}'", currentResult.exception), - "DependencyPackageNotFound", - ErrorCategory.ObjectNotFound, - this)); - - //return null; - } - else - { - // Check to see if version falls within version range - PSResourceInfo foundDep = currentResult.returnedObject; - string depVersionStr = $"{foundDep.Version}"; - if (foundDep.IsPrerelease) - { - depVersionStr += $"-{foundDep.Prerelease}"; - } - - if (NuGetVersion.TryParse(depVersionStr, out NuGetVersion depVersion) - && dep.VersionRange.Satisfies(depVersion)) - { - // Console.WriteLine($"**** Dependency package '{foundDep.Name}', version '{foundDep.Version}' saved as new depPkg"); - - depPkg = foundDep; - break; - } - - // Console.WriteLine($"Dependency package '{foundDep.Name}', version '{foundDep.Version}'"); - } - } - - if (depPkg == null) - { - // Console.WriteLine($"depPkg is null and I don't know what this means"); - } - else - { - if (!depPkgsFound.Contains(depPkg)) - { - // Console.WriteLine($"PackagesFound contains {depPkg.Name}"); - - // add pkg then find dependencies - depPkgsFound.Add(depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); - } - else - { - List pkgVersions = _packagesFound[depPkg.Name] as List; - // _packagesFound has depPkg.name in it, but the version is not the same - - // Console.WriteLine($" _packagesFound has ${depPkg.Name} in it, but the version '{depPkg.Version}' is not the same '{string.Join(",", pkgVersions)}'"); - - if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) - { - // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); - // add pkg then find dependencies - // for now depPkgsFound.Add(depPkg); - // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); - } - } - } + // // Console.WriteLine($"pkgVersions does not contain {FormatPkgVersionString(depPkg)}"); + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); } } + } + } + } + } - + private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) + { + PSResourceInfo depPkg = null; + // _knownLatestPkgVersion will tell us if we already have the latest version available for a particular package. + if (!_knownLatestPkgVersion.Contains(dep.Name)) + { + FindResults responses = currentServer.FindName(dep.Name, includePrerelease: true, _type, out ErrorRecord errRecord); + if (errRecord != null) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + // todo add error here? } + return depPkg; + } + if (responses == null) + { + // todo error + return depPkg; + } + PSResourceResult currentResult = currentResponseUtil.ConvertToPSResourceResult(responses).FirstOrDefault(); + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + return depPkg; + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + return depPkg; + } + + depPkg = currentResult.returnedObject; + if (!depPkgsFound.Contains(depPkg)) + { + _knownLatestPkgVersion.Add(depPkg.Name); + // add pkg then find dependencies + depPkgsFound.Add(depPkg); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); } + else + { + List pkgVersions = _packagesFound[depPkg.Name] as List; + // _packagesFound has depPkg.name in it, but the version is not the same + if (!pkgVersions.Contains(FormatPkgVersionString(depPkg))) + { + // // Console.WriteLine("Before recursive FindDependencyPackagesHelper 2"); + + // add pkg then find dependencies + // for now depPkgsFound.Add(depPkg); + // for now depPkgsFound.AddRange(FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository)); + // // Console.WriteLine("After recursive FindDependencyPackagesHelper 2"); + + } + } + } + } + } + + + private List ProcessErrorRecord(ErrorRecord errRecord, List errors) + { + if (errRecord.Exception is ResourceNotFoundException) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}': {errRecord.Exception.Message}"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); } - // Console.WriteLine($"Returning from helper method"); - // return depPkgsFound; + return errors; + } + private List ProcessPSResourceResult(PSResourceResult currentResult, List errors) + { + if (currentResult == null) + { + // This scenario may occur when the package version requested is unlisted. + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) + { + errors.Add(new ErrorRecord( + new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + "DependencyPackageNotFound", + ErrorCategory.ObjectNotFound, + this)); + } + + return errors; } - #endregion - } + #endregion +} } From b3683208795928e9cd212d02f10caa577d47ee37 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Wed, 8 May 2024 15:37:42 -0700 Subject: [PATCH 12/14] Quick updates --- src/code/FindHelper.cs | 28 +++++++++++-------- src/code/InstallHelper.cs | 5 +++- src/code/InstallPSResource.cs | 10 +++---- .../Microsoft.PowerShell.PSResourceGet.csproj | 12 ++++---- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 620573a5a..ec74f0a18 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -36,6 +36,9 @@ internal class FindHelper private bool _includeDependencies = false; private bool _repositoryNameContainsWildcard = true; private NetworkCredential _networkCredential; + + // TODO: Update to be concurrency safe; TryAdd needs to be updates as well + // TODO: look at # of allocations (lists, etc.) private Dictionary> _packagesFound; private HashSet _knownLatestPkgVersion; List depPkgsFound; @@ -245,7 +248,7 @@ public IEnumerable FindByResourceName( this)); } } -} + } public IEnumerable FindByCommandOrDscResource( bool isSearchingForCommands, @@ -1068,6 +1071,7 @@ internal IEnumerable FindDependencyPackages(ServerApiCall curren } } + // Method 2 internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { List errors = new List(); @@ -1083,27 +1087,27 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response }); - // todo: write any errors here? + // todo: write any errors here } else { foreach (var dep in currentPkg.Dependencies) { FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); - }g + } // todo write out errors here } } } - + // Method 3 private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) { PSResourceInfo depPkg = null; if (dep.VersionRange.Equals(VersionRange.All) || !dep.VersionRange.HasUpperBound) { - FindDependencyWithNoUpperBound(); + FindDependencyWithNoUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errors); } else if (dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) { @@ -1229,6 +1233,7 @@ private void FindDependencyPackageVersion(Dependency dep, ServerApiCall currentS } } + // Method 4 private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, List errors) { PSResourceInfo depPkg = null; @@ -1307,8 +1312,9 @@ private PSResourceInfo FindDependencyWithNoUpperBound(Dependency dep, ServerApiC } } - } } + + return depPkg; } @@ -1317,7 +1323,7 @@ private List ProcessErrorRecord(ErrorRecord errRecord, List ProcessErrorRecord(ErrorRecord errRecord, List ProcessPSResourceResult(PSResourceResult currentResult { // This scenario may occur when the package version requested is unlisted. errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'"), + new ResourceNotFoundException($"Dependency package with name '{currentResult.returnedObject.Name}' could not be found in repository '{currentResult.returnedObject.Repository}'."), "DependencyPackageNotFound", ErrorCategory.ObjectNotFound, this)); @@ -1348,7 +1354,7 @@ private List ProcessPSResourceResult(PSResourceResult currentResult else if (currentResult.exception != null && !currentResult.exception.Message.Equals(string.Empty)) { errors.Add(new ErrorRecord( - new ResourceNotFoundException($"Dependency package with name '{dep.Name}' could not be found in repository '{repository.Name}'", currentResult.exception), + new ResourceNotFoundException($"Dependency package with name '{currentResult.returnedObject.Name}' could not be found in repository '{currentResult.returnedObject.Repository}'", currentResult.exception), "DependencyPackageNotFound", ErrorCategory.ObjectNotFound, this)); @@ -1358,5 +1364,5 @@ private List ProcessPSResourceResult(PSResourceResult currentResult } #endregion -} + } } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index acb26ca7d..20143b2e2 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -798,6 +798,7 @@ private Hashtable BeginPackageInstall( } else { + // Concurrently Updates // Find all dependencies string pkgName = pkgToInstall.Name; if (!skipDependencyCheck) @@ -829,6 +830,7 @@ private Hashtable BeginPackageInstall( return updatedPackagesHash; } + // Concurrency Updates private List FindAllDependencies(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo pkgToInstall, PSRepositoryInfo repository) { if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.v3) @@ -851,6 +853,7 @@ private List FindAllDependencies(ServerApiCall currentServer, Re return allDependencies; } + // Concurrently Updates private Hashtable InstallParentAndDependencyPackages(PSResourceInfo parentPkg, List allDependencies, ServerApiCall currentServer, string tempInstallPath, Hashtable packagesHash, Hashtable updatedPackagesHash, PSResourceInfo pkgToInstall) { List errors = new List(); @@ -1351,7 +1354,7 @@ private bool TryMoveInstallContent(string tempInstallPath, ScopeType scope, Hash } } } - catch (Exception e) + catch (Exception) { /*_cmdletPassedIn.WriteError(new ErrorRecord( new PSInvalidOperationException( diff --git a/src/code/InstallPSResource.cs b/src/code/InstallPSResource.cs index 6199405d5..49f5cefb9 100644 --- a/src/code/InstallPSResource.cs +++ b/src/code/InstallPSResource.cs @@ -280,9 +280,9 @@ protected override void ProcessRecord() switch (ParameterSetName) { case NameParameterSet: - Stopwatch stopwatch = new Stopwatch(); + // Stopwatch stopwatch = new Stopwatch(); // Start measuring time - stopwatch.Start(); + // stopwatch.Start(); ProcessInstallHelper( pkgNames: Name, @@ -293,10 +293,10 @@ protected override void ProcessRecord() reqResourceParams: null); // Stop measuring time - stopwatch.Stop(); + // stopwatch.Stop(); // Get the elapsed time in milliseconds - long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; - // Console.WriteLine($"Total Elapsed Time: {elapsedMilliseconds} milliseconds"); + // long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; + // Console.WriteLine($"Total Elapsed Time: {elapsedMilliseconds} milliseconds"); break; diff --git a/src/code/Microsoft.PowerShell.PSResourceGet.csproj b/src/code/Microsoft.PowerShell.PSResourceGet.csproj index ca3a7d8e2..8c7caa15b 100644 --- a/src/code/Microsoft.PowerShell.PSResourceGet.csproj +++ b/src/code/Microsoft.PowerShell.PSResourceGet.csproj @@ -14,12 +14,12 @@ - - - - - - + + + + + + From 7291ddc17262fa4148437570ef9c649d4b22553b Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Wed, 8 May 2024 15:41:50 -0700 Subject: [PATCH 13/14] Clean up installPSResource --- src/code/InstallPSResource.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/code/InstallPSResource.cs b/src/code/InstallPSResource.cs index 49f5cefb9..f92a56574 100644 --- a/src/code/InstallPSResource.cs +++ b/src/code/InstallPSResource.cs @@ -6,7 +6,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Management.Automation; @@ -280,10 +279,6 @@ protected override void ProcessRecord() switch (ParameterSetName) { case NameParameterSet: - // Stopwatch stopwatch = new Stopwatch(); - // Start measuring time - // stopwatch.Start(); - ProcessInstallHelper( pkgNames: Name, pkgVersion: Version, @@ -292,12 +287,6 @@ protected override void ProcessRecord() pkgCredential: Credential, reqResourceParams: null); - // Stop measuring time - // stopwatch.Stop(); - // Get the elapsed time in milliseconds - // long elapsedMilliseconds = stopwatch.ElapsedMilliseconds; - // Console.WriteLine($"Total Elapsed Time: {elapsedMilliseconds} milliseconds"); - break; case InputObjectParameterSet: From d37ca5d0c7527118b66338289361706aff605783 Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Wed, 15 May 2024 18:10:32 -0700 Subject: [PATCH 14/14] Merge from upstream, build fixes --- src/code/InstallHelper.cs | 3 ++- src/code/ServerApiCall.cs | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index a0344afc2..f8cade695 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -644,6 +644,7 @@ private List InstallPackages( currentServer: currentServer, currentResponseUtil: currentResponseUtil, tempInstallPath: tempInstallPath, + skipDependencyCheck: skipDependencyCheck, packagesHash: packagesHash, errRecord: out ErrorRecord installPkgErrRecord); @@ -907,7 +908,7 @@ private Hashtable BeginPackageInstall( // Concurrency Updates private List FindAllDependencies(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo pkgToInstall, PSRepositoryInfo repository) { - if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.v3) + if (currentServer.Repository.ApiVersion == PSRepositoryInfo.APIVersion.V3) { _cmdletPassedIn.WriteWarning("Installing dependencies is not currently supported for V3 server protocol repositories. The package will be installed without installing dependencies."); } diff --git a/src/code/ServerApiCall.cs b/src/code/ServerApiCall.cs index 9f8bf4fe9..13e274fb1 100644 --- a/src/code/ServerApiCall.cs +++ b/src/code/ServerApiCall.cs @@ -10,7 +10,6 @@ using System.Text; using System.Runtime.ExceptionServices; using System.Management.Automation; -using System; namespace Microsoft.PowerShell.PSResourceGet.Cmdlets {