From 78c45310b54d40d205a241fa08f4b2c5ecc46645 Mon Sep 17 00:00:00 2001 From: MerlynOMsft <44985659+merlynomsft@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:14:49 -0700 Subject: [PATCH] Update build pipeline to support LocalPackage build config (parameterized; not enabled by default) (#20552) --- BuildConfigGen/GitUtil.cs | 24 --------- BuildConfigGen/Program.cs | 74 +++++++++++++++++++++++----- BuildConfigGen/dev.sh | 14 ------ azure-pipelines.yml | 15 +++++- ci/build-all-steps.yml | 2 +- ci/build-all-tasks.yml | 4 +- ci/build-single-steps.yml | 2 +- ci/ci-test-tasks/build-task.yml | 2 +- ci/ci-test-tasks/canary-tests-v2.yml | 18 +++++++ make-util.js | 16 +++--- make.js | 38 ++++++++++---- 11 files changed, 134 insertions(+), 75 deletions(-) diff --git a/BuildConfigGen/GitUtil.cs b/BuildConfigGen/GitUtil.cs index f4863e92228e..2aad907bf27b 100644 --- a/BuildConfigGen/GitUtil.cs +++ b/BuildConfigGen/GitUtil.cs @@ -95,30 +95,6 @@ public static void GetUntrackedFiles(string taskTarget, out IEnumerable toAdd = untrackedOuput2; toRemove = untrackedOuputRemove; } - internal static string GetGitRootPath(string currentDir) - { - const string args = "rev-parse --git-dir"; - string path = RunGitCommandScalar(currentDir, args); - - path = FixupPath(path); - - const string gitDir = ".git"; - if (path.EndsWith(gitDir)) - { - path = path.Substring(0, path.Length - gitDir.Length); - - if (path == "") - { - return currentDir; - } - - return path; - } - else - { - throw new Exception($"expected git {args} to return "); - } - } internal static IEnumerable GetNonIgnoredFileListFromPath(string gitRoot, string taskTarget) { diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs index b8e9f338a557..513a5f2b1b88 100644 --- a/BuildConfigGen/Program.cs +++ b/BuildConfigGen/Program.cs @@ -109,7 +109,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN } string currentDir = Environment.CurrentDirectory; - string gitRootPath = GitUtil.GetGitRootPath(currentDir); + string gitRootPath = GetTasksRootPath(currentDir); string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt"); TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath); @@ -208,7 +208,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN Console.WriteLine($"Global version: maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1"); } - Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}" ); + Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}"); } else { @@ -249,6 +249,14 @@ private static void MainInner(string? task, string? configs, int? currentSprintN } } } + else + { + // if we're not updating local packages, we need to ensure the global version is updated to the task major version. existing patch number is preserved + if (globalVersion is not null) + { + globalVersion = globalVersion.CloneWithMajor(taskMajorVersion); + } + } if (globalVersion is not null) { @@ -265,7 +273,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false); } - ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError("(global)", skipContentCheck: false); + ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(null, skipContentCheck: false); foreach (var t in tasks) { @@ -284,6 +292,37 @@ private static void MainInner(string? task, string? configs, int? currentSprintN } } + private static string GetTasksRootPath(string inputCurrentDir) + { + string? currentDir = inputCurrentDir; + string? tasksRootPath = null; + + do + { + string currentDirGit = Path.Combine(currentDir, ".git"); + + if (Directory.Exists(currentDirGit)) + { + tasksRootPath = currentDir; + } + + currentDir = (new DirectoryInfo(currentDir)).Parent?.FullName; + + } while (currentDir != null); + + if (tasksRootPath == null) + { + throw new Exception($"could not find .git in {currentDir}"); + } + + if (!File.Exists(Path.Combine(tasksRootPath, "make-options.json"))) + { + throw new Exception($"make-options.json not found in tasksRootPath={tasksRootPath}"); + } + + return tasksRootPath; + } + private static IEnumerable FilterConfigsForTask(string? configs, KeyValuePair t) { var configsList = t.Value.Configs.AsEnumerable(); @@ -343,7 +382,7 @@ private static void GetVersions(string task, string configsString, out List<(str string currentDir = Environment.CurrentDirectory; - string gitRootPath = GitUtil.GetGitRootPath(currentDir); + string gitRootPath = GetTasksRootPath(currentDir); string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); if (!Directory.Exists(taskTargetPath)) @@ -419,7 +458,7 @@ private static int GetCurrentSprint() return currentSprint; } - private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string task, bool skipContentCheck) + private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string? task, bool skipContentCheck) { // if !writeUpdates, error if we have written any updates var verifyErrors = ensureUpdateModeVerifier!.GetVerifyErrors(skipContentCheck).ToList(); @@ -433,7 +472,14 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr Console.WriteLine(s); } - throw new Exception($"Updates needed, please run npm make.js --task {task} "); + if (task is null) + { + throw new Exception($"Updates needed, please run node make.js"); + } + else + { + throw new Exception($"Updates needed, please run node make.js --task {task} "); + } } } @@ -459,8 +505,8 @@ private static void MainUpdateTask( try { string currentDir = Environment.CurrentDirectory; - string gitRootPath = GitUtil.GetGitRootPath(currentDir); - string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder); + string gitRootPath = GetTasksRootPath(currentDir); + string versionMapFile = GetVersionMapFile(task, generatedFolder); string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); if (!Directory.Exists(taskTargetPath)) @@ -489,7 +535,11 @@ private static void MainUpdateTask( foreach (var config in targetConfigs) { - if (config.useGlobalVersion && !hasGlobalVersion) + if (config.useGlobalVersion && !includeLocalPackagesBuildConfig) + { + Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c --include-local-packages-build-config. not specified. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion} includeLocalPackagesBuildConfig={includeLocalPackagesBuildConfig}"); + } + else if (config.useGlobalVersion && !hasGlobalVersion) { Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c GlobalVersion not initialized. (to opt-in and start producing LocalBuildConfig, run with --include-local-packages-build-config. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion}). Note: this is not an error!"); } @@ -608,7 +658,7 @@ private static void MainUpdateTask( } } - private static string GetVersionMapFile(string task, string gitRootPath, string generatedFolder) + private static string GetVersionMapFile(string task, string generatedFolder) { return Path.Combine(generatedFolder, @$"{task}.versionmap.txt"); } @@ -1078,7 +1128,7 @@ private static void CopyConfig(string gitRootPath, string taskTargetPathOrUnders private static void UpdateVersionsForTask(string task, TaskStateStruct taskState, HashSet targetConfigs, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, string generatedFolder) { string currentDir = Environment.CurrentDirectory; - string gitRootPath = GitUtil.GetGitRootPath(currentDir); + string gitRootPath = GetTasksRootPath(currentDir); string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task); if (!Directory.Exists(taskTargetPath)) @@ -1093,7 +1143,7 @@ private static void UpdateVersionsForTask(string task, TaskStateStruct taskState bool defaultVersionMatchesSourceVersion; - string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder); + string versionMapFile = GetVersionMapFile(task, generatedFolder); { TaskVersion? defaultVersion = null; diff --git a/BuildConfigGen/dev.sh b/BuildConfigGen/dev.sh index f2138539f3ad..33e82b2bbffb 100755 --- a/BuildConfigGen/dev.sh +++ b/BuildConfigGen/dev.sh @@ -36,17 +36,6 @@ function detect_platform_and_runtime_id () fi } -function cmd_build () -{ - heading "Building" - dotnet build -o bin $SOLUTION_PATH || failed build - #change execution flag to allow running with sudo - if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then - chmod +x "bin/BuildConfigGen" - fi - -} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" pushd "$SCRIPT_DIR" source "$SCRIPT_DIR/Misc/helpers.sh" @@ -91,6 +80,3 @@ echo "Adding .NET to PATH ${DOTNETSDK_INSTALLDIR}" export PATH=${DOTNETSDK_INSTALLDIR}:$PATH echo "Path = $PATH" echo ".NET Version = $(dotnet --version)" - -SOLUTION_PATH="$SCRIPT_DIR/BuildConfigGen.sln" -cmd_build diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 69ef2790f16b..ef1c7dbd1a27 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,6 +29,10 @@ parameters: displayName: Enable CodeQL for run type: boolean default: false +- name: includeLocalPackagesBuildConfig + displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later) + type: boolean + default: false # note: keep in sync with ci\ci-test-tasks\canary-tests-v2.yml variables: - name: currentDate @@ -41,7 +45,16 @@ variables: value: ${{ eq(parameters.enableCodeQL, true) }} - name: system.debug value: true - +- name: includeLocalPackagesBuildConfigParameter + ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: + value: '--includeLocalPackagesBuildConfig' + ${{ else }}: + value: '' +- name: IncludeLocalPackagesBuildConfigTest + ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: + value: '1' + ${{ else }}: + value: '' extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates diff --git a/ci/build-all-steps.yml b/ci/build-all-steps.yml index 645f824afd78..15e9a016e816 100644 --- a/ci/build-all-steps.yml +++ b/ci/build-all-steps.yml @@ -106,7 +106,7 @@ steps: displayName: Clean tasks # Build Tasks -- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)" +- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)" $(includeLocalPackagesBuildConfigParameter) displayName: Build Tasks condition: and(succeeded(), ne(variables['numTasks'], 0)) diff --git a/ci/build-all-tasks.yml b/ci/build-all-tasks.yml index a5e1d7b10f84..feb623e19ea3 100644 --- a/ci/build-all-tasks.yml +++ b/ci/build-all-tasks.yml @@ -30,12 +30,12 @@ steps: displayName: Clean tasks # Build tasks -- script: node make.js serverBuild --task "$(task_pattern)" +- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter) displayName: Build tasks using pattern condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', false)) # Build all tasks -- script: node make.js serverBuild +- script: node make.js serverBuild $(includeLocalPackagesBuildConfigParameter) displayName: Build all tasks condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', true)) diff --git a/ci/build-single-steps.yml b/ci/build-single-steps.yml index e994fed8d15b..7b77d9bcc682 100644 --- a/ci/build-single-steps.yml +++ b/ci/build-single-steps.yml @@ -50,7 +50,7 @@ steps: displayName: Clean tasks # Build Tasks -- script: node make.js serverBuild --task "$(task_pattern)" +- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter) displayName: Build Tasks # Check diff for task sources diff --git a/ci/ci-test-tasks/build-task.yml b/ci/ci-test-tasks/build-task.yml index 84107a088c13..efb835ff0fab 100644 --- a/ci/ci-test-tasks/build-task.yml +++ b/ci/ci-test-tasks/build-task.yml @@ -22,7 +22,7 @@ steps: displayName: Clean tasks # Build -- script: node make.js serverBuild --task "${{ parameters.task }}" +- script: node make.js serverBuild --task "${{ parameters.task }}" $(includeLocalPackagesBuildConfigParameter) displayName: ${{ parameters.task }} build # Check diff for task sources diff --git a/ci/ci-test-tasks/canary-tests-v2.yml b/ci/ci-test-tasks/canary-tests-v2.yml index a1f8a95d082d..945f0fb4dd02 100644 --- a/ci/ci-test-tasks/canary-tests-v2.yml +++ b/ci/ci-test-tasks/canary-tests-v2.yml @@ -1,3 +1,21 @@ +parameters: +- name: includeLocalPackagesBuildConfig + displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later) + type: boolean + default: false # Note: keep in sync with /azure-pipelines.yml + +variables: +- name: includeLocalPackagesBuildConfigParameter + ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: + value: '--includeLocalPackagesBuildConfig' + ${{ else }}: + value: '' +- name: IncludeLocalPackagesBuildConfigTest + ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: + value: '1' + ${{ else }}: + value: '' + trigger: - master - releases/* diff --git a/make-util.js b/make-util.js index 5f3074463004..3937440cbb41 100644 --- a/make-util.js +++ b/make-util.js @@ -1747,28 +1747,22 @@ exports.renameCodeCoverageOutput = renameCodeCoverageOutput; //------------------------------------------------------------------------------ /** - * Returns path to BuldConfigGenerator, build it if needed. Fail on compilation failure + * Ensure Pre-reqs for buildConfigGen (e.g. dotnet) * @param {String} baseConfigToolPath base build config tool path - * @returns {String} Path to the executed file */ -var getBuildConfigGenerator = function (baseConfigToolPath) { - var programPath = ""; +var ensureBuildConfigGeneratorPrereqs = function (baseConfigToolPath) { var configToolBuildUtility = ""; if (os.platform() === 'win32') { - programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen.exe'); configToolBuildUtility = path.join(baseConfigToolPath, "dev.cmd"); } else { - programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen'); configToolBuildUtility = path.join(baseConfigToolPath, "dev.sh"); } // build configToolBuildUtility if needed. (up-to-date check will skip build if not needed) run(configToolBuildUtility, true); - - return programPath; }; -exports.getBuildConfigGenerator = getBuildConfigGenerator; +exports.ensureBuildConfigGeneratorPrereqs = ensureBuildConfigGeneratorPrereqs; /** * Function to validate or write generated tasks @@ -1785,7 +1779,9 @@ var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions, if (sprintNumber && !Number.isInteger(sprintNumber)) fail("Sprint is not a number"); var tasks = taskList.join('|') - const programPath = getBuildConfigGenerator(baseConfigToolPath); + ensureBuildConfigGeneratorPrereqs(baseConfigToolPath); + var programPath = `dotnet run --project "${baseConfigToolPath}/BuildConfigGen.csproj" -- ` + const args = [ "--task", `"${tasks}"` diff --git a/make.js b/make.js index 525de280f0eb..1fd32931540d 100644 --- a/make.js +++ b/make.js @@ -1,6 +1,10 @@ // parse command line options var argv = require('minimist')(process.argv.slice(2)); +if (process.env.IncludeLocalPackagesBuildConfigTest === "1") { + argv.includeLocalPackagesBuildConfig=true; +} + // modules var fs = require('fs'); var os = require('os'); @@ -233,10 +237,35 @@ CLI.serverBuild = async function(/** @type {{ task: string }} */ argv) { } }); + // Need to validate generated tasks first + if (!argv.skipPrebuildSteps) + { + const makeOptions = fileToJson(makeOptionsPath); + + // Verify generated files across tasks are up-to-date + util.processGeneratedTasks(baseConfigToolPath, taskList, makeOptions, writeUpdatedsFromGenTasks, argv.sprint, argv['debug-agent-dir'], argv.includeLocalPackagesBuildConfig); + } + if (argv.includeLocalPackagesBuildConfig) { if (!argv.skipPrebuildSteps) { + // temp: clone for now prior to merging these as subtrees + if (!test('-d', 'task-lib')) { + run("git clone https://github.com/microsoft/azure-pipelines-task-lib task-lib"); + } + + if (!test('-d', 'tasks-common')) { + run("git clone https://github.com/microsoft/azure-pipelines-tasks-common-packages tasks-common"); + } + + cd(taskLibPath); + run("git pull"); + + cd(tasksCommonPath); + run("git pull"); + // end temp + // build task-lib cd(taskLibPath); run("npm install", /*inheritStreams:*/true); @@ -251,15 +280,6 @@ CLI.serverBuild = async function(/** @type {{ task: string }} */ argv) { } } - // Need to validate generated tasks first - if (!argv.skipPrebuildSteps) - { - const makeOptions = fileToJson(makeOptionsPath); - - // Verify generated files across tasks are up-to-date - util.processGeneratedTasks(baseConfigToolPath, taskList, makeOptions, writeUpdatedsFromGenTasks, argv.sprint, argv['debug-agent-dir'], argv.includeLocalPackagesBuildConfig); - } - const allTasks = getTaskList(taskList, argv.includeLocalPackagesBuildConfig); // Wrap build function to store files that changes after the build