From 66cae46ace0644b5c7ca91ed9f28dd8f73041980 Mon Sep 17 00:00:00 2001 From: MerlynOMsft <44985659+merlynomsft@users.noreply.github.com> Date: Wed, 5 Apr 2023 06:24:20 -0700 Subject: [PATCH] Updates to BuildConfigGen (#18069) * Node16 switch wip * wip * poc update to buildConfigMapping * wip * wip * _ instead of dash - * wip * wip * wip * wip * wip * wip * wip * wip * Include task switching CodeGen to make.js - Add dev.sh and dev.cmd to build CodeGen via cmd (need to add downloading dotnet) - Add logic to change version in task.loc.json as well as in task.json - Add logic to add node16 handler to the generated task to run it under node16 handler and not node10 - Add check that task has node handler - Add check that task doesn't have node16 handler - Fixed problem with slash for unix compatibility * Include task switching CodeGen to make.js - Add gentask argument to call code generator for specific/all tasks and compile code generator is it's not compiled - Add logic to make.js build to build task from _generated folder firstly, and from Tasks folder if it's not found in _generated - Add logic to make.js test to test _generated tasks as well as usual Tasks - Resolved problem with Common folder in _generated folder * clean up * cleanup * Make Build Configs extensible Refactor / clean up * added --write-updates parameter. Default is verify-only * improve json formatting, error handling * Include task switching CodeGen to make.js - Add downloading dotnet sdk to dev.sh for CodeGen - Add pass-through arguments to BuildGen AzureIoTEdgeV2 * make dev.sh compatible with unix-like systems * added system to "override" files per-buildconfig * suppress up-to-date check for _buildConfigs presence in tasks directory * Include task switching CodeGen to make.js - Clean up some of the code in make.js - replace --write-updates with --validate option in make.js * Include task switching CodeGen to make.js - Add rebuild argument for gentask - Fix problem when directory not exists - Fix problem with generating task first time * fix: re-add missing mappings * Clean ups * added preprocessor * update so mapping matches mapping name on server * fix unexpected invalidation * fix issues where verification might fail when preprocessor is used prevent preprocessor from being used in _buildConfig overrides (not necessary to use both) fix directories been created when --write-updates not specified * disable generating _buildConfig/[config] as the precompile should take care of it * refactored verifier to write changes to temp files to compare. This change shouldn't change the output, but makes the verifiction logic more consistent and maintaintable * merged Node16 tasks * write major/minor version (in addition to patch) to task.loc.json verification fix when using file overrides * Add Node16 configurations for tasks * add DockerComposeV0 node upgrade * Update launch settings with DockerComposeV0 * Bump node16 versions * Correctly update AndroidSigningV2 * Correct DockerComposeV0 * fixup DownloadPackageV1 * Run config generation * Include task switching CodeGen to make.js - add --config attribute to make.js - add possibility to build only _generated version of the task without main task * Include task switching CodeGen to make.js - rename parameter from config to configs * Include task switching CodeGen to make.js - Task compilation fix for generated tasks * Include task switching CodeGen to make.js - Generated task update to be able to compile them and pass BuildConfigGenerator validation * move node16 changes to users/merlynop/node16updates * Remove non-BuildconfigGen changes --------- Co-authored-by: Dmitrii Bobreshev (Akvelon INC) Co-authored-by: Your Name --- BuildConfigGen/EnsureUpdateModeVerifier.cs | 12 ++ BuildConfigGen/Preprocessor.cs | 109 ++++++++-------- BuildConfigGen/Program.cs | 120 +++++++++++------- BuildConfigGen/Properties/launchSettings.json | 8 ++ 4 files changed, 151 insertions(+), 98 deletions(-) create mode 100644 BuildConfigGen/Properties/launchSettings.json diff --git a/BuildConfigGen/EnsureUpdateModeVerifier.cs b/BuildConfigGen/EnsureUpdateModeVerifier.cs index b79634796883..c7b4797f687f 100644 --- a/BuildConfigGen/EnsureUpdateModeVerifier.cs +++ b/BuildConfigGen/EnsureUpdateModeVerifier.cs @@ -223,8 +223,20 @@ internal string [] FileReadAllLines(string filePath) } } + internal bool FilesEqual(string sourcePath, string targetPath) + { + var resolvedTargetPath = ResolveFile(targetPath); + + return Helpers.FilesEqual(sourcePath, resolvedTargetPath); + } + private string ResolveFile(string filePath) { + if(!verifyOnly) + { + return filePath; + } + filePath = NormalizeFile(filePath); string? sourceFile = null, tempFile = null; diff --git a/BuildConfigGen/Preprocessor.cs b/BuildConfigGen/Preprocessor.cs index 43b5dc2a31d5..6a73a51ab88c 100644 --- a/BuildConfigGen/Preprocessor.cs +++ b/BuildConfigGen/Preprocessor.cs @@ -74,6 +74,7 @@ internal static void Preprocess(string file, IEnumerable lines, ISet lines, ISetThe task to generate build configs for + /// List of configs to generate seperated by | /// Write updates if true, else validate that the output is up-to-date - static void Main(string task, bool writeUpdates = false) + static void Main(string task, string configs, bool writeUpdates = false) { // error handling strategy: // 1. design: anything goes wrong, try to detect and crash as early as possible to preserve the callstack to make debugging easier. // 2. we allow all exceptions to fall though. Non-zero exit code will be surfaced // 3. Ideally default windows exception will occur and errors reported to WER/watson. I'm not sure this is happening, perhaps DragonFruit is handling the exception - Main3(task, writeUpdates); + foreach (var t in task.Split(',')) + { + Main3(t, configs, writeUpdates); + } } - private static void Main3(string task, bool writeUpdates) + private static void Main3(string task, string configsString, bool writeUpdates) { if (string.IsNullOrEmpty(task)) { throw new Exception("task expected!"); } + if (string.IsNullOrEmpty(configsString)) + { + throw new Exception("configs expected!"); + } + + string[] configs = configsString.Split("|"); + + Dictionary configdefs = new(Config.Configs.Where(x => !x.isDefault).Select(x => new KeyValuePair(x.name, x))); + HashSet targetConfigs = new HashSet(); + targetConfigs.Add(Config.Default); + foreach (var config in configs) + { + if (configdefs.TryGetValue(config, out var matchedConfig)) + { + targetConfigs.Add(matchedConfig); + } + else + { + string configsList = "Configs specified must be one of: " + string.Join(',', Config.Configs.Where(x=>!x.isDefault).Select(x => x.name)); + throw new Exception(configsList); + } + } + try { ensureUpdateModeVerifier = new EnsureUpdateModeVerifier(!writeUpdates); - Main2(task); + Main2(task, targetConfigs); ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(task, skipContentCheck: false); } @@ -89,7 +119,7 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr } } - private static void Main2(string task) + private static void Main2(string task, HashSet targetConfigs) { string currentDir = Environment.CurrentDirectory; @@ -104,18 +134,14 @@ private static void Main2(string task) string taskHandler = Path.Combine(taskTargetPath, "task.json"); JsonNode taskHandlerContents = JsonNode.Parse(ensureUpdateModeVerifier!.FileReadAllText(taskHandler))!; - // Task may not have nodejs or packages.json (example: AutomatedAnalysisV0) - if (!hasNodeHandler(taskHandlerContents)) + if (targetConfigs.Any(x => x.isNode16)) { - Console.WriteLine($"Skipping {task} because task doesn't have node handler does not exist"); - return; - } - - // If target task already has node16 handlers, skip it - if (taskHandlerContents["execution"]!["Node16"] != null) - { - Console.WriteLine($"Skipping {task} because it already has a Node16 handler"); - return; + // Task may not have nodejs or packages.json (example: AutomatedAnalysisV0) + if (!hasNodeHandler(taskHandlerContents)) + { + Console.WriteLine($"Skipping {task} because task doesn't have node handler does not exist"); + return; + } } // Create _generated @@ -125,9 +151,9 @@ private static void Main2(string task) ensureUpdateModeVerifier!.DirectoryCreateDirectory(generatedFolder, false); } - UpdateVersions(gitRootPath, task, taskTargetPath, out var configTaskVersionMapping); + UpdateVersions(gitRootPath, task, taskTargetPath, out var configTaskVersionMapping, targetConfigs: targetConfigs); - foreach (var config in Config.Configs) + foreach (var config in targetConfigs) { string taskOutput; if (config.isDefault) @@ -139,7 +165,7 @@ private static void Main2(string task) taskOutput = Path.Combine(gitRootPath, "_generated", @$"{task}_{config.name}"); } - if (Knob.Default.EnableBuildConfigOverrides) + if (config.enableBuildConfigOverrides) { EnsureBuildConfigFileOverrides(config, taskTargetPath); } @@ -148,7 +174,7 @@ private static void Main2(string task) - if (Knob.Default.EnableBuildConfigOverrides) + if (config.enableBuildConfigOverrides) { CopyConfigOverrides(taskTargetPath, taskOutput, config); } @@ -174,9 +200,9 @@ private static void Main2(string task) private static void EnsureBuildConfigFileOverrides(Config.ConfigRecord config, string taskTargetPath) { - if(!Knob.Default.EnableBuildConfigOverrides) + if(!config.enableBuildConfigOverrides) { - throw new Exception("BUG: should not get here: !Knob.Default.EnableBuildConfigOverrides"); + throw new Exception("BUG: should not get here: !config.enableBuildConfigOverrides"); } string path, readmeFile; @@ -192,9 +218,9 @@ private static void EnsureBuildConfigFileOverrides(Config.ConfigRecord config, s private static void GetBuildConfigFileOverridePaths(Config.ConfigRecord config, string taskTargetPath, out string path, out string readmeFile) { - if (!Knob.Default.EnableBuildConfigOverrides) + if (!config.enableBuildConfigOverrides) { - throw new Exception("BUG: should not get here: !Knob.Default.EnableBuildConfigOverrides"); + throw new Exception("BUG: should not get here: !config.enableBuildConfigOverrides"); } path = Path.Combine(taskTargetPath, buildConfigs, config.name); @@ -203,9 +229,9 @@ private static void GetBuildConfigFileOverridePaths(Config.ConfigRecord config, private static void CopyConfigOverrides(string taskTargetPath, string taskOutput, Config.ConfigRecord config) { - if (!Knob.Default.EnableBuildConfigOverrides) + if (!config.enableBuildConfigOverrides) { - throw new Exception("BUG: should not get here: !Knob.Default.EnableBuildConfigOverrides"); + throw new Exception("BUG: should not get here: !config.enableBuildConfigOverrides"); } string overridePathForBuildConfig; @@ -241,7 +267,7 @@ private static void HandlePreprocessingInTarget(string taskOutput, Config.Config private static void PreprocessIfExtensionEnabledInConfig(string file, Config.ConfigRecord config, bool validateAndWriteChanges, out bool madeChanges) { - HashSet extensions = new HashSet(config.extensionsToPreprocess); + HashSet extensions = new HashSet(Config.ExtensionsToPreprocess); bool preprocessExtension = extensions.Contains(Path.GetExtension(file)); if (preprocessExtension) { @@ -251,9 +277,9 @@ private static void PreprocessIfExtensionEnabledInConfig(string file, Config.Con } else { - if (!Knob.Default.EnableBuildConfigOverrides) + if (!config.enableBuildConfigOverrides) { - throw new Exception("BUG: should not get here: !Knob.Default.EnableBuildConfigOverrides"); + throw new Exception("BUG: should not get here: !config.enableBuildConfigOverrides"); } Console.WriteLine($"Checking if {file} has preprocessor directives ..."); @@ -296,6 +322,8 @@ private static void WriteTaskJson(string taskPath, Dictionary configTaskVersionMapping) + private static void UpdateVersions(string gitRootPath, string task, string taskTarget, out Dictionary configTaskVersionMapping, HashSet targetConfigs) { Dictionary versionMap; TaskVersion? maxVersion; @@ -444,7 +472,7 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT // copy the mappings. As we go check if any configs not mapped. If so, invalidate. bool allConfigsMappedAndValid = true; - foreach (var config in Config.Configs) + foreach (var config in targetConfigs) { if (versionMap.ContainsKey(config.constMappingKey)) { @@ -474,7 +502,7 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT { configTaskVersionMapping.Clear(); - foreach (var config in Config.Configs) + foreach (var config in targetConfigs) { if (!config.isDefault) { @@ -487,7 +515,7 @@ private static void UpdateVersions(string gitRootPath, string task, string taskT configTaskVersionMapping.Add(Config.Default, inputVersion.CloneWithPatch(inputVersion.Patch + c)); } - WriteVersionMapFile(versionMapFile, configTaskVersionMapping); + WriteVersionMapFile(versionMapFile, configTaskVersionMapping, targetConfigs: targetConfigs); } private static TaskVersion GetInputVersion(string taskTarget) @@ -516,17 +544,19 @@ private static void WriteInputTaskJson(string taskTarget, Dictionary configTaskVersion) + private static void WriteVersionMapFile(string versionMapFile, Dictionary configTaskVersion, HashSet targetConfigs) { StringBuilder sb = new StringBuilder(); using (var sw = new StringWriter(sb)) { - foreach (var config in Config.Configs) + foreach (var config in targetConfigs) { sw.WriteLine(string.Concat(config.constMappingKey, "|", configTaskVersion[config])); } @@ -620,7 +650,7 @@ private static void CopyFile(string sourcePath, string targetPath) Console.Write($"Copy from={sourcePath} to={targetPath}..."); - if (Helpers.FilesEqual(sourcePath, targetPath)) + if (ensureUpdateModeVerifier!.FilesEqual(sourcePath, targetPath)) { Console.WriteLine("files same, skipping"); } diff --git a/BuildConfigGen/Properties/launchSettings.json b/BuildConfigGen/Properties/launchSettings.json new file mode 100644 index 000000000000..13b1276043bd --- /dev/null +++ b/BuildConfigGen/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "BuildConfigGen": { + "commandName": "Project", + "commandLineArgs": "--task AndroidSigningV2,AndroidSigningV3,DockerComposeV0,DownloadFileshareArtifactsV0,DownloadPackageV1,FuncToolsInstallerV0,GitHubCommentV0,KubernetesManifestV0,KubernetesV0,MysqlDeploymentOnMachineGroupV1,NuGetRestoreV1,PackerBuildV0,PackerBuildV1,PublishCodeCoverageResultsV1 --configs Node16 --write-updates" + } + } +} \ No newline at end of file