[wasm] CI build improvements

eng/pipelines/common/evaluate-default-paths.yml

@@ -106,7 +106,8 @@ jobs:
     - subset: wasmbuildtests
       include:
       - src/tasks/*
-      - src/tests/BuildWasmApps/*
+      - src/tests/BuildWasmApps/Directory.Build*
+      - src/tests/BuildWasmApps/Wasm.Build.Tests/*
       - src/mono/wasm/build/*
       - src/mono/wasm/runtime/*
       - src/mono/wasm/templates/*
@@ -122,13 +123,14 @@ jobs:
       - eng/Version.Details.xml
       - eng/Versions.props
       - src/mono/wasm/emscripten-version.txt
+      - src/libraries/sendtohelix*
     - subset: wasmdebuggertests
       include:
       - src/mono/wasm/debugger/*
       - src/mono/wasm/runtime/*
-      - src/tests/BuildWasmApps/*
+      - src/tests/BuildWasmApps/Directory.Build*
+      - src/tests/BuildWasmApps/Wasm.Debugger.Tests/*
       - eng/testing/ProvisioningVersions.props
-      - eng/testing/scenarios/WasmDebuggerTestsJobsList.txt
       - src/mono/mono/*
     - subset: allwasm
       include:

eng/pipelines/common/templates/wasm-debugger-tests.yml

@@ -3,6 +3,7 @@ parameters:
   isExtraPlatformsBuild: false
   browser: 'chrome'
   platforms: []
+  shouldContinueOnError: false
 
 jobs:
 
@@ -41,3 +42,4 @@ jobs:
         extraHelixArguments: /p:BrowserHost=$(_hostedOs) /p:_DebuggerHosts=${{ parameters.browser }}
         scenarios:
         - wasmdebuggertests
+        shouldContinueOnError: ${{ parameters.shouldContinueOnError }}

eng/pipelines/runtime-extra-platforms-wasm.yml

@@ -124,3 +124,5 @@ jobs:
         - Browser_wasm_firefox
       browser: firefox
       alwaysRun: ${{ parameters.isWasmOnlyBuild }}
+      # don't fail because firefox tests are known to be unstable right now
+      shouldContinueOnError: true

src/libraries/sendtohelix-wasm.targets

@@ -14,7 +14,6 @@
     <EnableDefaultBuildHelixWorkItems Condition="'$(IsRunningLibraryTests)' != 'true'">false</EnableDefaultBuildHelixWorkItems>
 
     <BuildWasmAppsJobsList>$(RepositoryEngineeringDir)\testing\scenarios\BuildWasmAppsJobsList.txt</BuildWasmAppsJobsList>
-    <WasmDebuggerTestsJobsList>$(RepositoryEngineeringDir)\testing\scenarios\WasmDebuggerTestsJobsList.txt</WasmDebuggerTestsJobsList>
 
     <!-- on unix CI has emscripten provisioned in $(EMSDK_PATH) as `/usr/local/emscripten`. -->
     <EMSDK_PATH Condition="$([MSBuild]::IsOSPlatform('WINDOWS')) and '$(EMSDK_PATH)' == ''">$(RepoRoot)src\mono\wasm\emsdk\</EMSDK_PATH>
@@ -23,7 +22,7 @@
     <DebuggerHost Condition="'$(DebuggerHost)' == ''">chrome</DebuggerHost>
 
     <NeedsWorkload Condition="'$(Scenario)' == 'BuildWasmApps'">true</NeedsWorkload>
-    <NeedsEMSDK Condition="'$(NeedsToBuildWasmAppsOnHelix)' == 'true' or '$(Scenario)' == 'BuildWasmApps'">true</NeedsEMSDK>
+    <NeedsEMSDK Condition="'$(NeedsToBuildWasmAppsOnHelix)' == 'true'">true</NeedsEMSDK>
     <NeedsEMSDKNode Condition="'$(Scenario)' == 'WasmTestOnNodeJs' or '$(Scenario)' == 'BuildWasmApps'">true</NeedsEMSDKNode>
     <NeedsToRunOnBrowser Condition="'$(Scenario)' == 'WasmTestOnBrowser' or '$(Scenario)' == 'BuildWasmApps'">true</NeedsToRunOnBrowser>
     <NeedsToRunOnBrowser Condition="'$(NeedsToRunOnBrowser)' == '' and '$(IsWasmDebuggerTests)' == 'true'">true</NeedsToRunOnBrowser>
@@ -83,6 +82,15 @@
     <HelixPreCommand Include="set PATH=$(_HelixLocalNodePath)/%_HELIX_NODEJS_VERSION%/bin%3B%PATH%" />
   </ItemGroup>
 
+  <PropertyGroup Condition="'$(Scenario)' == 'BuildWasmApps'">
+    <_XUnitTraitArg Condition="'$(TestUsingWorkloads)' == 'true'">-notrait category=no-workload</_XUnitTraitArg>
+    <_XUnitTraitArg Condition="'$(TestUsingWorkloads)' != 'true'">-trait category=no-workload</_XUnitTraitArg>
+  </PropertyGroup>
+  <ItemGroup Condition="'$(Scenario)' == 'BuildWasmApps'">
+    <HelixPreCommand Condition="'$(WindowsShell)' == 'true'" Include="set &quot;XUnitTraitArg=$(_XUnitTraitArg)&quot;" />
+    <HelixPreCommand Condition="'$(WindowsShell)' != 'true'" Include="export &quot;XUnitTraitArg=$(_XUnitTraitArg)&quot;" />
+  </ItemGroup>
+
   <PropertyGroup>
 
     <!--
@@ -123,7 +131,8 @@
       <TestEchoMiddleware>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin', 'NetCoreServer', '$(Configuration)', '$(AspNetCoreAppCurrent)'))</TestEchoMiddleware>
       <RemoteLoopMiddleware>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin', 'RemoteLoopServer', '$(Configuration)', '$(AspNetCoreAppCurrent)'))</RemoteLoopMiddleware>
       <WorkItemPrefix Condition="'$(Scenario)' == 'BuildWasmApps' and '$(TestUsingWorkloads)' == 'true'">Workloads-</WorkItemPrefix>
-      <WorkItemPrefix Condition="'$(Scenario)' == 'BuildWasmApps' and '$(TestUsingWorkloads)' != 'true'">EMSDK-</WorkItemPrefix>
+      <WorkItemPrefix Condition="'$(Scenario)' == 'BuildWasmApps' and '$(TestUsingWorkloads)' != 'true'">NoWorkload-</WorkItemPrefix>
+      <WorkItemPrefix Condition="'$(IsWasmDebuggerTests)' == 'true'">$(DebuggerHost)-</WorkItemPrefix>
       <WorkItemPrefix Condition="'$(WorkItemPrefix)' == '' and '$(Scenario)' != ''">$(Scenario)-</WorkItemPrefix>
     </PropertyGroup>
 
@@ -155,14 +164,10 @@
       <HelixCorrelationPayload Include="$(RemoteLoopMiddleware)" Destination="xharness/RemoteLoopMiddleware" Condition="Exists('$(RemoteLoopMiddleware)')" />
     </ItemGroup>
 
-    <ReadLinesFromFile File="$(BuildWasmAppsJobsList)" Condition="Exists($(BuildWasmAppsJobsList)) and '$(Scenario)' == 'BuildWasmApps'">
+    <ReadLinesFromFile File="$(BuildWasmAppsJobsList)" Condition="Exists($(BuildWasmAppsJobsList)) and '$(Scenario)' == 'BuildWasmApps' and '$(TestUsingWorkloads)' == 'true'">
       <Output TaskParameter="Lines" ItemName="BuildWasmApps_PerJobList" />
     </ReadLinesFromFile>
 
-    <ReadLinesFromFile File="$(WasmDebuggerTestsJobsList)" Condition="Exists($(WasmDebuggerTestsJobsList)) and '$(Scenario)' == 'WasmDebuggerTests'">
-      <Output TaskParameter="Lines" ItemName="WasmDebuggerTests_PerJobList" />
-    </ReadLinesFromFile>
-
     <!-- library tests get added as default work items -->
     <ItemGroup Condition="'$(IsRunningLibraryTests)' != 'true'">
       <!-- for buildwasmapps, the archive path is set in src/libraries/Directory.Build.props, so use that -->
@@ -191,24 +196,26 @@
       </HelixWorkItem>
     </ItemGroup>
 
+    <!-- for testing with workloads, we use separate items -->
     <ItemGroup Condition="'$(Scenario)' == 'BuildWasmApps'">
-      <HelixWorkItem Include="@(BuildWasmApps_PerJobList->'$(WorkItemPrefix)%(Extension)')">
+      <HelixWorkItem Include="@(BuildWasmApps_PerJobList->'$(WorkItemPrefix)%(Extension)')" Condition="'$(TestUsingWorkloads)' == 'true'">
         <PayloadArchive>$(_BuildWasmAppsPayloadArchive)</PayloadArchive>
         <PreCommands Condition="'$(OS)' == 'Windows_NT'">set &quot;HELIX_XUNIT_ARGS=-class %(Identity)&quot;</PreCommands>
         <PreCommands Condition="'$(OS)' != 'Windows_NT'">export &quot;HELIX_XUNIT_ARGS=-class %(Identity)&quot;</PreCommands>
         <Command>$(HelixCommand)</Command>
         <Timeout>$(_workItemTimeout)</Timeout>
       </HelixWorkItem>
+
+      <HelixWorkItem Include="NoWorkload-Wasm.Build.Tests" Condition="'$(TestUsingWorkloads)' != 'true'">
+        <PayloadArchive>$(_BuildWasmAppsPayloadArchive)</PayloadArchive>
+        <Command>$(HelixCommand)</Command>
+        <Timeout>$(_workItemTimeout)</Timeout>
+      </HelixWorkItem>
     </ItemGroup>
 
     <ItemGroup Condition="'$(Scenario)' == 'WasmDebuggerTests'">
-      <HelixWorkItem Include="@(WasmDebuggerTests_PerJobList -> '$(DebuggerHost)-%(Identity)')">
+      <HelixWorkItem Include="$(DebuggerHost)-DebuggerTests">
         <PayloadArchive>$(_WasmDebuggerTestsPayloadArchive)</PayloadArchive>
-
-        <!-- FIXME: workaround for https://github.com/dotnet/runtime/issues/62660 -->
-        <PreCommands Condition="'$(OS)' == 'Windows_NT'">set TEST_ARGS=--filter &quot;FullyQualifiedName~%(Identity)&amp;Category^!=windows-failing&amp;Category^!=failing&quot;</PreCommands>
-        <PreCommands Condition="'$(OS)' != 'Windows_NT'">export TEST_ARGS=&quot;--filter FullyQualifiedName~%(Identity)&amp;Category!=linux-failing&amp;Category!=failing&quot;</PreCommands>
-
         <Command>$(HelixCommand)</Command>
         <Timeout>$(_workItemTimeout)</Timeout>
       </HelixWorkItem>

src/libraries/sendtohelix.proj

@@ -79,9 +79,7 @@
 
     <!-- For BuildWasmApps we want to build the project twice, with: TestUsingWorkloads=true, and TestUsingWorkloads=false -->
     <ItemGroup Condition="'@(_Scenarios -> AnyHaveMetadataValue('Identity', 'buildwasmapps'))' == 'true'">
-      <!-- TestUsingWorkloads=false - Disabled due to https://github.com/dotnet/arcade/issues/9025 -->
-      <!--<_TestUsingWorkloadsValues Include="true;false" />-->
-      <_TestUsingWorkloadsValues Include="true" />
+      <_TestUsingWorkloadsValues Include="true;false" />
 
       <_BuildWasmAppsProjectsToBuild Include="$(PerScenarioProjectFile)">
         <AdditionalProperties>$(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingWorkloadsValues.Identity)</AdditionalProperties>

src/mono/wasm/debugger/Directory.Build.props

@@ -1,6 +1,6 @@
 <Project>
-    <PropertyGroup>
-        <RuntimeConfiguration Condition="'$(RuntimeConfiguration)' == ''">Release</RuntimeConfiguration>
-    </PropertyGroup>
+  <PropertyGroup>
+    <RuntimeConfiguration Condition="'$(RuntimeConfiguration)' == ''">Release</RuntimeConfiguration>
+  </PropertyGroup>
   <Import Project="..\..\Directory.Build.props" />
 </Project>

src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs

@@ -21,7 +21,7 @@ public BlazorWasmBuildPublishTests(ITestOutputHelper output, SharedBuildPerTestC
             _enablePerTestCleanup = true;
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsNotUsingWorkloads))]
+        [Theory]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void DefaultTemplate_WithoutWorkload(string config)
@@ -38,7 +38,7 @@ public void DefaultTemplate_WithoutWorkload(string config)
             AssertBlazorBootJson(config, isPublish: true);
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Theory]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void DefaultTemplate_NoAOT_WithWorkload(string config)
@@ -58,7 +58,7 @@ public void DefaultTemplate_NoAOT_WithWorkload(string config)
             }
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Theory]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void DefaultTemplate_AOT_InProjectFile(string config)
@@ -77,7 +77,7 @@ public void DefaultTemplate_AOT_InProjectFile(string config)
             BlazorBuild(new BlazorBuildOptions(id, config, NativeFilesType.FromRuntimePack));
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Theory]
         [InlineData("Debug", true)]
         [InlineData("Debug", false)]
         [InlineData("Release", true)]
@@ -121,7 +121,7 @@ public void NativeBuild_WithDeployOnBuild_UsedByVS(string config, bool nativeRel
 
         // Disabling for now - publish folder can have more than one dotnet*hash*js, and not sure
         // how to pick which one to check, for the test
-        //[ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        //[Theory]
         //[InlineData("Debug")]
         //[InlineData("Release")]
         //public void DefaultTemplate_AOT_OnlyWithPublishCommandLine_Then_PublishNoAOT(string config)
@@ -140,7 +140,7 @@ public void NativeBuild_WithDeployOnBuild_UsedByVS(string config, bool nativeRel
             //BlazorPublish(new BlazorBuildOptions(id, config, NativeFilesType.Relinked);
         //}
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Theory]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void WithNativeReference_AOTInProjectFile(string config)
@@ -157,7 +157,7 @@ public void WithNativeReference_AOTInProjectFile(string config)
             BlazorBuild(new BlazorBuildOptions(id, config, NativeFilesType.Relinked));
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Theory]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void WithNativeReference_AOTOnCommandLine(string config)
@@ -173,7 +173,7 @@ public void WithNativeReference_AOTOnCommandLine(string config)
             BlazorPublish(new BlazorBuildOptions(id, config, NativeFilesType.Relinked));
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Theory]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void WithDllImportInMainAssembly(string config)
@@ -227,7 +227,7 @@ void CheckNativeFileLinked(bool forPublish)
             }
         }
 
-        [ConditionalFact(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
+        [Fact]
         public void BugRegression_60479_WithRazorClassLib()
         {
             string id = "blz_razor_lib_top";

src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmTests.cs

@@ -16,19 +16,17 @@ public BlazorWasmTests(ITestOutputHelper output, SharedBuildPerTestClassFixture
         {
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsNotUsingWorkloads))]
+        [Theory, TestCategory("no-workload")]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void NativeRef_EmitsWarningBecauseItRequiresWorkload(string config)
         {
             CommandResult res = PublishForRequiresWorkloadTest(config, extraItems: "<NativeFileReference Include=\"native-lib.o\" />");
             res.EnsureSuccessful();
-            AssertBlazorBundle(config, isPublish: true, dotnetWasmFromRuntimePack: true);
-
-            Assert.Contains("but the native references won't be linked in", res.Output);
+            Assert.Matches("warning : .*but the native references won't be linked in", res.Output);
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsNotUsingWorkloads))]
+        [Theory, TestCategory("no-workload")]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void AOT_FailsBecauseItRequiresWorkload(string config)
@@ -38,7 +36,7 @@ public void AOT_FailsBecauseItRequiresWorkload(string config)
             Assert.Contains("following workloads must be installed: wasm-tools", res.Output);
         }
 
-        [ConditionalTheory(typeof(BuildTestBase), nameof(IsNotUsingWorkloads))]
+        [Theory, TestCategory("no-workload")]
         [InlineData("Debug")]
         [InlineData("Release")]
         public void AOT_And_NativeRef_FailBecauseTheyRequireWorkload(string config)
@@ -56,13 +54,6 @@ private CommandResult PublishForRequiresWorkloadTest(string config, string extra
             string id = $"needs_workload_{config}_{Path.GetRandomFileName()}";
             CreateBlazorWasmTemplateProject(id);
 
-            if (IsNotUsingWorkloads)
-            {
-                // no packs installed, so no need to update the paths for runtime pack etc
-                File.Copy(Path.Combine(BuildEnvironment.TestDataPath, "Blazor.Local.Directory.Build.props"), Path.Combine(_projectDir!, "Directory.Build.props"), overwrite: true);
-                File.Copy(Path.Combine(BuildEnvironment.TestDataPath, "Blazor.Local.Directory.Build.targets"), Path.Combine(_projectDir!, "Directory.Build.targets"), overwrite: true);
-            }
-
             AddItemsPropertiesToProject(Path.Combine(_projectDir!, $"{id}.csproj"),
                                         extraProperties: extraProperties,
                                         extraItems: extraItems);
@@ -72,8 +63,7 @@ private CommandResult PublishForRequiresWorkloadTest(string config, string extra
                             .WithWorkingDirectory(_projectDir!)
                             .ExecuteWithCapturedOutput("publish",
                                                         $"-bl:{publishLogPath}",
-                                                        $"-p:Configuration={config}",
-                                                        "-p:MSBuildEnableWorkloadResolver=true"); // WasmApp.LocalBuild.* disables this, but it is needed for this test
+                                                        $"-p:Configuration={config}");
         }
 
         [Theory]

src/tests/BuildWasmApps/Wasm.Build.Tests/BuildEnvironment.cs

@@ -27,8 +27,6 @@ public class BuildEnvironment
         public static readonly string           TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets");
         public static readonly string           TestDataPath = Path.Combine(AppContext.BaseDirectory, "data");
 
-        private static string s_runtimeConfig = "Release";
-
         public BuildEnvironment()
         {
             DirectoryInfo? solutionRoot = new (AppContext.BaseDirectory);
@@ -59,6 +57,7 @@ public BuildEnvironment()
             if (!Directory.Exists(sdkForWorkloadPath))
                 throw new Exception($"Could not find SDK_FOR_WORKLOAD_TESTING_PATH={sdkForWorkloadPath}");
 
+            EnvVars = new Dictionary<string, string>();
             bool workloadInstalled = EnvironmentVariables.SdkHasWorkloadInstalled != null && EnvironmentVariables.SdkHasWorkloadInstalled == "true";
             if (workloadInstalled)
             {
@@ -69,7 +68,6 @@ public BuildEnvironment()
                 RuntimePackDir = Path.Combine(sdkForWorkloadPath, "packs", "Microsoft.NETCore.App.Runtime.Mono.browser-wasm", workloadPacksVersion);
                 DirectoryBuildPropsContents = s_directoryBuildPropsForWorkloads;
                 DirectoryBuildTargetsContents = s_directoryBuildTargetsForWorkloads;
-                EnvVars = new Dictionary<string, string>();
 
                 var appRefDir = EnvironmentVariables.AppRefDir;
                 if (string.IsNullOrEmpty(appRefDir))
@@ -80,36 +78,12 @@ public BuildEnvironment()
             }
             else
             {
-                string emsdkPath;
-                if (solutionRoot == null)
-                {
-                    string? buildDir = EnvironmentVariables.WasmBuildSupportDir;
-                    if (buildDir == null || !Directory.Exists(buildDir))
-                        throw new Exception($"Could not find the solution root, or a build dir: {buildDir}");
-
-                    emsdkPath = Path.Combine(buildDir, "emsdk");
-                    RuntimePackDir = Path.Combine(buildDir, "microsoft.netcore.app.runtime.browser-wasm");
-                    DefaultBuildArgs = $" /p:WasmBuildSupportDir={buildDir} /p:EMSDK_PATH={emsdkPath} ";
-                }
-                else
-                {
-                    string artifactsBinDir = Path.Combine(solutionRoot.FullName, "artifacts", "bin");
-                    RuntimePackDir = Path.Combine(artifactsBinDir, "microsoft.netcore.app.runtime.browser-wasm", s_runtimeConfig);
-
-                    if (string.IsNullOrEmpty(EnvironmentVariables.EMSDK_PATH))
-                        emsdkPath = Path.Combine(solutionRoot.FullName, "src", "mono", "wasm", "emsdk");
-                    else
-                        emsdkPath = EnvironmentVariables.EMSDK_PATH;
-
-                    DefaultBuildArgs = $" /p:RuntimeSrcDir={solutionRoot.FullName} /p:RuntimeConfig={s_runtimeConfig} /p:EMSDK_PATH={emsdkPath} ";
-                }
-
-                IsWorkload = false;
-                EnvVars = new Dictionary<string, string>()
-                {
-                    ["EMSDK_PATH"] = emsdkPath
-                };
+                RuntimePackDir = "/dont-check-runtime-pack-dir-for-no-workloads-case";
+                var appRefDir = EnvironmentVariables.AppRefDir;
+                if (string.IsNullOrEmpty(appRefDir))
+                    throw new Exception($"Cannot test with workloads without AppRefDir environment variable being set");
 
+                DefaultBuildArgs = $" /p:AppRefDir={appRefDir}";
                 DirectoryBuildPropsContents = s_directoryBuildPropsForLocal;
                 DirectoryBuildTargetsContents = s_directoryBuildTargetsForLocal;
             }