From ccb263daa86c4105a1a303b9c2c86693966bb163 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 18 Apr 2023 15:44:05 +0000 Subject: [PATCH 01/47] Update dependencies from https://github.com/dotnet/arcade build 20230417.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23217.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/arm/sources.list.xenial | 2 +- eng/common/cross/arm64/sources.list.xenial | 2 +- eng/common/cross/toolchain.cmake | 6 ++++-- global.json | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b76230aac04..d824a4e2499 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 1177b37e60c7271ab461098aefbdd03edc15b90d + cb54ca21431ee8d96f91abfbc42237bcb001f9d1 - + https://github.com/dotnet/arcade - 1177b37e60c7271ab461098aefbdd03edc15b90d + cb54ca21431ee8d96f91abfbc42237bcb001f9d1 diff --git a/eng/Versions.props b/eng/Versions.props index ccd682f639b..9e0aa26e0fb 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23174.1 + 8.0.0-beta.23217.1 diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial index eacd86b7df3..56fbb36a59f 100644 --- a/eng/common/cross/arm/sources.list.xenial +++ b/eng/common/cross/arm/sources.list.xenial @@ -8,4 +8,4 @@ deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial index eacd86b7df3..56fbb36a59f 100644 --- a/eng/common/cross/arm64/sources.list.xenial +++ b/eng/common/cross/arm64/sources.list.xenial @@ -8,4 +8,4 @@ deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index d813dde6de2..1c9d212d135 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -76,7 +76,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(TOOLCHAIN "s390x-linux-gnu") elseif(TARGET_ARCH_NAME STREQUAL "x64") set(CMAKE_SYSTEM_PROCESSOR x86_64) - if(LINUX) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl) + set(TOOLCHAIN "x86_64-alpine-linux-musl") + elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") @@ -279,7 +281,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/global.json b/global.json index 17dbc652d01..edc094165be 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-preview.1.23115.2" + "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23174.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23217.1" } } From e457cbe8bb1f412b000f768961578de70a39097e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 24 Apr 2023 12:14:52 +0000 Subject: [PATCH 02/47] Update dependencies from https://github.com/dotnet/arcade build 20230424.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23224.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/build-rootfs.sh | 2 +- eng/common/templates/job/job.yml | 15 +++++++++------ .../templates/steps/component-governance.yml | 13 +++++++++++++ global.json | 2 +- 6 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 eng/common/templates/steps/component-governance.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d824a4e2499..02bc2dd3d35 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - cb54ca21431ee8d96f91abfbc42237bcb001f9d1 + d14c4adbdbb8c7ee060f9e333883fc9890939b66 - + https://github.com/dotnet/arcade - cb54ca21431ee8d96f91abfbc42237bcb001f9d1 + d14c4adbdbb8c7ee060f9e333883fc9890939b66 diff --git a/eng/Versions.props b/eng/Versions.props index 9e0aa26e0fb..0f3a8a531a8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23217.1 + 8.0.0-beta.23224.1 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 1deadcc676e..79bb058ab02 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -70,7 +70,7 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.3-RELEASE" +__FreeBSDBase="12.4-RELEASE" __FreeBSDPkg="1.17.0" __FreeBSDABI="12" __FreeBSDPackages="libunwind" diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index b214a31db23..a21533fdd31 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -25,7 +25,7 @@ parameters: enablePublishTestResults: false enablePublishUsingPipelines: false enableBuildRetry: false - disableComponentGovernance: false + disableComponentGovernance: '' componentGovernanceIgnoreDirectories: '' mergeTestResults: false testRunTitle: '' @@ -155,11 +155,14 @@ jobs: uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} continueOnError: true - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + - template: /eng/common/templates/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(contains(variables['Build.SourceBranch'], 'internal/release'), eq(variables['Build.SourceBranch'], 'main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml new file mode 100644 index 00000000000..0ecec47b0c9 --- /dev/null +++ b/eng/common/templates/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/global.json b/global.json index edc094165be..53005e50598 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23217.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23224.1" } } From 81947216cda41340109523bf2e0cd4b9594c0ea0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 1 May 2023 12:30:22 +0000 Subject: [PATCH 03/47] Update dependencies from https://github.com/dotnet/arcade build 20230426.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23226.4 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/build-rootfs.sh | 2 +- eng/common/templates/job/job.yml | 4 +++- global.json | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 02bc2dd3d35..392bac1d8fc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - d14c4adbdbb8c7ee060f9e333883fc9890939b66 + 469dcc0cbcba5221727c8a5b9eec4a478e24a780 - + https://github.com/dotnet/arcade - d14c4adbdbb8c7ee060f9e333883fc9890939b66 + 469dcc0cbcba5221727c8a5b9eec4a478e24a780 diff --git a/eng/Versions.props b/eng/Versions.props index 0f3a8a531a8..5ef87758dff 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23224.1 + 8.0.0-beta.23226.4 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 79bb058ab02..6a59f753b5d 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -330,7 +330,7 @@ while :; do ;; freebsd13) __CodeName=freebsd - __FreeBSDBase="13.0-RELEASE" + __FreeBSDBase="13.2-RELEASE" __FreeBSDABI="13" __SkipUnmount=1 ;; diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index a21533fdd31..a7bbf0f1fee 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -158,10 +158,12 @@ jobs: - template: /eng/common/templates/steps/component-governance.yml parameters: ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(contains(variables['Build.SourceBranch'], 'internal/release'), eq(variables['Build.SourceBranch'], 'main'))) }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: disableComponentGovernance: false ${{ else }}: disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableMicrobuild, 'true') }}: diff --git a/global.json b/global.json index 53005e50598..b21303560bf 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23224.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23226.4" } } From 65303efba905c8dd12a32a5867bb51fa09b5d317 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 8 May 2023 12:31:50 +0000 Subject: [PATCH 04/47] Update dependencies from https://github.com/dotnet/arcade build 20230505.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23255.2 --- eng/Version.Details.xml | 8 +++---- eng/Versions.props | 2 +- eng/common/cross/toolchain.cmake | 32 ++++++++++++++++++++++----- eng/common/native/init-compiler.sh | 2 +- eng/common/templates/job/job.yml | 2 +- eng/common/tools.ps1 | 28 ++++++++++++++++++++---- eng/common/tools.sh | 35 +++++++++++++++++++++++++++--- global.json | 2 +- 8 files changed, 91 insertions(+), 20 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 392bac1d8fc..49bedd654c8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 469dcc0cbcba5221727c8a5b9eec4a478e24a780 + 5dff25b814b90abb85c847ed274cf4afb2c1b276 - + https://github.com/dotnet/arcade - 469dcc0cbcba5221727c8a5b9eec4a478e24a780 + 5dff25b814b90abb85c847ed274cf4afb2c1b276 diff --git a/eng/Versions.props b/eng/Versions.props index 5ef87758dff..aae03060866 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23226.4 + 8.0.0-beta.23255.2 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 1c9d212d135..ce01673c7a6 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -67,13 +67,25 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6") endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) - set(TOOLCHAIN "powerpc64le-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) + set(TOOLCHAIN "powerpc64le-alpine-linux-musl") + else() + set(TOOLCHAIN "powerpc64le-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "riscv64") set(CMAKE_SYSTEM_PROCESSOR riscv64) - set(TOOLCHAIN "riscv64-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) + set(TOOLCHAIN "riscv64-alpine-linux-musl") + else() + set(TOOLCHAIN "riscv64-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(CMAKE_SYSTEM_PROCESSOR s390x) - set(TOOLCHAIN "s390x-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl) + set(TOOLCHAIN "s390x-alpine-linux-musl") + else() + set(TOOLCHAIN "s390x-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x64") set(CMAKE_SYSTEM_PROCESSOR x86_64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl) @@ -92,7 +104,11 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) - set(TOOLCHAIN "i686-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + set(TOOLCHAIN "i586-alpine-linux-musl") + else() + set(TOOLCHAIN "i686-linux-gnu") + endif() if(TIZEN) set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") endif() @@ -266,8 +282,11 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_toolchain_linker_flag("--target=${TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") + endif() add_toolchain_linker_flag(-m32) - if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") @@ -307,6 +326,9 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_compile_options(-mfloat-abi=softfp) endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_compile_options(--target=${TOOLCHAIN}) + endif() add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 7aee4213e1b..517401b688b 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -64,7 +64,7 @@ if [ -z "$CLR_CC" ]; then if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi + elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do _major="${version%%.*}" diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index a7bbf0f1fee..44ad26abf54 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -158,7 +158,7 @@ jobs: - template: /eng/common/templates/steps/component-governance.yml parameters: ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: disableComponentGovernance: false ${{ else }}: disableComponentGovernance: true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 8ad03be3ecc..38cf94ff88c 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -287,6 +287,25 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtimeSourceFeedKey = '', [switch] $noPath) { + $dotnetVersionLabel = "'sdk v$version'" + + if ($runtime -ne '' -and $runtime -ne 'sdk') { + $runtimePath = $dotnetRoot + $runtimePath = $runtimePath + "\shared" + if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" } + if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" } + if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" } + $runtimePath = $runtimePath + "\" + $version + + $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'" + + if (Test-Path $runtimePath) { + Write-Host " Runtime toolset '$runtime/$architecture v$version' already installed." + $installSuccess = $true + Exit + } + } + $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ Version = $version @@ -323,18 +342,18 @@ function InstallDotNet([string] $dotnetRoot, } else { $location = "public location"; } - Write-Host "Attempting to install dotnet from $location." + Write-Host " Attempting to install $dotnetVersionLabel from $location." try { & $installScript @variation $installSuccess = $true break } catch { - Write-Host "Failed to install dotnet from $location." + Write-Host " Failed to install $dotnetVersionLabel from $location." } } if (-not $installSuccess) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 } } @@ -399,7 +418,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # Locate Visual Studio installation or download x-copy msbuild. $vsInfo = LocateVisualStudio $vsRequirements if ($vsInfo -ne $null) { - $vsInstallDir = $vsInfo.installationPath + # Ensure vsInstallDir has a trailing slash + $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion diff --git a/eng/common/tools.sh b/eng/common/tools.sh index cf9fb1ea2d3..e8d47894334 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -184,6 +184,35 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 + local runtime=$4 + + local dotnetVersionLabel="'$runtime v$version'" + if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then + runtimePath="$root" + runtimePath="$runtimePath/shared" + case "$runtime" in + dotnet) + runtimePath="$runtimePath/Microsoft.NETCore.App" + ;; + aspnetcore) + runtimePath="$runtimePath/Microsoft.AspNetCore.App" + ;; + windowsdesktop) + runtimePath="$runtimePath/Microsoft.WindowsDesktop.App" + ;; + *) + ;; + esac + runtimePath="$runtimePath/$version" + + dotnetVersionLabel="runtime toolset '$runtime/$architecture v$version'" + + if [ -d "$runtimePath" ]; then + echo " Runtime toolset '$runtime/$architecture v$version' already installed." + local installSuccess=1 + return + fi + fi GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -228,17 +257,17 @@ function InstallDotNet { for variationName in "${variations[@]}"; do local name="$variationName[@]" local variation=("${!name}") - echo "Attempting to install dotnet from $variationName." + echo " Attempting to install $dotnetVersionLabel from $variationName." bash "$install_script" "${variation[@]}" && installSuccess=1 if [[ "$installSuccess" -eq 1 ]]; then break fi - echo "Failed to install dotnet from $variationName." + echo " Failed to install $dotnetVersionLabel from $variationName." done if [[ "$installSuccess" -eq 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 fi } diff --git a/global.json b/global.json index b21303560bf..56f4145dda4 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23226.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23255.2" } } From 3c25a8a38776cff4029d71a714876ba0cc8c327b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 15 May 2023 12:30:22 +0000 Subject: [PATCH 05/47] Update dependencies from https://github.com/dotnet/arcade build 20230512.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23262.5 --- eng/Version.Details.xml | 8 +- eng/Versions.props | 2 +- eng/common/cross/build-rootfs.sh | 94 ++++++++++++------------ eng/common/cross/toolchain.cmake | 13 ++-- eng/common/cross/x64/sources.list.bionic | 11 +++ global.json | 2 +- 6 files changed, 70 insertions(+), 60 deletions(-) create mode 100644 eng/common/cross/x64/sources.list.bionic diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 49bedd654c8..2e7d11adf74 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 5dff25b814b90abb85c847ed274cf4afb2c1b276 + 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 - + https://github.com/dotnet/arcade - 5dff25b814b90abb85c847ed274cf4afb2c1b276 + 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 diff --git a/eng/Versions.props b/eng/Versions.props index aae03060866..57c64d564b1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23255.2 + 8.0.0-beta.23262.5 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6a59f753b5d..9caf9b021db 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -27,6 +27,7 @@ __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 __IllumosArch=arm7 +__HaikuArch=arm __QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" @@ -85,8 +86,12 @@ __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" __IllumosPackages+=" zlib" -__HaikuPackages="gmp" +__HaikuPackages="gcc_syslibs" +__HaikuPackages+=" gcc_syslibs_devel" +__HaikuPackages+=" gmp" __HaikuPackages+=" gmp_devel" +__HaikuPackages+=" icu66" +__HaikuPackages+=" icu66_devel" __HaikuPackages+=" krb5" __HaikuPackages+=" krb5_devel" __HaikuPackages+=" libiconv" @@ -95,6 +100,10 @@ __HaikuPackages+=" llvm12_libunwind" __HaikuPackages+=" llvm12_libunwind_devel" __HaikuPackages+=" mpfr" __HaikuPackages+=" mpfr_devel" +__HaikuPackages+=" openssl" +__HaikuPackages+=" openssl_devel" +__HaikuPackages+=" zlib" +__HaikuPackages+=" zlib_devel" # ML.NET dependencies __UbuntuPackages+=" libomp5" @@ -210,6 +219,7 @@ while :; do __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 __illumosArch=x86_64 + __HaikuArch=x86_64 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; x86) @@ -340,7 +350,6 @@ while :; do ;; haiku) __CodeName=haiku - __BuildArch=x64 __SkipUnmount=1 ;; --skipunmount) @@ -559,67 +568,54 @@ elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - echo "Building Haiku sysroot for x86_64" + echo "Building Haiku sysroot for $__HaikuArch" mkdir -p "$__RootfsDir/tmp" - cd "$__RootfsDir/tmp" - git clone -b hrev56235 https://review.haiku-os.org/haiku - git clone -b btrev43195 https://review.haiku-os.org/buildtools - cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d - - # Fetch some unmerged patches - cd "$__RootfsDir/tmp/haiku" - ## Add development build profile (slimmer than nightly) - git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD - - # Build jam - cd "$__RootfsDir/tmp/buildtools/jam" - make - - # Configure cross tools - echo "Building cross-compiler" - mkdir -p "$__RootfsDir/generated" - cd "$__RootfsDir/generated" - "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 - - # Build Haiku packages - echo "Building Haiku" - echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig - "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' - - BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - - # Download additional packages - echo "Downloading additional required packages" + pushd "$__RootfsDir/tmp" + + mkdir "$__RootfsDir/tmp/download" + + echo "Downloading Haiku package tool" + git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script + wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" + + DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + + # Download Haiku packages + echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + done + for package in haiku haiku_devel; do + echo "Downloading $package..." + hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" done - # Setup the sysroot - echo "Setting up sysroot and extracting needed packages" + # Set up the sysroot + echo "Setting up sysroot and extracting required packages" mkdir -p "$__RootfsDir/boot/system" - for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" - done - for file in "$__RootfsDir/generated/download/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + for file in "$__RootfsDir/tmp/download/"*.hpkg; do + echo "Extracting $file..." + LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file" done + # Download buildtools + echo "Downloading Haiku buildtools" + wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) + unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" + # Cleaning up temporary files echo "Cleaning up temporary files" + popd rm -rf "$__RootfsDir/tmp" - for name in "$__RootfsDir/generated/"*; do - if [[ "$name" =~ "cross-tools-" ]]; then - : # Keep the cross-compiler - else - rm -rf "$name" - fi - done elif [[ -n "$__CodeName" ]]; then if [[ "$__SkipSigCheck" == "0" ]]; then diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ce01673c7a6..a88d643c8a7 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -6,6 +6,7 @@ unset(FREEBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) +unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) @@ -16,6 +17,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(ILLUMOS 1) elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) set(CMAKE_SYSTEM_NAME Haiku) + set(HAIKU 1) else() set(CMAKE_SYSTEM_NAME Linux) set(LINUX 1) @@ -76,7 +78,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64") set(CMAKE_SYSTEM_PROCESSOR riscv64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) set(TOOLCHAIN "riscv64-alpine-linux-musl") - else() + else() set(TOOLCHAIN "riscv64-linux-gnu") endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") @@ -100,7 +102,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) - set(TOOLCHAIN "x64_64-unknown-haiku") + set(TOOLCHAIN "x86_64-unknown-haiku") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) @@ -214,10 +216,8 @@ elseif(HAIKU) return() endif() - set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") - find_program(EXEC_LOCATION_${exec} - PATHS ${SEARCH_PATH} + PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin" NAMES "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" "${TOOLSET_PREFIX}${exec}") @@ -296,6 +296,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") +elseif(HAIKU) + add_toolchain_linker_flag("-lnetwork") + add_toolchain_linker_flag("-lroot") endif() # Specify compile options diff --git a/eng/common/cross/x64/sources.list.bionic b/eng/common/cross/x64/sources.list.bionic new file mode 100644 index 00000000000..a71ccadcffa --- /dev/null +++ b/eng/common/cross/x64/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse diff --git a/global.json b/global.json index 56f4145dda4..12ea219fe63 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23255.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23262.5" } } From 941172b8b1424980907198f8b7c3b1df7c308b63 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 22 May 2023 12:32:19 +0000 Subject: [PATCH 06/47] Update dependencies from https://github.com/dotnet/arcade build 20230519.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23269.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2e7d11adf74..eefc1e722e3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 + 792c346a52b2388152c2fad6f626b88ed8736d6f - + https://github.com/dotnet/arcade - 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 + 792c346a52b2388152c2fad6f626b88ed8736d6f diff --git a/eng/Versions.props b/eng/Versions.props index 57c64d564b1..92d0b5eed2a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23262.5 + 8.0.0-beta.23269.2 diff --git a/global.json b/global.json index 12ea219fe63..84f4a9f7e66 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23262.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23269.2" } } From 7316a4fed07154416298ca41d056afa6f8521581 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 29 May 2023 12:32:35 +0000 Subject: [PATCH 07/47] Update dependencies from https://github.com/dotnet/arcade build 20230529.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23279.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/sdk-task.ps1 | 2 +- eng/common/templates/job/job.yml | 12 ++++++++---- eng/common/templates/steps/source-build.yml | 6 ++++++ eng/common/tools.ps1 | 4 ++-- global.json | 4 ++-- 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index eefc1e722e3..2e588fc4576 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 792c346a52b2388152c2fad6f626b88ed8736d6f + 174c08a43b26fc26ba98975065f1125f910f5c37 - + https://github.com/dotnet/arcade - 792c346a52b2388152c2fad6f626b88ed8736d6f + 174c08a43b26fc26ba98975065f1125f910f5c37 diff --git a/eng/Versions.props b/eng/Versions.props index 92d0b5eed2a..a97f8b036ef 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23269.2 + 8.0.0-beta.23279.1 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index e10a5968797..6c4ac6fec1a 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.6.0-2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 44ad26abf54..e20ee3a983c 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -75,6 +75,10 @@ jobs: - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - name: EnableRichCodeNavigation value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: @@ -83,7 +87,7 @@ jobs: - ${{ if ne(variable.name, '') }}: - name: ${{ variable.name }} value: ${{ variable.value }} - + # handle variable groups - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} @@ -169,7 +173,7 @@ jobs: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks + displayName: Execute Microbuild cleanup tasks condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} env: @@ -219,7 +223,7 @@ jobs: displayName: Publish XUnit Test Results inputs: testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit mergeTestResults: ${{ parameters.mergeTestResults }} @@ -230,7 +234,7 @@ jobs: displayName: Publish TRX Test Results inputs: testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx mergeTestResults: ${{ parameters.mergeTestResults }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index a97a185a367..1100521834a 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -68,6 +68,11 @@ steps: runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' fi + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + publishArgs= if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then publishArgs='--publish' @@ -86,6 +91,7 @@ steps: $internalRestoreArgs \ $targetRidArgs \ $runtimeOsArgs \ + $baseOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true \ /p:AssetManifestFileName=$assetManifestFileName diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 38cf94ff88c..ffe0b4e2dfc 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview - $defaultXCopyMSBuildVersion = '17.4.1' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.6.0-2 + $defaultXCopyMSBuildVersion = '17.6.0-2' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index 84f4a9f7e66..758328b5776 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-preview.3.23178.7" + "dotnet": "8.0.100-preview.4.23260.5" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23269.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23279.1" } } From 9c18087e4b88b5d9bc491029b395e12ca37ec05f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 5 Jun 2023 12:32:41 +0000 Subject: [PATCH 08/47] Update dependencies from https://github.com/dotnet/arcade build 20230602.3 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23302.3 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2e588fc4576..b500ef7cfa7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 174c08a43b26fc26ba98975065f1125f910f5c37 + 234e0726c7384ee84bf08550f2d16a1ff2d5c543 - + https://github.com/dotnet/arcade - 174c08a43b26fc26ba98975065f1125f910f5c37 + 234e0726c7384ee84bf08550f2d16a1ff2d5c543 diff --git a/eng/Versions.props b/eng/Versions.props index a97f8b036ef..2a47425f734 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23279.1 + 8.0.0-beta.23302.3 diff --git a/global.json b/global.json index 758328b5776..1dd36eef716 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.4.23260.5" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23279.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23302.3" } } From 252fb5a35deb98ae77bdf9e17b7cbff45d802075 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 12 Jun 2023 12:34:50 +0000 Subject: [PATCH 09/47] Update dependencies from https://github.com/dotnet/arcade build 20230609.8 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23309.8 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b500ef7cfa7..7087f73dbb7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 234e0726c7384ee84bf08550f2d16a1ff2d5c543 + 9994c5ed43856c35591d94ead68ef501a020eb2e - + https://github.com/dotnet/arcade - 234e0726c7384ee84bf08550f2d16a1ff2d5c543 + 9994c5ed43856c35591d94ead68ef501a020eb2e diff --git a/eng/Versions.props b/eng/Versions.props index 2a47425f734..3391e343727 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23302.3 + 8.0.0-beta.23309.8 diff --git a/global.json b/global.json index 1dd36eef716..aadcde8f29d 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.4.23260.5" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23302.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23309.8" } } From 3cd76f6ed75608dd48f7040d842c479bb9a9291a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 19 Jun 2023 12:31:50 +0000 Subject: [PATCH 10/47] Update dependencies from https://github.com/dotnet/arcade build 20230616.6 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23316.6 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/dotnet-install.sh | 4 ++++ eng/common/tools.ps1 | 2 +- global.json | 4 ++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7087f73dbb7..18bdff5f3c6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + e2334b2be36919347923d0ec872a46acddb1e385 - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + e2334b2be36919347923d0ec872a46acddb1e385 diff --git a/eng/Versions.props b/eng/Versions.props index 3391e343727..4de3f86135d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23309.8 + 8.0.0-beta.23316.6 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index abd045a3247..7e69e3a9e24 100644 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -54,6 +54,10 @@ cpuname=$(uname -m) case $cpuname in arm64|aarch64) buildarch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + buildarch=arm + fi ;; loongarch64) buildarch=loongarch64 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index ffe0b4e2dfc..c9eced9f7df 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,7 +379,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '16.8' + $vsMinVersionReqdStr = '17.6' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) # If the version of msbuild is going to be xcopied, diff --git a/global.json b/global.json index aadcde8f29d..bff3b58bbb7 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-preview.4.23260.5" + "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23309.8" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23316.6" } } From 436ad652cb31a5e2f11301b0599028dcc05fbf2e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 26 Jun 2023 12:34:00 +0000 Subject: [PATCH 11/47] Update dependencies from https://github.com/dotnet/arcade build 20230622.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23322.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 18bdff5f3c6..79dd873ca2a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - e2334b2be36919347923d0ec872a46acddb1e385 + 9747cf5ac4abb6a5a13cf31fa78b91d599180e07 - + https://github.com/dotnet/arcade - e2334b2be36919347923d0ec872a46acddb1e385 + 9747cf5ac4abb6a5a13cf31fa78b91d599180e07 diff --git a/eng/Versions.props b/eng/Versions.props index 4de3f86135d..a91f376de3d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23316.6 + 8.0.0-beta.23322.2 diff --git a/global.json b/global.json index bff3b58bbb7..8862e023479 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23316.6" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23322.2" } } From 3c0a883a4158e67c37a33f9fccfc79131ae8722d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 3 Jul 2023 12:35:05 +0000 Subject: [PATCH 12/47] Update dependencies from https://github.com/dotnet/arcade build 20230630.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23330.1 --- eng/Version.Details.xml | 8 ++--- eng/Versions.props | 2 +- eng/common/sdl/configure-sdl-tool.ps1 | 40 ++++++++++++++-------- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 ++- eng/common/templates/steps/execute-sdl.yml | 2 +- global.json | 2 +- 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 79dd873ca2a..1a79be4a91e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 9747cf5ac4abb6a5a13cf31fa78b91d599180e07 + 9fc23a576f443284d25cd4311f5778158ad289c9 - + https://github.com/dotnet/arcade - 9747cf5ac4abb6a5a13cf31fa78b91d599180e07 + 9fc23a576f443284d25cd4311f5778158ad289c9 diff --git a/eng/Versions.props b/eng/Versions.props index a91f376de3d..ac251b37f4b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23322.2 + 8.0.0-beta.23330.1 diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 index bdbf49e6c71..ee9eb893321 100644 --- a/eng/common/sdl/configure-sdl-tool.ps1 +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -17,7 +17,9 @@ Param( # Optional: Additional params to add to any tool using PoliCheck. [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional params to add to any tool using CodeQL/Semmle. - [string[]] $CodeQLAdditionalRunConfigParams + [string[]] $CodeQLAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using Binskim. + [string[]] $BinskimAdditionalRunConfigParams ) $ErrorActionPreference = 'Stop' @@ -69,22 +71,32 @@ try { $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" # For some tools, add default and automatic args. - if ($tool.Name -eq 'credscan') { - if ($targetDirectory) { - $tool.Args += "`"TargetDirectory < $TargetDirectory`"" + switch -Exact ($tool.Name) { + 'credscan' { + if ($targetDirectory) { + $tool.Args += "`"TargetDirectory < $TargetDirectory`"" + } + $tool.Args += "`"OutputType < pre`"" + $tool.Args += $CrScanAdditionalRunConfigParams } - $tool.Args += "`"OutputType < pre`"" - $tool.Args += $CrScanAdditionalRunConfigParams - } elseif ($tool.Name -eq 'policheck') { - if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" + 'policheck' { + if ($targetDirectory) { + $tool.Args += "`"Target < $TargetDirectory`"" + } + $tool.Args += $PoliCheckAdditionalRunConfigParams } - $tool.Args += $PoliCheckAdditionalRunConfigParams - } elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') { - if ($targetDirectory) { - $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" + {$_ -in 'semmle', 'codeql'} { + if ($targetDirectory) { + $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" + } + $tool.Args += $CodeQLAdditionalRunConfigParams + } + 'binskim' { + if ($targetDirectory) { + $tool.Args += "`"Target < $TargetDirectory`"" + } + $tool.Args += $BinskimAdditionalRunConfigParams } - $tool.Args += $CodeQLAdditionalRunConfigParams } # Create variable pointing to the args array directly so we can use splat syntax later. diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 4797e012c7d..4715d75e974 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -35,6 +35,7 @@ Param( [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1") + [string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1") [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run ) @@ -107,7 +108,8 @@ try { -GuardianLoggerLevel $GuardianLoggerLevel ` -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` - -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams + -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams ` + -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams if ($BreakOnFailure) { Exit-IfNZEC "Sdl" } diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 9dd5709f66d..07426fde05d 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -33,7 +33,7 @@ steps: - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL + displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} diff --git a/global.json b/global.json index 8862e023479..8ebcc5211b0 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23322.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23330.1" } } From a92049091aa49ad12af785a80fdb3b620ba5ed3a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 10 Jul 2023 12:34:02 +0000 Subject: [PATCH 13/47] Update dependencies from https://github.com/dotnet/arcade build 20230710.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23360.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1a79be4a91e..83cae60f112 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 9fc23a576f443284d25cd4311f5778158ad289c9 + 226c103b56a979243f08032e6cddb4deb66ea54b - + https://github.com/dotnet/arcade - 9fc23a576f443284d25cd4311f5778158ad289c9 + 226c103b56a979243f08032e6cddb4deb66ea54b diff --git a/eng/Versions.props b/eng/Versions.props index ac251b37f4b..97f9b925d12 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23330.1 + 8.0.0-beta.23360.1 diff --git a/global.json b/global.json index 8ebcc5211b0..bb0221d04eb 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23330.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23360.1" } } From ee891bc17984c3a54a1fa2faeb78f9c2888ab155 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 17 Jul 2023 12:15:19 +0000 Subject: [PATCH 14/47] Update dependencies from https://github.com/dotnet/arcade build 20230714.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23364.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/sdl/configure-sdl-tool.ps1 | 2 +- global.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 83cae60f112..42407c79b96 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 226c103b56a979243f08032e6cddb4deb66ea54b + 60ea5b2eca5af06fc63b250f8669d2c70179b18c - + https://github.com/dotnet/arcade - 226c103b56a979243f08032e6cddb4deb66ea54b + 60ea5b2eca5af06fc63b250f8669d2c70179b18c diff --git a/eng/Versions.props b/eng/Versions.props index 97f9b925d12..a57fd482787 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23360.1 + 8.0.0-beta.23364.2 diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 index ee9eb893321..e4108e39d9a 100644 --- a/eng/common/sdl/configure-sdl-tool.ps1 +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -93,7 +93,7 @@ try { } 'binskim' { if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" + $tool.Args += "`"Target < $TargetDirectory\**`"" } $tool.Args += $BinskimAdditionalRunConfigParams } diff --git a/global.json b/global.json index bb0221d04eb..afc5a096cdb 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-preview.5.23303.2" + "dotnet": "8.0.100-preview.6.23330.14" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23360.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23364.2" } } From c34958f58e34f34dc9a1c2cb3b63d68c55ab1d13 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 24 Jul 2023 12:15:01 +0000 Subject: [PATCH 15/47] Update dependencies from https://github.com/dotnet/arcade build 20230721.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23371.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/sdl/configure-sdl-tool.ps1 | 4 +++- global.json | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 42407c79b96..4b443c4ac0f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 60ea5b2eca5af06fc63b250f8669d2c70179b18c + 602351e3681015ea789b2aeaa7b2a9156a8baf38 - + https://github.com/dotnet/arcade - 60ea5b2eca5af06fc63b250f8669d2c70179b18c + 602351e3681015ea789b2aeaa7b2a9156a8baf38 diff --git a/eng/Versions.props b/eng/Versions.props index a57fd482787..fa3328f651f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23364.2 + 8.0.0-beta.23371.1 diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 index e4108e39d9a..27f5a4115fc 100644 --- a/eng/common/sdl/configure-sdl-tool.ps1 +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -93,7 +93,9 @@ try { } 'binskim' { if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory\**`"" + # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. + # We are excluding all `_.pdb` files from the scan. + $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" } $tool.Args += $BinskimAdditionalRunConfigParams } diff --git a/global.json b/global.json index afc5a096cdb..b7f3df84b5b 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.6.23330.14" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23364.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23371.1" } } From e132872300737e3965727daf0d392f44babfb661 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 31 Jul 2023 12:14:43 +0000 Subject: [PATCH 16/47] Update dependencies from https://github.com/dotnet/arcade build 20230728.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23378.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4b443c4ac0f..b59aadb8e9b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 602351e3681015ea789b2aeaa7b2a9156a8baf38 + 54dd37d44a2adfb8b966fac466c2ece40f8b20dd - + https://github.com/dotnet/arcade - 602351e3681015ea789b2aeaa7b2a9156a8baf38 + 54dd37d44a2adfb8b966fac466c2ece40f8b20dd diff --git a/eng/Versions.props b/eng/Versions.props index fa3328f651f..2025c18af9f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23371.1 + 8.0.0-beta.23378.2 diff --git a/global.json b/global.json index b7f3df84b5b..7be04750dbd 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.6.23330.14" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23371.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23378.2" } } From 4bc5bbd17ddd174f0f88d72b27af499825810653 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 7 Aug 2023 12:14:41 +0000 Subject: [PATCH 17/47] Update dependencies from https://github.com/dotnet/arcade build 20230804.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23404.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/loc/P22DotNetHtmlLocalization.lss | Bin 3810 -> 3842 bytes eng/common/sdl/extract-artifact-packages.ps1 | 20 ++++++++++--------- eng/common/templates/steps/source-build.yml | 9 +++++++++ global.json | 2 +- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b59aadb8e9b..bcd570b7e35 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 54dd37d44a2adfb8b966fac466c2ece40f8b20dd + 1d39647dd408f7afd99cce01f26bba1d6bdeb248 - + https://github.com/dotnet/arcade - 54dd37d44a2adfb8b966fac466c2ece40f8b20dd + 1d39647dd408f7afd99cce01f26bba1d6bdeb248 diff --git a/eng/Versions.props b/eng/Versions.props index 2025c18af9f..a8b9966af16 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23378.2 + 8.0.0-beta.23404.2 diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss index 6661fed566e49b0c206665bc21f135e06c9b89c4..858a0b237c62ce4f2ee12aa14794f160f6b122b7 100644 GIT binary patch delta 274 zcmaDP+a$;Ke*az|n#iQX$U9L_gN>JqL4iSGqQ3ORGAW?Q1Qjlb2!joS{zOAbi0B6! zCSI<|ip=VhLl{FCc_;TV>rD1wb_28WKvIp|l9MO!aluqwVe-JE3Mi$(QiGy^k2Qyp zcXJgh2O}fzByO?EbC~%yFJp^h0r5&8LVSE82)RSthrlX5CZFL2dIY2eC=as!FRvoZ VC5(KsFeZ}w7+K^&o(6kQ830GCL398B delta 289 zcmZpYdnC*B|KCJ59Y(H+iP97Oc_zkKO`NVWagPln*JOPr<;k*)5sCa-6G z0Wy(Sbn`m43>Gj`eDVTrmdU$#btcDi1KkdioE*R>0M!WMtz(ne{FHkKBZ~3}45bsf c6(=X~aZS$V(*VjreBsR}4-AfjvJ?He0GoYJrT_o{ diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 7f28d9c59ec..f031ed5b25e 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -35,31 +35,33 @@ try { param( [string] $PackagePath # Full path to a NuGet package ) - + if (!(Test-Path $PackagePath)) { Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" ExitWithExitCode 1 } - + $RelevantExtensions = @('.dll', '.exe', '.pdb') Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - + Add-Type -AssemblyName System.IO.Compression.FileSystem - + [System.IO.Directory]::CreateDirectory($ExtractPath); - + try { $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) $zip.Entries | Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | ForEach-Object { - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) + [System.IO.Directory]::CreateDirectory($TargetPath); + + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) } } catch { diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 1100521834a..41bbb915736 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -118,3 +118,12 @@ steps: artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/global.json b/global.json index 7be04750dbd..15717532fa0 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.6.23330.14" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23378.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23404.2" } } From 9bc45f58fc67c3114b08755e737f5e84bd38b06a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 14 Aug 2023 12:12:39 +0000 Subject: [PATCH 18/47] Update dependencies from https://github.com/dotnet/arcade build 20230811.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23411.1 --- eng/Version.Details.xml | 8 +- eng/Versions.props | 2 +- eng/common/native/init-distro-rid.sh | 132 ++++++++++++++++++++++++++ eng/common/native/init-os-and-arch.sh | 80 ++++++++++++++++ global.json | 4 +- 5 files changed, 219 insertions(+), 7 deletions(-) create mode 100644 eng/common/native/init-distro-rid.sh create mode 100644 eng/common/native/init-os-and-arch.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index bcd570b7e35..632a8334859 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 1d39647dd408f7afd99cce01f26bba1d6bdeb248 + 9b2af35a6702526dc8a7c5fcadcc44efd0dca170 - + https://github.com/dotnet/arcade - 1d39647dd408f7afd99cce01f26bba1d6bdeb248 + 9b2af35a6702526dc8a7c5fcadcc44efd0dca170 diff --git a/eng/Versions.props b/eng/Versions.props index a8b9966af16..5cb0851f5e7 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23404.2 + 8.0.0-beta.23411.1 diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh new file mode 100644 index 00000000000..aba9fe24028 --- /dev/null +++ b/eng/common/native/init-distro-rid.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +# getNonPortableDistroRid +# +# Input: +# targetOs: (str) +# targetArch: (str) +# rootfsDir: (str) +# +# Return: +# non-portable rid +getNonPortableDistroRid() +{ + local targetOs="$1" + local targetArch="$2" + local rootfsDir="$3" + local nonPortableRid="" + + if [ "$targetOs" = "linux" ]; then + if [ -e "${rootfsDir}/etc/os-release" ]; then + source "${rootfsDir}/etc/os-release" + + if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then + # remove the last version digit + VERSION_ID="${VERSION_ID%.*}" + fi + + if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then + nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" + else + # Rolling release distros either do not set VERSION_ID, set it as blank or + # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux); + # so omit it here to be consistent with everything else. + nonPortableRid="${ID}-${targetArch}" + fi + + elif [ -e "${rootfsDir}/android_platform" ]; then + source "$rootfsDir"/android_platform + nonPortableRid="$RID" + fi + fi + + if [ "$targetOs" = "freebsd" ]; then + # $rootfsDir can be empty. freebsd-version is shell script and it should always work. + __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; }) + nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then + __android_sdk_version=$(getprop ro.build.version.sdk) + nonPortableRid="android.$__android_sdk_version-${targetArch}" + elif [ "$targetOs" = "illumos" ]; then + __uname_version=$(uname -v) + case "$__uname_version" in + omnios-*) + __omnios_major_version=$(echo "${__uname_version:8:2}") + nonPortableRid=omnios."$__omnios_major_version"-"$targetArch" + ;; + joyent_*) + __smartos_major_version=$(echo "${__uname_version:7:4}") + nonPortableRid=smartos."$__smartos_major_version"-"$targetArch" + ;; + illumos_*) + nonPortableRid=openindiana-"$targetArch" + ;; + esac + elif [ "$targetOs" = "solaris" ]; then + __uname_version=$(uname -v) + __solaris_major_version=$(echo "${__uname_version%.*}") + nonPortableRid=solaris."$__solaris_major_version"-"$targetArch" + elif [ "$targetOs" = "haiku" ]; then + __uname_release=$(uname -r) + nonPortableRid=haiku.r"$__uname_release"-"$targetArch" + fi + + echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')" +} + +# initDistroRidGlobal +# +# Input: +# os: (str) +# arch: (str) +# isPortable: (int) +# rootfsDir?: (nullable:string) +# +# Return: +# None +# +# Notes: +# +# It is important to note that the function does not return anything, but it +# exports the following variables on success: +# +# __DistroRid : Non-portable rid of the target platform. +# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. +# +initDistroRidGlobal() +{ + local targetOs="$1" + local targetArch="$2" + local isPortable="$3" + local rootfsDir="" + if [ "$#" -ge 4 ]; then + rootfsDir="$4" + fi + + if [ -n "${rootfsDir}" ]; then + # We may have a cross build. Check for the existence of the rootfsDir + if [ ! -e "${rootfsDir}" ]; then + echo "Error rootfsDir has been passed, but the location is not valid." + exit 1 + fi + fi + + __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}") + + if [ -z "${__PortableTargetOS:-}" ]; then + __PortableTargetOS="$targetOs" + + STRINGS="$(command -v strings || true)" + if [ -z "$STRINGS" ]; then + STRINGS="$(command -v llvm-strings || true)" + fi + + # Check for musl-based distros (e.g Alpine Linux, Void Linux). + if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || + ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then + __PortableTargetOS="linux-musl" + fi + fi + + export __DistroRid __PortableTargetOS +} diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh new file mode 100644 index 00000000000..e693617a6c2 --- /dev/null +++ b/eng/common/native/init-os-and-arch.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Use uname to determine what the OS is. +OSName=$(uname -s | tr '[:upper:]' '[:lower:]') + +if command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then + OSName="android" +fi + +case "$OSName" in +freebsd|linux|netbsd|openbsd|sunos|android|haiku) + os="$OSName" ;; +darwin) + os=osx ;; +*) + echo "Unsupported OS $OSName detected!" + exit 1 ;; +esac + +# On Solaris, `uname -m` is discouraged, see https://docs.oracle.com/cd/E36784_01/html/E36870/uname-1.html +# and `uname -p` returns processor type (e.g. i386 on amd64). +# The appropriate tool to determine CPU is isainfo(1) https://docs.oracle.com/cd/E36784_01/html/E36870/isainfo-1.html. +if [ "$os" = "sunos" ]; then + if uname -o 2>&1 | grep -q illumos; then + os="illumos" + else + os="solaris" + fi + CPUName=$(isainfo -n) +else + # For the rest of the operating systems, use uname(1) to determine what the CPU is. + CPUName=$(uname -m) +fi + +case "$CPUName" in + arm64|aarch64) + arch=arm64 + ;; + + loongarch64) + arch=loongarch64 + ;; + + riscv64) + arch=riscv64 + ;; + + amd64|x86_64) + arch=x64 + ;; + + armv7l|armv8l) + if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then + arch=armel + else + arch=arm + fi + ;; + + armv6l) + arch=armv6 + ;; + + i[3-6]86) + echo "Unsupported CPU $CPUName detected, build might not succeed!" + arch=x86 + ;; + + s390x) + arch=s390x + ;; + + ppc64le) + arch=ppc64le + ;; + *) + echo "Unknown CPU $CPUName detected!" + exit 1 + ;; +esac diff --git a/global.json b/global.json index 15717532fa0..8ad5854c3ec 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-preview.6.23330.14" + "dotnet": "8.0.100-preview.7.23376.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23404.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23411.1" } } From 772a1cbf966c9412b4ad1566871d86d8de024e40 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 21 Aug 2023 12:23:45 +0000 Subject: [PATCH 19/47] Update dependencies from https://github.com/dotnet/arcade build 20230819.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23419.1 --- eng/Version.Details.xml | 8 +-- eng/Versions.props | 2 +- eng/common/SetupNugetSources.ps1 | 2 +- eng/common/SetupNugetSources.sh | 2 +- eng/common/native/init-compiler.sh | 2 +- eng/common/native/init-distro-rid.sh | 6 +- eng/common/sdl/trim-assets-version.ps1 | 75 ++++++++++++++++++++++++ eng/common/templates/job/execute-sdl.yml | 5 ++ eng/common/tools.ps1 | 6 +- global.json | 2 +- 10 files changed, 96 insertions(+), 14 deletions(-) create mode 100644 eng/common/sdl/trim-assets-version.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 632a8334859..d8cdb24a6ec 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 9b2af35a6702526dc8a7c5fcadcc44efd0dca170 + 385129cbc980a515ddee2fa56f6b16f3183ed9bc - + https://github.com/dotnet/arcade - 9b2af35a6702526dc8a7c5fcadcc44efd0dca170 + 385129cbc980a515ddee2fa56f6b16f3183ed9bc diff --git a/eng/Versions.props b/eng/Versions.props index 5cb0851f5e7..a530aa359af 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23411.1 + 8.0.0-beta.23419.1 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6e997239451..6c65e81925f 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -153,7 +153,7 @@ if ($dotnet31Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } -$dotnetVersions = @('5','6','7') +$dotnetVersions = @('5','6','7','8') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 8af7d899db1..d387c7eac95 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -105,7 +105,7 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet3.1-internal-transport') fi -DotNetVersions=('5' '6' '7') +DotNetVersions=('5' '6' '7' '8') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 517401b688b..f5c1ec7eafe 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index aba9fe24028..de1687b2ccb 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -79,7 +79,6 @@ getNonPortableDistroRid() # Input: # os: (str) # arch: (str) -# isPortable: (int) # rootfsDir?: (nullable:string) # # Return: @@ -97,10 +96,9 @@ initDistroRidGlobal() { local targetOs="$1" local targetArch="$2" - local isPortable="$3" local rootfsDir="" - if [ "$#" -ge 4 ]; then - rootfsDir="$4" + if [ "$#" -ge 3 ]; then + rootfsDir="$3" fi if [ -n "${rootfsDir}" ]; then diff --git a/eng/common/sdl/trim-assets-version.ps1 b/eng/common/sdl/trim-assets-version.ps1 new file mode 100644 index 00000000000..a2e00487704 --- /dev/null +++ b/eng/common/sdl/trim-assets-version.ps1 @@ -0,0 +1,75 @@ +<# +.SYNOPSIS +Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name. + +.PARAMETER InputPath +Full path to directory where artifact packages are stored + +.PARAMETER Recursive +Search for NuGet packages recursively + +#> + +Param( + [string] $InputPath, + [bool] $Recursive = $true +) + +$CliToolName = "Microsoft.DotNet.VersionTools.Cli" + +function Install-VersionTools-Cli { + param( + [Parameter(Mandatory=$true)][string]$Version + ) + + Write-Host "Installing the package '$CliToolName' with a version of '$version' ..." + $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" + + $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed") + Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait +} + +# ------------------------------------------------------------------- + +if (!(Test-Path $InputPath)) { + Write-Host "Input Path '$InputPath' does not exist" + ExitWithExitCode 1 +} + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +# `tools.ps1` checks $ci to perform some actions. Since the SDL +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +. $PSScriptRoot\..\tools.ps1 + +try { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + + $toolsetVersion = Read-ArcadeSdkVersion + Install-VersionTools-Cli -Version $toolsetVersion + + $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName}) + if ($null -eq $cliToolFound) { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed." + ExitWithExitCode 1 + } + + Exec-BlockVerbosely { + & "$dotnet" $CliToolName trim-assets-version ` + --assets-path $InputPath ` + --recursive $Recursive + Exit-IfNZEC "Sdl" + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} \ No newline at end of file diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 7aabaa18017..7870f93bc17 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -105,6 +105,11 @@ jobs: downloadPath: $(Build.ArtifactStagingDirectory)\artifacts checkDownloadedFiles: true + - powershell: eng/common/sdl/trim-assets-version.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts + displayName: Trim the version from the NuGet packages + continueOnError: ${{ parameters.sdlContinueOnError }} + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index c9eced9f7df..aa74ab4a81e 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -671,6 +671,10 @@ function InitializeNativeTools() { } } +function Read-ArcadeSdkVersion() { + return $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' +} + function InitializeToolset() { if (Test-Path variable:global:_ToolsetBuildProj) { return $global:_ToolsetBuildProj @@ -678,7 +682,7 @@ function InitializeToolset() { $nugetCache = GetNuGetPackageCachePath - $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' + $toolsetVersion = Read-ArcadeSdkVersion $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" if (Test-Path $toolsetLocationFile) { diff --git a/global.json b/global.json index 8ad5854c3ec..6437085581f 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.7.23376.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23411.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23419.1" } } From af0aabae68dd8f61e300acdfa012f2e26f7dd445 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 28 Aug 2023 12:24:28 +0000 Subject: [PATCH 20/47] Update dependencies from https://github.com/dotnet/arcade build 20230825.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23425.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/loc/P22DotNetHtmlLocalization.lss | Bin 3842 -> 1876 bytes global.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d8cdb24a6ec..6434bd51f15 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 385129cbc980a515ddee2fa56f6b16f3183ed9bc + 90c167d5c57de4a8bced566379dbd893556c94e8 - + https://github.com/dotnet/arcade - 385129cbc980a515ddee2fa56f6b16f3183ed9bc + 90c167d5c57de4a8bced566379dbd893556c94e8 diff --git a/eng/Versions.props b/eng/Versions.props index a530aa359af..5aee6308807 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23419.1 + 8.0.0-beta.23425.2 diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss index 858a0b237c62ce4f2ee12aa14794f160f6b122b7..5d892d619398f9feeac4cefc1c53ff18a807f732 100644 GIT binary patch literal 1876 zcmd5-O>Y`85WVMDO!tse0|`=3Kq5qFvJwFroLr(PGHdUyD`QjJ8$y14$Ln3lZW2{J zwGs!I&o^(Lhrz|mTEa6oR%kVh&N`>j#re@-x_nwr2#IG`%ct-0baH+K&@f(3mgC!a zLE`z$`_TL4VSG6vqld@GGPwJ;L@RorHxAn^xdXw5(R4X4f7_@k)pTF-lorz22$--N zNp~~4=EJDUfxSBovWjDy#&0y*A$Y%{951Lgf#O!hPmkZU}#I!)Wng z1f<%)R3}u5SL$^NOII+VJC!`W+V>zz0i%$|xtEKJRoWJsS2-4>j6oFy0;9}2)ZXS? z!4+5B;BS?^g*W;n#c6Okuah`~R7l#LlUM`Wwgi;X`GgRCXg2Aj93+p!!wlyufdm#b2J0d!FWGIX~B#Nn{?*T z-%qWy16eI?CAp^@t!rT|vicQ-vsH(!kP=oLa9cnNr~Mvq*K+4!SBpR#0ixlQVLn!4 z_-b`9x-gH#c?z0gti{WerraQ#kQ%)%r?a{+35PnWsUFnNN^8lf0v68Oy&oSBa0E$V ze5J~88to5MDZA?699~0Him^%vH|v?chtS*Ws!BWhyEn^bjuBh5M+wg@h`3AQ cw&^?j6MgS4IM#l5)SHHef6(}&EB`Bh1GJK71^@s6 literal 3842 zcmd^?%T5$g5Qb}Q;yWB!uo4gwSD=IdDhbFa0~=!u%y4NlOykf9`fh|r;uZYH%XVd_Hn3nn?a)ScZi%Hfv^O?o^EPX*xG&pV zdv4F4*^+JAj(y{^Ze6Yo`)Kdu$~&GnS-les=(}Hd4Z$5-wa;woL^azeDs1)bFYF6K zYqo;O3eR7;3PMXZcdt9G7M2L|%(A;+cL+Vh<;40ia7DRYcz+HE1jrRpV_p*&!nF z$63@IaaFcvPRXqUT!EoEiMw~cG<%Xu1rqKRFp4w~aJ8hjgHT7ZDe-mb8W8O!t5`M} ztjgBRJpt|=SdiLxc+i9U&FGh9|IZv$NUelFF6qG*xY~f;wk`WWC*7iH2HNhpSVO7O zyjh))z5&gu%l(Qp>Nx07A`jIOXn!e02v|!Y!amv(`^`P?xq%*=(IC|pQx2Mm#Wop? zaijLey=o_a!^t>PhS7L4AbHDoC--rqkF#Y>vg5i&^y;GPmsmXHUEuiBT|Lj=5A8Un zViJHWkF=5jOaLV5s#-dX{vP4sfVZ=}?;@uYMqc~3{L6QHIy{PC)9lJ@k#y=HUqMdP z=#t_YRzBG-DDIK;Y5PHn(kQ#ieaMsI&5-MIikcQm zB9?pn%E>6AT}IUVSi2;S8K#g0d(Ee4XBw2{|D?DN(56#J`IN_F465Cx8|@w=tHXbj zU&@|NZLLZ7222|77ga@RWjf<2u>)ikm8K_A{31S!HRE05qRWsu=1qcb*PrKMEaNuH zxG7hoiSg&T9>yF6JSZ32L1c$;kl)UW(|FgMr)Xcsb Date: Mon, 4 Sep 2023 12:25:39 +0000 Subject: [PATCH 21/47] Update dependencies from https://github.com/dotnet/arcade build 20230901.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23451.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/toolchain.cmake | 2 +- global.json | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6434bd51f15..b6eab3435b9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 90c167d5c57de4a8bced566379dbd893556c94e8 + 4665b3d04e1da3796b965c3c3e3b97f55c449a6e - + https://github.com/dotnet/arcade - 90c167d5c57de4a8bced566379dbd893556c94e8 + 4665b3d04e1da3796b965c3c3e3b97f55c449a6e diff --git a/eng/Versions.props b/eng/Versions.props index 5aee6308807..0e0b45c502c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23425.2 + 8.0.0-beta.23451.1 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index a88d643c8a7..0998e875e5f 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -207,6 +207,7 @@ elseif(ILLUMOS) set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") set(TOOLSET_PREFIX ${TOOLCHAIN}-) function(locate_toolchain_exec exec var) @@ -217,7 +218,6 @@ elseif(HAIKU) endif() find_program(EXEC_LOCATION_${exec} - PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin" NAMES "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" "${TOOLSET_PREFIX}${exec}") diff --git a/global.json b/global.json index ed336ca476f..7a912544c1d 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.7.23376.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23425.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23451.1" } } From 0a21a56c61837aacc6e29f5f59f3a5e2d16a654f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 11 Sep 2023 12:08:16 +0000 Subject: [PATCH 22/47] Update dependencies from https://github.com/dotnet/arcade build 20230901.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23451.1 From 86f13c70ab77cfb36bf3572693425f858d3804d4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 18 Sep 2023 12:16:25 +0000 Subject: [PATCH 23/47] Update dependencies from https://github.com/dotnet/arcade build 20230913.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23463.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b6eab3435b9..147a8411834 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 4665b3d04e1da3796b965c3c3e3b97f55c449a6e + 1d451c32dda2314c721adbf8829e1c0cd4e681ff - + https://github.com/dotnet/arcade - 4665b3d04e1da3796b965c3c3e3b97f55c449a6e + 1d451c32dda2314c721adbf8829e1c0cd4e681ff diff --git a/eng/Versions.props b/eng/Versions.props index 0e0b45c502c..52286cca5b8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23451.1 + 8.0.0-beta.23463.1 diff --git a/global.json b/global.json index 7a912544c1d..cb0412d0c25 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-preview.7.23376.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23451.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23463.1" } } From 688c81c437a81de34973228771d2efe1d3ad7af3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 25 Sep 2023 12:08:18 +0000 Subject: [PATCH 24/47] Update dependencies from https://github.com/dotnet/arcade build 20230913.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23463.1 From 0556c28c6d8d0f0d9c1eb859aa2c035d876e4528 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 2 Oct 2023 12:07:06 +0000 Subject: [PATCH 25/47] Update dependencies from https://github.com/dotnet/arcade build 20230913.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 8.0.0-beta.23463.1 From 276b6fdd7bc57d3fcfb3536a97cc79700913796b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 9 Oct 2023 12:15:30 +0000 Subject: [PATCH 26/47] Update dependencies from https://github.com/dotnet/arcade build 20231008.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23508.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/sdk-task.ps1 | 2 +- eng/common/tools.ps1 | 6 +++--- global.json | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 147a8411834..b7095ab0a1e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 1d451c32dda2314c721adbf8829e1c0cd4e681ff + 822f095b8c815dd7b9161140a9ff8151de593f82 - + https://github.com/dotnet/arcade - 1d451c32dda2314c721adbf8829e1c0cd4e681ff + 822f095b8c815dd7b9161140a9ff8151de593f82 diff --git a/eng/Versions.props b/eng/Versions.props index 52286cca5b8..a097e39ac27 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.0-beta.23463.1 + 9.0.0-beta.23508.1 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 6c4ac6fec1a..91f8196cc80 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.6.0-2" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.7.2-1" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index aa74ab4a81e..84cfe7cd9cb 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,13 +379,13 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.6' + $vsMinVersionReqdStr = '17.7' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.6.0-2 - $defaultXCopyMSBuildVersion = '17.6.0-2' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.7.2-1 + $defaultXCopyMSBuildVersion = '17.7.2-1' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index cb0412d0c25..125470f1fb5 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-preview.7.23376.3" + "dotnet": "8.0.100-rc.1.23455.8" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23463.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23508.1" } } From 54cf363ac4ca991f17d830cd0291a2cf53242a47 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 16 Oct 2023 12:15:46 +0000 Subject: [PATCH 27/47] Update dependencies from https://github.com/dotnet/arcade build 20231010.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23510.4 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/post-build/check-channel-consistency.ps1 | 2 +- global.json | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b7095ab0a1e..01377c5e4d1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 822f095b8c815dd7b9161140a9ff8151de593f82 + 0e89c8d7ddbb54510d9d0a757fda5126021b79f0 - + https://github.com/dotnet/arcade - 822f095b8c815dd7b9161140a9ff8151de593f82 + 0e89c8d7ddbb54510d9d0a757fda5126021b79f0 diff --git a/eng/Versions.props b/eng/Versions.props index a097e39ac27..f80124e6734 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23508.1 + 9.0.0-beta.23510.4 diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 index 63f3464c986..1728f035a93 100644 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -7,7 +7,7 @@ try { . $PSScriptRoot\post-build-utils.ps1 if ($PromoteToChannels -eq "") { - Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." ExitWithExitCode 0 } diff --git a/global.json b/global.json index 125470f1fb5..f6da3ee6a32 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-rc.1.23455.8" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23508.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23510.4" } } From 0514a042e287f3ff07b2dd106600bd5e009c8f73 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 23 Oct 2023 12:15:23 +0000 Subject: [PATCH 28/47] Update dependencies from https://github.com/dotnet/arcade build 20231018.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23518.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/build.sh | 2 +- eng/common/cross/build-rootfs.sh | 2 +- eng/common/cross/riscv64/tizen/tizen.patch | 9 +++++++++ eng/common/cross/tizen-build-rootfs.sh | 21 +++++++++++++++++++++ eng/common/cross/tizen-fetch.sh | 17 ++++++++++++++--- eng/common/cross/toolchain.cmake | 19 +++++++++++++------ eng/common/sdk-task.ps1 | 2 +- eng/common/templates/job/job.yml | 2 +- eng/common/tools.ps1 | 4 ++-- global.json | 4 ++-- 12 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 eng/common/cross/riscv64/tizen/tizen.patch diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 01377c5e4d1..857bb5ff21e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 0e89c8d7ddbb54510d9d0a757fda5126021b79f0 + 6a5ca678aef84f34a1fccc46f76623f41da56553 - + https://github.com/dotnet/arcade - 0e89c8d7ddbb54510d9d0a757fda5126021b79f0 + 6a5ca678aef84f34a1fccc46f76623f41da56553 diff --git a/eng/Versions.props b/eng/Versions.props index f80124e6734..7c7d5cf5123 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23510.4 + 9.0.0-beta.23518.2 diff --git a/eng/common/build.sh b/eng/common/build.sh index 50af40cdd2c..2c17ba529b9 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -105,7 +105,7 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; - -excludeCIBinarylog|-nobl) + -excludecibinarylog|-nobl) exclude_ci_binary_log=true ;; -pipelineslog|-pl) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 9caf9b021db..4228f202e5b 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -487,7 +487,7 @@ if [[ "$__CodeName" == "alpine" ]]; then -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ - search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')" + search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')" fi # install all packages in one go diff --git a/eng/common/cross/riscv64/tizen/tizen.patch b/eng/common/cross/riscv64/tizen/tizen.patch new file mode 100644 index 00000000000..eb6d1c07470 --- /dev/null +++ b/eng/common/cross/riscv64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-littleriscv) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-riscv64-lp64d.so.1 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-riscv64-lp64d.so.1 ) ) diff --git a/eng/common/cross/tizen-build-rootfs.sh b/eng/common/cross/tizen-build-rootfs.sh index ac84173d44f..ba31c93285f 100644 --- a/eng/common/cross/tizen-build-rootfs.sh +++ b/eng/common/cross/tizen-build-rootfs.sh @@ -22,6 +22,10 @@ case "$ARCH" in TIZEN_ARCH="x86_64" LINK_ARCH="x86" ;; + riscv64) + TIZEN_ARCH="riscv64" + LINK_ARCH="riscv" + ;; *) echo "Unsupported architecture for tizen: $ARCH" exit 1 @@ -58,4 +62,21 @@ rm -rf $TIZEN_TMP_DIR echo ">>Start configuring Tizen rootfs" ln -sfn asm-${LINK_ARCH} ./usr/include/asm patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +if [[ "$TIZEN_ARCH" == "riscv64" ]]; then + echo "Fixing broken symlinks in $PWD" + rm ./usr/lib64/libresolv.so + ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so + rm ./usr/lib64/libpthread.so + ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so + rm ./usr/lib64/libdl.so + ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so + rm ./usr/lib64/libutil.so + ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so + rm ./usr/lib64/libm.so + ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so + rm ./usr/lib64/librt.so + ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so + rm ./lib/ld-linux-riscv64-lp64d.so.1 + ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1 +fi echo "< Date: Mon, 30 Oct 2023 12:14:30 +0000 Subject: [PATCH 29/47] Update dependencies from https://github.com/dotnet/arcade build 20231028.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23528.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/toolchain.cmake | 10 +++++----- eng/common/tools.ps1 | 4 ++-- eng/common/tools.sh | 4 ++-- global.json | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 857bb5ff21e..8c2bcc4b01c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 6a5ca678aef84f34a1fccc46f76623f41da56553 + bb2b4326ddf03d86c92bd125dba87e9d8df63463 - + https://github.com/dotnet/arcade - 6a5ca678aef84f34a1fccc46f76623f41da56553 + bb2b4326ddf03d86c92bd125dba87e9d8df63463 diff --git a/eng/Versions.props b/eng/Versions.props index 7c7d5cf5123..88d59d0c07b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23518.2 + 9.0.0-beta.23528.2 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 38e96724f5b..00288a30065 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm") set(TOOLCHAIN "arm-linux-gnueabihf") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/13.1.0") + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) @@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(LINUX) set(TOOLCHAIN "aarch64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/13.1.0") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") @@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/13.1.0") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "armv6") set(CMAKE_SYSTEM_PROCESSOR armv6l) @@ -98,7 +98,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/13.1.0") + set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") @@ -115,7 +115,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TOOLCHAIN "i686-linux-gnu") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/13.1.0") + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") endif() else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index fdd0cbb91f8..e8def7e6a85 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -158,7 +158,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_MULTILEVEL_LOOKUP=0 # Disable first run since we do not need all ASP.NET packages restored. - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + $env:DOTNET_NOLOGO=1 # Disable telemetry on CI. if ($ci) { @@ -228,7 +228,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' - Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e8d47894334..710e64df967 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -112,7 +112,7 @@ function InitializeDotNetCli { export DOTNET_MULTILEVEL_LOOKUP=0 # Disable first run since we want to control all package sources - export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + export DOTNET_NOLOGO=1 # Disable telemetry on CI if [[ $ci == true ]]; then @@ -165,7 +165,7 @@ function InitializeDotNetCli { Write-PipelinePrependPath -path "$dotnet_root" Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" - Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" + Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" # return value _InitializeDotNetCli="$dotnet_root" diff --git a/global.json b/global.json index 28f9b248b1f..cbf549d39e8 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-rtm.23506.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23518.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23528.2" } } From 95ea7fb3fba938e45a8b8ec3212389c88af0add3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 6 Nov 2023 13:15:15 +0000 Subject: [PATCH 30/47] Update dependencies from https://github.com/dotnet/arcade build 20231103.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23553.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/tools.sh | 7 ++++++- global.json | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8c2bcc4b01c..de0f9d161d7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - bb2b4326ddf03d86c92bd125dba87e9d8df63463 + 1f067970472e5eb1877797bc4304dd13f284c7ba - + https://github.com/dotnet/arcade - bb2b4326ddf03d86c92bd125dba87e9d8df63463 + 1f067970472e5eb1877797bc4304dd13f284c7ba diff --git a/eng/Versions.props b/eng/Versions.props index 88d59d0c07b..82ed2fe82e8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23528.2 + 9.0.0-beta.23553.1 diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 710e64df967..e7ec245bc4a 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -341,7 +341,12 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="net8.0" + # use override if it exists - commonly set by source-build + if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then + _InitializeBuildToolFramework="net8.0" + else + _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" + fi } # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 diff --git a/global.json b/global.json index cbf549d39e8..837495dcb40 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-rtm.23506.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23528.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23553.1" } } From 33e854b7262c407df1ebad7e81d3e11e71034339 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 13 Nov 2023 13:14:56 +0000 Subject: [PATCH 31/47] Update dependencies from https://github.com/dotnet/arcade build 20231110.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23560.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/toolchain.cmake | 4 ++++ eng/common/tools.sh | 2 +- global.json | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index de0f9d161d7..15ade47d7a3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 1f067970472e5eb1877797bc4304dd13f284c7ba + 576b0a6fca70266087aa247d68896376ecec8c6e - + https://github.com/dotnet/arcade - 1f067970472e5eb1877797bc4304dd13f284c7ba + 576b0a6fca70266087aa247d68896376ecec8c6e diff --git a/eng/Versions.props b/eng/Versions.props index 82ed2fe82e8..8d233749cbc 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23553.1 + 9.0.0-beta.23560.1 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 00288a30065..3762640fdcf 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -288,6 +288,8 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64|riscv64)$") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() +elseif(TARGET_ARCH_NAME STREQUAL "s390x") + add_toolchain_linker_flag("--target=${TOOLCHAIN}") elseif(TARGET_ARCH_NAME STREQUAL "x86") if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) add_toolchain_linker_flag("--target=${TOOLCHAIN}") @@ -335,6 +337,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) endif() +elseif(TARGET_ARCH_NAME STREQUAL "s390x") + add_compile_options("--target=${TOOLCHAIN}") elseif(TARGET_ARCH_NAME STREQUAL "x86") if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) add_compile_options(--target=${TOOLCHAIN}) diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e7ec245bc4a..e98daf50c6b 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -310,7 +310,7 @@ function GetDotNetInstallScript { curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { if command -v openssl &> /dev/null; then echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" - echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 || true fi echo "Will now retry the same URL with verbose logging." with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { diff --git a/global.json b/global.json index 837495dcb40..30c51ac9408 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100-rtm.23506.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23553.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23560.1" } } From 411a691668db65ce8932f5cbac892cf8752cbaaf Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 20 Nov 2023 13:14:30 +0000 Subject: [PATCH 32/47] Update dependencies from https://github.com/dotnet/arcade build 20231117.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23567.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 15ade47d7a3..9112dba3a4f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 576b0a6fca70266087aa247d68896376ecec8c6e + b6801170e6ca8bf12bdff1f10d410c5e01660681 - + https://github.com/dotnet/arcade - 576b0a6fca70266087aa247d68896376ecec8c6e + b6801170e6ca8bf12bdff1f10d410c5e01660681 diff --git a/eng/Versions.props b/eng/Versions.props index 8d233749cbc..93a74a0c6a3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23560.1 + 9.0.0-beta.23567.1 diff --git a/global.json b/global.json index 30c51ac9408..aeca6a511fb 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100-rtm.23506.1" + "dotnet": "8.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23560.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23567.1" } } From ce873eaac1388cac884dd8f065f355618a95dc1e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 27 Nov 2023 13:14:35 +0000 Subject: [PATCH 33/47] Update dependencies from https://github.com/dotnet/arcade build 20231122.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23572.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9112dba3a4f..61757497109 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - b6801170e6ca8bf12bdff1f10d410c5e01660681 + c5e5051496efbcc9046b25628666377d6b97a670 - + https://github.com/dotnet/arcade - b6801170e6ca8bf12bdff1f10d410c5e01660681 + c5e5051496efbcc9046b25628666377d6b97a670 diff --git a/eng/Versions.props b/eng/Versions.props index 93a74a0c6a3..8c33010d62f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23567.1 + 9.0.0-beta.23572.2 diff --git a/global.json b/global.json index aeca6a511fb..487d556fc03 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23567.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23572.2" } } From 9471a374eecac12645a913181d20b1a8e80b78fa Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 4 Dec 2023 13:15:18 +0000 Subject: [PATCH 34/47] Update dependencies from https://github.com/dotnet/arcade build 20231201.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23601.1 --- eng/Version.Details.xml | 8 +- eng/Versions.props | 2 +- eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 2 +- .../post-build/add-build-to-channel.ps1 | 2 +- eng/common/post-build/publish-using-darc.ps1 | 2 +- eng/common/post-build/redact-logs.ps1 | 81 +++++++++++++++++++ .../post-build/trigger-subscriptions.ps1 | 2 +- .../templates/job/publish-build-assets.yml | 6 +- .../templates/post-build/common-variables.yml | 4 +- .../templates/post-build/post-build.yml | 1 + eng/common/templates/steps/publish-logs.yml | 28 ++++++- global.json | 2 +- 13 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 eng/common/post-build/redact-logs.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 61757497109..4c317649159 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - c5e5051496efbcc9046b25628666377d6b97a670 + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - c5e5051496efbcc9046b25628666377d6b97a670 + 38aa367e014493c6f6ebfe94d9029bea01d072c4 diff --git a/eng/Versions.props b/eng/Versions.props index 8c33010d62f..e1e0f122047 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23572.2 + 9.0.0-beta.23601.1 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 435e7641341..8fda30bdce2 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,6 +1,6 @@ param ( $darcVersion = $null, - $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16', + $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16', $verbosity = 'minimal', $toolpath = $null ) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 84c1d0cc2e7..c305ae6bd77 100644 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -2,7 +2,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' -versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16' +versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16' verbosity='minimal' while [[ $# > 0 ]]; do diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1 index de2d957922a..49938f0c89f 100644 --- a/eng/common/post-build/add-build-to-channel.ps1 +++ b/eng/common/post-build/add-build-to-channel.ps1 @@ -2,7 +2,7 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' ) diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 8508397d776..1e779fec4dd 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -3,7 +3,7 @@ param( [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, [Parameter(Mandatory=$true)][string] $MaestroToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 new file mode 100644 index 00000000000..82d91f6fd02 --- /dev/null +++ b/eng/common/post-build/redact-logs.ps1 @@ -0,0 +1,81 @@ +[CmdletBinding(PositionalBinding=$False)] +param( + [Parameter(Mandatory=$true, Position=0)][string] $InputPath, + [Parameter(Mandatory=$true)][string] $BinlogToolVersion, + [Parameter(Mandatory=$false)][string] $DotnetPath, + [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json', + # File with strings to redact - separated by newlines. + # For comments start the line with '# ' - such lines are ignored + [Parameter(Mandatory=$false)][string] $TokensFilePath, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + $packageName = 'binlogtool' + + $dotnet = $DotnetPath + + if (!$dotnet) { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + } + + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$packageName*") { + & "$dotnet" tool uninstall $packageName -g + } + + $toolPath = "$PSScriptRoot\..\..\..\.tools" + $verbosity = 'minimal' + + New-Item -ItemType Directory -Force -Path $toolPath + + Push-Location -Path $toolPath + + try { + Write-Host "Installing Binlog redactor CLI..." + Write-Host "'$dotnet' new tool-manifest" + & "$dotnet" new tool-manifest + Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + + if (Test-Path $TokensFilePath) { + Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath + $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " } + } + + $optionalParams = [System.Collections.ArrayList]::new() + + Foreach ($p in $TokensToRedact) + { + if($p -match '^\$\(.*\)$') + { + Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable" -f $p) + } + elseif($p) + { + $optionalParams.Add("-p:" + $p) | Out-Null + } + } + + & $dotnet binlogtool redact --input:$InputPath --recurse --in-place ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now." + } + } + finally { + Pop-Location + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_" + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index 55dea518ac5..ac9a95778fc 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -2,7 +2,7 @@ param( [Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' ) diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 42017109f37..3b25fd97923 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -48,8 +48,8 @@ jobs: - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/common-variables.yml + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/templates/post-build/common-variables.yml pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) @@ -81,7 +81,7 @@ jobs: arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index c24193acfc9..4ef7bd271fb 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -7,7 +7,7 @@ variables: # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" + value: "https://maestro.dot.net" - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion @@ -17,6 +17,8 @@ variables: value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.8 - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ef720f9d781..d64236b28cf 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -187,6 +187,7 @@ stages: parameters: StageLabel: 'Validation' JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) - job: displayName: SourceLink Validation diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 88f238f36bf..dadf1c464cd 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,6 +1,9 @@ parameters: StageLabel: '' JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.8' steps: - task: Powershell@2 @@ -12,7 +15,30 @@ steps: Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() - + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: diff --git a/global.json b/global.json index 487d556fc03..ea0627b2477 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23572.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23601.1" } } From 477cc8e1a628daaa9bfa964778ec0157b8418611 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 11 Dec 2023 13:29:26 +0000 Subject: [PATCH 35/47] Update dependencies from https://github.com/dotnet/arcade build 20231207.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23607.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4c317649159..e5be113e08e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 38aa367e014493c6f6ebfe94d9029bea01d072c4 + 3faeb9817f465151aa4bbcdb315f0a6170206760 - + https://github.com/dotnet/arcade - 38aa367e014493c6f6ebfe94d9029bea01d072c4 + 3faeb9817f465151aa4bbcdb315f0a6170206760 diff --git a/eng/Versions.props b/eng/Versions.props index e1e0f122047..42aded965f6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23601.1 + 9.0.0-beta.23607.2 diff --git a/global.json b/global.json index ea0627b2477..c7f5885a0ae 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "8.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23601.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23607.2" } } From 786281ebdd03659d2b1675fbe3ad1841dc6e8539 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 18 Dec 2023 13:30:59 +0000 Subject: [PATCH 36/47] Update dependencies from https://github.com/dotnet/arcade build 20231215.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23615.2 --- eng/Version.Details.xml | 8 ++--- eng/Versions.props | 2 +- eng/common/build.ps1 | 3 ++ eng/common/build.sh | 8 +++++ eng/common/sdl/trim-assets-version.ps1 | 2 +- .../templates/job/source-index-stage1.yml | 10 +++--- eng/common/templates/steps/source-build.yml | 4 +-- eng/common/tools.ps1 | 32 +++++++++---------- eng/common/tools.sh | 13 ++------ global.json | 4 +-- 10 files changed, 45 insertions(+), 41 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e5be113e08e..a0478aba7df 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 3faeb9817f465151aa4bbcdb315f0a6170206760 + 2f7acdc5f9b87f260119bb32d01c24d2773773eb - + https://github.com/dotnet/arcade - 3faeb9817f465151aa4bbcdb315f0a6170206760 + 2f7acdc5f9b87f260119bb32d01c24d2773773eb diff --git a/eng/Versions.props b/eng/Versions.props index 42aded965f6..1737f9ff636 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23607.2 + 9.0.0-beta.23615.2 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 33a6f2d0e24..066044f62f2 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -19,6 +19,7 @@ Param( [switch] $pack, [switch] $publish, [switch] $clean, + [switch] $verticalBuild, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -58,6 +59,7 @@ function Print-Usage() { Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" Write-Host " -clean Clean the solution" + Write-Host " -verticalBuild Run in 'vertical build' infra mode." Write-Host "" Write-Host "Advanced settings:" @@ -120,6 +122,7 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` + /p:ArcadeBuildVertical=$verticalBuild ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index 2c17ba529b9..5ce01dd161a 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -59,6 +59,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" restore=false build=false source_build=false +vertical_build=false rebuild=false test=false integration_test=false @@ -129,6 +130,12 @@ while [[ $# > 0 ]]; do restore=true pack=true ;; + -verticalbuild|-vb) + build=true + vertical_build=true + restore=true + pack=true + ;; -test|-t) test=true ;; @@ -220,6 +227,7 @@ function Build { /p:Restore=$restore \ /p:Build=$build \ /p:ArcadeBuildFromSource=$source_build \ + /p:ArcadeBuildVertical=$vertical_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ diff --git a/eng/common/sdl/trim-assets-version.ps1 b/eng/common/sdl/trim-assets-version.ps1 index a2e00487704..0daa2a9e946 100644 --- a/eng/common/sdl/trim-assets-version.ps1 +++ b/eng/common/sdl/trim-assets-version.ps1 @@ -72,4 +72,4 @@ catch { Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b98202aa02d..79523366262 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageVersion: 1.0.1-20231213.4 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -30,20 +30,20 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64.open + demands: ImageOverride -equals windows.vs2022.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 + demands: ImageOverride -equals windows.vs2022.amd64 steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 41bbb915736..1d79797363d 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -105,7 +105,7 @@ steps: Contents: | **/*.log **/*.binlog - artifacts/source-build/self/prebuilt-report/** + artifacts/sb/prebuilt-report/** TargetFolder: '$(Build.StagingDirectory)/BuildLogs' CleanTargetFolder: true continueOnError: true @@ -126,4 +126,4 @@ steps: - task: ComponentGovernanceComponentDetection@0 displayName: Component Detection (Exclude upstream cache) inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index e8def7e6a85..9048d603245 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -165,11 +165,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 } - # Source Build uses DotNetCoreSdkDir variable - if ($env:DotNetCoreSdkDir -ne $null) { - $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir - } - # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetExecutable = GetExecutableFileName 'dotnet' @@ -601,7 +596,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' } + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net9.0' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -676,8 +671,14 @@ function Read-ArcadeSdkVersion() { } function InitializeToolset() { - if (Test-Path variable:global:_ToolsetBuildProj) { - return $global:_ToolsetBuildProj + # For Unified Build/Source-build support, check whether the environment variable is + # set. If it is, then use this as the toolset build project. + if ($env:_InitializeToolset -ne $null) { + return $global:_InitializeToolset = $env:_InitializeToolset + } + + if (Test-Path variable:global:_InitializeToolset) { + return $global:_InitializeToolset } $nugetCache = GetNuGetPackageCachePath @@ -688,7 +689,7 @@ function InitializeToolset() { if (Test-Path $toolsetLocationFile) { $path = Get-Content $toolsetLocationFile -TotalCount 1 if (Test-Path $path) { - return $global:_ToolsetBuildProj = $path + return $global:_InitializeToolset = $path } } @@ -711,7 +712,7 @@ function InitializeToolset() { throw "Invalid toolset path: $path" } - return $global:_ToolsetBuildProj = $path + return $global:_InitializeToolset = $path } function ExitWithExitCode([int] $exitCode) { @@ -763,12 +764,10 @@ function MSBuild() { # new scripts need to work with old packages, so we need to look for the old names/versions (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) - (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll')) (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')) + (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')), + (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll')) ) $selectedPath = $null foreach ($path in $possiblePaths) { @@ -827,7 +826,8 @@ function MSBuild-Core() { } } - $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + # Be sure quote the path in case there are spaces in the dotnet installation location. + $env:ARCADE_BUILD_TOOL_COMMAND = "`"$($buildTool.Path)`" $cmdArgs" $exitCode = Exec-Process $buildTool.Path $cmdArgs diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e98daf50c6b..ece4b730795 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -123,11 +123,6 @@ function InitializeDotNetCli { # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" - # Source Build uses DotNetCoreSdkDir variable - if [[ -n "${DotNetCoreSdkDir:-}" ]]; then - export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" - fi - # Find the first path on $PATH that contains the dotnet.exe if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` @@ -343,7 +338,7 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" # use override if it exists - commonly set by source-build if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then - _InitializeBuildToolFramework="net8.0" + _InitializeBuildToolFramework="net9.0" else _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" fi @@ -458,12 +453,10 @@ function MSBuild { local possiblePaths=() possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" ) possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" ) + possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" ) for path in "${possiblePaths[@]}"; do if [[ -f $path ]]; then selectedPath=$path diff --git a/global.json b/global.json index c7f5885a0ae..d80a60c9458 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "8.0.100" + "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23607.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23615.2" } } From d07e6c0773f15a1d75bb98e5305e53390b0b59ed Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 25 Dec 2023 13:34:32 +0000 Subject: [PATCH 37/47] Update dependencies from https://github.com/dotnet/arcade build 20231221.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23621.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/build.cmd | 3 +++ eng/common/templates/job/job.yml | 2 +- eng/common/templates/job/publish-build-assets.yml | 2 +- eng/common/templates/post-build/common-variables.yml | 2 +- eng/common/templates/post-build/post-build.yml | 4 ++-- eng/common/templates/steps/publish-logs.yml | 2 +- global.json | 2 +- 9 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 eng/common/build.cmd diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a0478aba7df..9b50cf9c482 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 2f7acdc5f9b87f260119bb32d01c24d2773773eb + a1e4795fcba619bba3534700b660ad6d2e3ce859 - + https://github.com/dotnet/arcade - 2f7acdc5f9b87f260119bb32d01c24d2773773eb + a1e4795fcba619bba3534700b660ad6d2e3ce859 diff --git a/eng/Versions.props b/eng/Versions.props index 1737f9ff636..b10afc5ad62 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23615.2 + 9.0.0-beta.23621.2 diff --git a/eng/common/build.cmd b/eng/common/build.cmd new file mode 100644 index 00000000000..99daf368aba --- /dev/null +++ b/eng/common/build.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*" +exit /b %ErrorLevel% diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 0b01531357e..01c0dd995e4 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -136,7 +136,7 @@ jobs: condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 3b25fd97923..d9c5b4aaf9b 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -72,7 +72,7 @@ jobs: condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - task: PowerShell@2 displayName: Publish Build Assets diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 4ef7bd271fb..d7bf5c6e352 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -18,7 +18,7 @@ variables: - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion - value: 1.0.8 + value: 1.0.9 - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index d64236b28cf..bbc010fe732 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -169,7 +169,7 @@ stages: # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 displayName: 'Authenticate to AzDO Feeds' # Signing validation will optionally work with the buildmanifest file which is downloaded from @@ -267,7 +267,7 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - task: PowerShell@2 displayName: Publish Using Darc diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index dadf1c464cd..835e5275128 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -3,7 +3,7 @@ parameters: JobLabel: '' CustomSensitiveDataList: '' # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.8' + BinlogToolVersion: '1.0.9' steps: - task: Powershell@2 diff --git a/global.json b/global.json index d80a60c9458..6fd6b1cfbb6 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23615.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23621.2" } } From c0a10df8dbb07e71a4421fe2a11b8825cf88267c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 1 Jan 2024 13:34:59 +0000 Subject: [PATCH 38/47] Update dependencies from https://github.com/dotnet/arcade build 20231228.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.23628.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9b50cf9c482..152e4351d0a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - a1e4795fcba619bba3534700b660ad6d2e3ce859 + 9e5fd26c0d1a187dfa08242d83e341634d4b1b07 - + https://github.com/dotnet/arcade - a1e4795fcba619bba3534700b660ad6d2e3ce859 + 9e5fd26c0d1a187dfa08242d83e341634d4b1b07 diff --git a/eng/Versions.props b/eng/Versions.props index b10afc5ad62..92bf6c28438 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23621.2 + 9.0.0-beta.23628.1 diff --git a/global.json b/global.json index 6fd6b1cfbb6..d7bd361449b 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23621.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23628.1" } } From 32855a6925912bb1ecc29cf8a5d49768d2c06a5f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 15 Jan 2024 13:36:12 +0000 Subject: [PATCH 39/47] Update dependencies from https://github.com/dotnet/arcade build 20240112.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24062.5 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/cross/build-rootfs.sh | 6 +++--- eng/common/cross/riscv64/sources.list.sid | 2 +- eng/common/helixpublish.proj | 1 + eng/common/internal/Directory.Build.props | 2 ++ eng/common/internal/Tools.csproj | 2 ++ eng/common/templates/job/publish-build-assets.yml | 4 ++++ eng/common/tools.ps1 | 10 +++++++++- global.json | 2 +- 10 files changed, 28 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 152e4351d0a..c1fd88cc9a2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 9e5fd26c0d1a187dfa08242d83e341634d4b1b07 + d731f58a502086842739a358ab490bec08fdb8a7 - + https://github.com/dotnet/arcade - 9e5fd26c0d1a187dfa08242d83e341634d4b1b07 + d731f58a502086842739a358ab490bec08fdb8a7 diff --git a/eng/Versions.props b/eng/Versions.props index 92bf6c28438..41719fa4599 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.23628.1 + 9.0.0-beta.24062.5 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 4228f202e5b..9b2791cf568 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -182,12 +182,12 @@ while :; do __AlpinePackages="${__AlpinePackages// lldb-dev/}" __QEMUArch=riscv64 __UbuntuArch=riscv64 - __UbuntuRepo="http://deb.debian.org/debian-ports" + __UbuntuRepo="http://deb.debian.org/debian" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" unset __LLDB_Package - if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring" + if [[ -e "/usr/share/keyrings/debian-archive-keyring.gpg" ]]; then + __Keyring="--keyring /usr/share/keyrings/debian-archive-keyring.gpg --include=debian-archive-keyring" fi ;; ppc64le) diff --git a/eng/common/cross/riscv64/sources.list.sid b/eng/common/cross/riscv64/sources.list.sid index 65f730d224c..b5f7a7e6e1e 100644 --- a/eng/common/cross/riscv64/sources.list.sid +++ b/eng/common/cross/riscv64/sources.list.sid @@ -1 +1 @@ -deb http://deb.debian.org/debian-ports sid main +deb http://deb.debian.org/debian sid main diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj index d7f185856e7..c1323bf4121 100644 --- a/eng/common/helixpublish.proj +++ b/eng/common/helixpublish.proj @@ -1,3 +1,4 @@ + diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index dbf99d82a5c..a735fe9a133 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,4 +1,6 @@ + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 7f5ce6d6081..8fa77e5b181 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,5 +1,6 @@ + net472 false @@ -27,4 +28,5 @@ + diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index d9c5b4aaf9b..3115990d511 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -63,6 +63,10 @@ jobs: steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - checkout: self + fetchDepth: 3 + clean: true + - task: DownloadBuildArtifacts@0 displayName: Download artifact inputs: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9048d603245..0da65b5748a 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -596,7 +596,15 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net9.0' } + + # Use override if it exists - commonly set by source-build + if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { + $initializeBuildToolFramework="net9.0" + } else { + $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework + } + + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore diff --git a/global.json b/global.json index d7bd361449b..eecc4cff979 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23628.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24062.5" } } From 22945008b57c9761df93f98e214d777b0ab865b3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 22 Jan 2024 13:32:55 +0000 Subject: [PATCH 40/47] Update dependencies from https://github.com/dotnet/arcade build 20240119.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24069.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/build.ps1 | 3 +++ eng/common/build.sh | 13 +++++++++++++ eng/common/cross/build-rootfs.sh | 16 ++++++++-------- .../templates/post-build/common-variables.yml | 2 +- eng/common/templates/steps/publish-logs.yml | 2 +- eng/common/templates/steps/source-build.yml | 1 + global.json | 2 +- 9 files changed, 33 insertions(+), 16 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c1fd88cc9a2..4825a38b3a5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - d731f58a502086842739a358ab490bec08fdb8a7 + abddd0bd5145578246dcadda264c7557f2a935a9 - + https://github.com/dotnet/arcade - d731f58a502086842739a358ab490bec08fdb8a7 + abddd0bd5145578246dcadda264c7557f2a935a9 diff --git a/eng/Versions.props b/eng/Versions.props index 41719fa4599..3487eea6c10 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24062.5 + 9.0.0-beta.24069.2 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 066044f62f2..510458eb35b 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -20,6 +20,7 @@ Param( [switch] $publish, [switch] $clean, [switch] $verticalBuild, + [switch][Alias('pb')]$productBuild, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -60,6 +61,7 @@ function Print-Usage() { Write-Host " -publish Publish artifacts (e.g. symbols)" Write-Host " -clean Clean the solution" Write-Host " -verticalBuild Run in 'vertical build' infra mode." + Write-Host " -productBuild Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)" Write-Host "" Write-Host "Advanced settings:" @@ -122,6 +124,7 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` + /p:DotNetBuildRepo=$($productBuild -or $verticalBuild) ` /p:ArcadeBuildVertical=$verticalBuild ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` diff --git a/eng/common/build.sh b/eng/common/build.sh index 5ce01dd161a..2dfb32f2ec4 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -22,6 +22,9 @@ usage() echo " --sourceBuild Source-build the solution (short: -sb)" echo " Will additionally trigger the following actions: --restore, --build, --pack" echo " If --configuration is not set explicitly, will also set it to 'Release'" + echo " --productBuild Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)" + echo " Will additionally trigger the following actions: --restore, --build, --pack" + echo " If --configuration is not set explicitly, will also set it to 'Release'" echo " --rebuild Rebuild solution" echo " --test Run all unit tests in the solution (short: -t)" echo " --integrationTest Run all integration tests in the solution" @@ -60,6 +63,7 @@ restore=false build=false source_build=false vertical_build=false +product_build=false rebuild=false test=false integration_test=false @@ -127,12 +131,20 @@ while [[ $# > 0 ]]; do -sourcebuild|-sb) build=true source_build=true + product_build=true + restore=true + pack=true + ;; + -productBuild|-pb) + build=true + product_build=true restore=true pack=true ;; -verticalbuild|-vb) build=true vertical_build=true + product_build=true restore=true pack=true ;; @@ -226,6 +238,7 @@ function Build { /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ + /p:DotNetBuildRepo=$product_build \ /p:ArcadeBuildFromSource=$source_build \ /p:ArcadeBuildVertical=$vertical_build \ /p:Rebuild=$rebuild \ diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 9b2791cf568..9fa764e78b0 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -8,7 +8,7 @@ usage() echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" - echo " for FreeBSD can be: freebsd12, freebsd13" + echo " for FreeBSD can be: freebsd13, freebsd14" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -71,9 +71,9 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.4-RELEASE" -__FreeBSDPkg="1.17.0" -__FreeBSDABI="12" +__FreeBSDBase="13.2-RELEASE" +__FreeBSDPkg="1.20.0" +__FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" @@ -334,14 +334,14 @@ while :; do __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinoVersion" fi ;; - freebsd12) + freebsd13) __CodeName=freebsd __SkipUnmount=1 ;; - freebsd13) + freebsd14) __CodeName=freebsd - __FreeBSDBase="13.2-RELEASE" - __FreeBSDABI="13" + __FreeBSDBase="14.0-RELEASE" + __FreeBSDABI="14" __SkipUnmount=1 ;; illumos) diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index d7bf5c6e352..b9ede10bf09 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -18,7 +18,7 @@ variables: - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion - value: 1.0.9 + value: 1.0.11 - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 835e5275128..80861297ddc 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -3,7 +3,7 @@ parameters: JobLabel: '' CustomSensitiveDataList: '' # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.9' + BinlogToolVersion: '1.0.11' steps: - task: Powershell@2 diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 1d79797363d..1793eb6beb9 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -94,6 +94,7 @@ steps: $baseOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true \ + /p:DotNetBuildRepo=true \ /p:AssetManifestFileName=$assetManifestFileName displayName: Build diff --git a/global.json b/global.json index eecc4cff979..17a781ea135 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24062.5" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24069.2" } } From fc41f7e87c4afad883f3dea78b590d8b62e1ed2c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 29 Jan 2024 13:34:44 +0000 Subject: [PATCH 41/47] Update dependencies from https://github.com/dotnet/arcade build 20240126.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24076.5 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/build.sh | 1 + eng/common/cross/build-rootfs.sh | 2 +- eng/common/templates/steps/source-build.yml | 1 + global.json | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4825a38b3a5..2e3c6c2c02e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - abddd0bd5145578246dcadda264c7557f2a935a9 + 66c9c5397d599af40f2a94989241944f5a73442a - + https://github.com/dotnet/arcade - abddd0bd5145578246dcadda264c7557f2a935a9 + 66c9c5397d599af40f2a94989241944f5a73442a diff --git a/eng/Versions.props b/eng/Versions.props index 3487eea6c10..1d14c0da655 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24069.2 + 9.0.0-beta.24076.5 diff --git a/eng/common/build.sh b/eng/common/build.sh index 2dfb32f2ec4..bec7d02594f 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -240,6 +240,7 @@ function Build { /p:Build=$build \ /p:DotNetBuildRepo=$product_build \ /p:ArcadeBuildFromSource=$source_build \ + /p:DotNetBuildSourceOnly=$source_build \ /p:ArcadeBuildVertical=$vertical_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 9fa764e78b0..99a9dd6bb60 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -72,7 +72,7 @@ __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" __FreeBSDBase="13.2-RELEASE" -__FreeBSDPkg="1.20.0" +__FreeBSDPkg="1.17.0" __FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 1793eb6beb9..32738aa9380 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -94,6 +94,7 @@ steps: $baseOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true \ + /p:DotNetBuildSourceOnly=true \ /p:DotNetBuildRepo=true \ /p:AssetManifestFileName=$assetManifestFileName displayName: Build diff --git a/global.json b/global.json index 17a781ea135..890e389a615 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24069.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24076.5" } } From 49ae840b1082a2a6fddb3bc7d2fff885dccd0076 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 5 Feb 2024 13:33:11 +0000 Subject: [PATCH 42/47] Update dependencies from https://github.com/dotnet/arcade build 20240202.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24102.4 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/sdk-task.ps1 | 2 +- eng/common/templates/job/source-index-stage1.yml | 2 +- eng/common/tools.ps1 | 4 ++-- global.json | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2e3c6c2c02e..7210b4ed3bd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 66c9c5397d599af40f2a94989241944f5a73442a + 2fb543a45580400a559b5ae41c96a815ea14dac5 - + https://github.com/dotnet/arcade - 66c9c5397d599af40f2a94989241944f5a73442a + 2fb543a45580400a559b5ae41c96a815ea14dac5 diff --git a/eng/Versions.props b/eng/Versions.props index 1d14c0da655..5e2130d12fc 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24076.5 + 9.0.0-beta.24102.4 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 73828dd30d3..091023970f1 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.5" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 79523366262..b5a3e5c4a6c 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20231213.4 + sourceIndexPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 0da65b5748a..a8f8edb3291 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,8 +379,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2 - $defaultXCopyMSBuildVersion = '17.8.1-2' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.5 + $defaultXCopyMSBuildVersion = '17.8.5' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index 890e389a615..52f1352d639 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24076.5" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24102.4" } } From fae10a36326d0f14e8458f8af33338ffc46dd3e5 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 12 Feb 2024 13:28:13 +0000 Subject: [PATCH 43/47] Update dependencies from https://github.com/dotnet/arcade build 20240206.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24106.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/native/init-os-and-arch.sh | 4 ++++ eng/common/tools.ps1 | 6 +++--- global.json | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7210b4ed3bd..3eef2c82410 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 2fb543a45580400a559b5ae41c96a815ea14dac5 + f7eb7794c703dc29a83b414b786e9a154f0ca042 - + https://github.com/dotnet/arcade - 2fb543a45580400a559b5ae41c96a815ea14dac5 + f7eb7794c703dc29a83b414b786e9a154f0ca042 diff --git a/eng/Versions.props b/eng/Versions.props index 5e2130d12fc..4de86b1f6ea 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24102.4 + 9.0.0-beta.24106.2 diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh index e693617a6c2..caa448ff030 100644 --- a/eng/common/native/init-os-and-arch.sh +++ b/eng/common/native/init-os-and-arch.sh @@ -35,6 +35,10 @@ fi case "$CPUName" in arm64|aarch64) arch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + arch=arm + fi ;; loongarch64) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a8f8edb3291..7d8dc89b919 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,7 +379,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.5 + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.8.5 $defaultXCopyMSBuildVersion = '17.8.5' if (!$vsRequirements) { @@ -445,7 +445,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." + throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." } } if ($vsInstallDir -eq $null) { @@ -482,7 +482,7 @@ function InstallXCopyMSBuild([string]$packageVersion) { } function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'RoslynTools.MSBuild' + $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" diff --git a/global.json b/global.json index 52f1352d639..38ba7ab1128 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24102.4" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24106.2" } } From 206f21ec1c500dac6d112c6781852707f4681343 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 19 Feb 2024 13:33:31 +0000 Subject: [PATCH 44/47] Update dependencies from https://github.com/dotnet/arcade build 20240214.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24114.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- eng/common/post-build/publish-using-darc.ps1 | 4 ++-- .../templates/job/publish-build-assets.yml | 16 ++++++++-------- eng/common/templates/post-build/post-build.yml | 16 ++++++++-------- eng/common/templates/steps/send-to-helix.yml | 6 ++++-- .../templates/variables/pool-providers.yml | 12 ++++++------ global.json | 2 +- 8 files changed, 34 insertions(+), 32 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3eef2c82410..26b33a88224 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - f7eb7794c703dc29a83b414b786e9a154f0ca042 + d5b02a4900c4d521cb48b8f0d7e3f28175268f7c - + https://github.com/dotnet/arcade - f7eb7794c703dc29a83b414b786e9a154f0ca042 + d5b02a4900c4d521cb48b8f0d7e3f28175268f7c diff --git a/eng/Versions.props b/eng/Versions.props index 4de86b1f6ea..b6e4ff3482d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24106.2 + 9.0.0-beta.24114.1 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1e779fec4dd..5a3a32ea8d7 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -12,7 +12,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() @@ -46,7 +46,7 @@ try { } Write-Host 'done.' -} +} catch { Write-Host $_ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 3115990d511..bb42240f865 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -58,7 +58,7 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: @@ -66,7 +66,7 @@ jobs: - checkout: self fetchDepth: 3 clean: true - + - task: DownloadBuildArtifacts@0 displayName: Download artifact inputs: @@ -75,7 +75,7 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 - task: PowerShell@2 @@ -90,7 +90,7 @@ jobs: /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -99,7 +99,7 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: @@ -125,7 +125,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') + condition: eq(variables['SymbolExclusionFile'], 'true') inputs: PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' PublishLocation: Container @@ -141,7 +141,7 @@ jobs: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' @@ -152,4 +152,4 @@ jobs: - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index bbc010fe732..ee70e2b399c 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -39,7 +39,7 @@ parameters: displayName: Enable NuGet validation type: boolean default: true - + - name: publishInstallersAndChecksums displayName: Publish installers and checksums type: boolean @@ -131,8 +131,8 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - job: displayName: Signing Validation @@ -222,9 +222,9 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) continueOnError: true @@ -259,7 +259,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -273,7 +273,7 @@ stages: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 3eb7e2d5f84..68fa739c4ab 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,6 +5,8 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution @@ -29,7 +31,7 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -59,7 +61,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 9cc5c550d3b..d236f9fdbb1 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,15 +1,15 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. -# Motivation: +# Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing # (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. -# How to use: +# How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). # If we find alternate naming conventions in broad usage it can be added to the condition below. # @@ -54,4 +54,4 @@ variables: False, 'NetCore1ESPool-Internal' ) - ] \ No newline at end of file + ] diff --git a/global.json b/global.json index 38ba7ab1128..4640aabf3dc 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24106.2" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24114.1" } } From a76ffd52e35aad596fad497bc2581e6f227e123c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 26 Feb 2024 13:17:06 +0000 Subject: [PATCH 45/47] Update dependencies from https://github.com/dotnet/arcade build 20240214.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24114.1 From cf0320641e3cb7a8ce5ece0969ae832ef4f3c48c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 4 Mar 2024 13:37:49 +0000 Subject: [PATCH 46/47] Update dependencies from https://github.com/dotnet/arcade build 20240301.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24151.5 --- eng/Version.Details.xml | 8 +- eng/Versions.props | 2 +- eng/common/native/init-distro-rid.sh | 80 +++-- eng/common/native/init-os-and-arch.sh | 3 +- eng/common/templates-official/job/job.yml | 255 ++++++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++++ .../job/publish-build-assets.yml | 157 ++++++++++ .../templates-official/job/source-build.yml | 67 ++++ .../job/source-index-stage1.yml | 67 ++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 ++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 24 ++ .../post-build/post-build.yml | 285 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../steps/component-governance.yml | 13 + .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 49 +++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 ++++++ .../templates-official/steps/source-build.yml | 129 ++++++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + eng/common/tools.ps1 | 8 +- eng/common/tools.sh | 8 +- global.json | 4 +- 28 files changed, 1707 insertions(+), 53 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 26b33a88224..58ad7b1bb93 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - d5b02a4900c4d521cb48b8f0d7e3f28175268f7c + 87d89025bdd8827c016e4083660d31f497670e5c - + https://github.com/dotnet/arcade - d5b02a4900c4d521cb48b8f0d7e3f28175268f7c + 87d89025bdd8827c016e4083660d31f497670e5c diff --git a/eng/Versions.props b/eng/Versions.props index b6e4ff3482d..3c70b8432c1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24114.1 + 9.0.0-beta.24151.5 diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index de1687b2ccb..5dcbfd700f0 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # getNonPortableDistroRid # @@ -11,21 +11,20 @@ # non-portable rid getNonPortableDistroRid() { - local targetOs="$1" - local targetArch="$2" - local rootfsDir="$3" - local nonPortableRid="" + targetOs="$1" + targetArch="$2" + rootfsDir="$3" + nonPortableRid="" if [ "$targetOs" = "linux" ]; then + # shellcheck disable=SC1091 if [ -e "${rootfsDir}/etc/os-release" ]; then - source "${rootfsDir}/etc/os-release" - - if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then - # remove the last version digit - VERSION_ID="${VERSION_ID%.*}" + . "${rootfsDir}/etc/os-release" + if [ "${ID}" = "rhel" ] || [ "${ID}" = "rocky" ] || [ "${ID}" = "alpine" ]; then + VERSION_ID="${VERSION_ID%.*}" # Remove the last version digit for these distros fi - if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then + if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" else # Rolling release distros either do not set VERSION_ID, set it as blank or @@ -33,45 +32,45 @@ getNonPortableDistroRid() # so omit it here to be consistent with everything else. nonPortableRid="${ID}-${targetArch}" fi - elif [ -e "${rootfsDir}/android_platform" ]; then - source "$rootfsDir"/android_platform + # shellcheck disable=SC1091 + . "${rootfsDir}/android_platform" nonPortableRid="$RID" fi fi if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is shell script and it should always work. - __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; }) + # $rootfsDir can be empty. freebsd-version is a shell script and should always work. + __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then + elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" elif [ "$targetOs" = "illumos" ]; then __uname_version=$(uname -v) case "$__uname_version" in omnios-*) - __omnios_major_version=$(echo "${__uname_version:8:2}") - nonPortableRid=omnios."$__omnios_major_version"-"$targetArch" - ;; + __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) + nonPortableRid="omnios.$__omnios_major_version-${targetArch}" + ;; joyent_*) - __smartos_major_version=$(echo "${__uname_version:7:4}") - nonPortableRid=smartos."$__smartos_major_version"-"$targetArch" - ;; - illumos_*) - nonPortableRid=openindiana-"$targetArch" - ;; + __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) + nonPortableRid="smartos.$__smartos_major_version-${targetArch}" + ;; + *) + nonPortableRid="illumos-${targetArch}" + ;; esac elif [ "$targetOs" = "solaris" ]; then __uname_version=$(uname -v) - __solaris_major_version=$(echo "${__uname_version%.*}") - nonPortableRid=solaris."$__solaris_major_version"-"$targetArch" + __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) + nonPortableRid="solaris.$__solaris_major_version-${targetArch}" elif [ "$targetOs" = "haiku" ]; then - __uname_release=$(uname -r) + __uname_release="$(uname -r)" nonPortableRid=haiku.r"$__uname_release"-"$targetArch" fi - echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')" + echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' } # initDistroRidGlobal @@ -85,26 +84,23 @@ getNonPortableDistroRid() # None # # Notes: -# -# It is important to note that the function does not return anything, but it -# exports the following variables on success: -# -# __DistroRid : Non-portable rid of the target platform. -# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. -# +# It is important to note that the function does not return anything, but it +# exports the following variables on success: +# __DistroRid : Non-portable rid of the target platform. +# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. initDistroRidGlobal() { - local targetOs="$1" - local targetArch="$2" - local rootfsDir="" - if [ "$#" -ge 3 ]; then + targetOs="$1" + targetArch="$2" + rootfsDir="" + if [ $# -ge 3 ]; then rootfsDir="$3" fi if [ -n "${rootfsDir}" ]; then # We may have a cross build. Check for the existence of the rootfsDir if [ ! -e "${rootfsDir}" ]; then - echo "Error rootfsDir has been passed, but the location is not valid." + echo "Error: rootfsDir has been passed, but the location is not valid." exit 1 fi fi @@ -119,7 +115,7 @@ initDistroRidGlobal() STRINGS="$(command -v llvm-strings || true)" fi - # Check for musl-based distros (e.g Alpine Linux, Void Linux). + # Check for musl-based distros (e.g. Alpine Linux, Void Linux). if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then __PortableTargetOS="linux-musl" diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh index caa448ff030..38921d4338f 100644 --- a/eng/common/native/init-os-and-arch.sh +++ b/eng/common/native/init-os-and-arch.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # Use uname to determine what the OS is. OSName=$(uname -s | tr '[:upper:]' '[:lower:]') @@ -54,6 +54,7 @@ case "$CPUName" in ;; armv7l|armv8l) + # shellcheck disable=SC1091 if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then arch=armel else diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 00000000000..352607308fd --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,255 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration + artifact: BuildConfiguration + displayName: Publish build retry configuration + continueOnError: true diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 00000000000..ba9ba493032 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 00000000000..5f54135569b --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,157 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - checkout: self + fetchDepth: 3 + clean: true + + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 00000000000..50f04e642a3 --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals build.ubuntu.1804.amd64 + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 00000000000..757af7c7c4f --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,67 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20231213.4 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: windows.vs2022.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET 8 SDK + inputs: + packageType: sdk + version: 8.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 00000000000..b68d3c2f319 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 00000000000..857a0f8ba43 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 00000000000..08e5db9bb11 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 00000000000..b9ede10bf09 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,24 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 00000000000..5c98fe1c0f3 --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 00000000000..0c87f149a4a --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 00000000000..da669030daf --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 00000000000..f67a210d62f --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 00000000000..0ecec47b0c9 --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 00000000000..488b560e8ba --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 00000000000..84b2f559c56 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,49 @@ +parameters: + StageLabel: '' + JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/templates-official/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.11' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 00000000000..83d97a26a01 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 00000000000..3eb7e2d5f84 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 00000000000..b1db70842f5 --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/sb/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 00000000000..beab7d1bfba --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 00000000000..dbdd66d4a4b --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7d8dc89b919..9bf873e3c25 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,6 +65,11 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } +# True if the build is a product build +[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } + +[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } + function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -850,7 +855,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) { + # Skip this when the build is a child of the VMR orchestrator build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and $properties -notlike "*DotNetBuildRepo=true*") { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index ece4b730795..db64e298ff6 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,6 +68,9 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} +# True if the build is a product build +product_build=${product_build:-false} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -141,7 +144,7 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="$repo_root/.dotnet" + dotnet_root="${repo_root}.dotnet" export DOTNET_INSTALL_DIR="$dotnet_root" @@ -503,7 +506,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then + # Skip this when the build is a child of the VMR orchestrator build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/global.json b/global.json index 4640aabf3dc..e620bdc15a4 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "__comment__": "This .NET global.json file is required to be in this location by the Arcade SDK and is otherwise unused. If it isn't present, attempting to auto-update the Arcade SDK throws errors.", "tools": { - "dotnet": "9.0.100-alpha.1.23615.4" + "dotnet": "9.0.100-preview.1.24101.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24114.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24151.5" } } From ba7803349b5a4252c6e18ddf86f4a43319f8071d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 11 Mar 2024 12:36:06 +0000 Subject: [PATCH 47/47] Update dependencies from https://github.com/dotnet/arcade build 20240309.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.SignTool From Version 8.0.0-beta.23174.1 -> To Version 9.0.0-beta.24159.1 --- eng/Version.Details.xml | 8 +++--- eng/Versions.props | 2 +- eng/common/SetupNugetSources.ps1 | 26 +++++++++---------- eng/common/build.ps1 | 1 - eng/common/build.sh | 1 - eng/common/templates-official/job/job.yml | 4 +++ .../job/source-index-stage1.yml | 2 +- .../steps/send-to-helix.yml | 6 +++-- .../templates-official/steps/source-build.yml | 2 ++ eng/common/templates/job/job.yml | 4 +++ eng/common/tools.ps1 | 2 +- global.json | 2 +- 12 files changed, 35 insertions(+), 25 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 58ad7b1bb93..b5583e1710f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,13 +1,13 @@ - + https://github.com/dotnet/arcade - 87d89025bdd8827c016e4083660d31f497670e5c + f4538b1f8ff5ceb197aea509f3f61872b217b09e - + https://github.com/dotnet/arcade - 87d89025bdd8827c016e4083660d31f497670e5c + f4538b1f8ff5ceb197aea509f3f61872b217b09e diff --git a/eng/Versions.props b/eng/Versions.props index 3c70b8432c1..2ac872a577e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 9.0.0-beta.24151.5 + 9.0.0-beta.24159.1 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925f..efa2fd72bfa 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 510458eb35b..83e6d82e027 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -125,7 +125,6 @@ function Build { /p:Test=$test ` /p:Pack=$pack ` /p:DotNetBuildRepo=$($productBuild -or $verticalBuild) ` - /p:ArcadeBuildVertical=$verticalBuild ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index bec7d02594f..d82ebf74280 100644 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -241,7 +241,6 @@ function Build { /p:DotNetBuildRepo=$product_build \ /p:ArcadeBuildFromSource=$source_build \ /p:DotNetBuildSourceOnly=$source_build \ - /p:ArcadeBuildVertical=$vertical_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 352607308fd..1df0201243b 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 757af7c7c4f..53a9ef51fd8 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20231213.4 + sourceIndexPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml index 3eb7e2d5f84..68fa739c4ab 100644 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -5,6 +5,8 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution @@ -29,7 +31,7 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -59,7 +61,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index b1db70842f5..53ed57b6d48 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -94,6 +94,8 @@ steps: $baseOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true \ + /p:DotNetBuildSourceOnly=true \ + /p:DotNetBuildRepo=true \ /p:AssetManifestFileName=$assetManifestFileName displayName: Build diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 01c0dd995e4..a3277bf15c5 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9bf873e3c25..a58aef2847e 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -856,7 +856,7 @@ function MSBuild-Core() { # When running on Azure Pipelines, override the returned exit code to avoid double logging. # Skip this when the build is a child of the VMR orchestrator build. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and $properties -notlike "*DotNetBuildRepo=true*") { + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/global.json b/global.json index e620bdc15a4..ef6a00186c1 100644 --- a/global.json +++ b/global.json @@ -4,6 +4,6 @@ "dotnet": "9.0.100-preview.1.24101.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24151.5" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24159.1" } }