diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json new file mode 100644 index 0000000..bd2a6a0 --- /dev/null +++ b/.config/tsaoptions.json @@ -0,0 +1,11 @@ +{ + "instanceUrl": "https://msazure.visualstudio.com", + "projectName": "One", + "areaPath": "One\\MGMT\\Compute\\Powershell\\Powershell\\PowerShell Core", + "notificationAliases": [ + "adityap@microsoft.com", + "dongbow@microsoft.com", + "pmeinecke@microsoft.com", + "tplunk@microsoft.com" + ] +} diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml new file mode 100644 index 0000000..2d81bb6 --- /dev/null +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -0,0 +1,97 @@ + +parameters: +- name: release + type: string + displayName: | + Use `stable` to release by default. + + Use `private` if you want to create a branch on the store to test the package. + This will create a branch under the edge branch automatically that are difficult, but not impossible to find. + + `candidate`, `beta`, and `edge` are public but we don't have any existing meaning for these channels. + values: + - private + - stable + - candidate + - beta + - edge + default: stable + +trigger: none + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] # needed for onebranch.pipeline.version task + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 # Docker image which is used to build the project + - name: DEBIAN_FRONTEND + value: noninteractive + - group: poolNames + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates + parameters: + git: + fetchDepth: 1 + # windows only feature + #longpaths: true + retryCount: 3 + # we don't use this and some of our agents doesn't have the feature installed + lfs: false + cloudvault: + enabled: false # set to true to enable cloudvault + runmode: stage # linux can run CloudVault upload as a separate stage + dependsOn: linux_build + artifacts: + - drop_linux_stage_linux_job + customTags: 'ES365AIMigrationTooling' + globalSdl: + disableLegacyManifest: true + # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + compiled: + enabled: false + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + cg: + enabled: true + asyncSdl: # https://aka.ms/obpipelines/asyncsdl + enabled: true + forStages: [scan_lts, scan_stable, scan_preview] + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + #suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + binskim: + enabled: false + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config\tsaoptions.json + + stages: + - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self + parameters: + channel: stable + release: ${{ parameters.release }} + + - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self + parameters: + channel: preview + release: ${{ parameters.release }} + + - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self + parameters: + channel: lts + release: ${{ parameters.release }} diff --git a/.pipelines/templates/Approval.yml b/.pipelines/templates/Approval.yml new file mode 100644 index 0000000..4d9d476 --- /dev/null +++ b/.pipelines/templates/Approval.yml @@ -0,0 +1,31 @@ +parameters: + - name: displayName + type: string + - name: instructions + type: string + - name: jobName + type: string + - name: timeoutInMinutes + type: number + # 2 days + default: 2880 + - name: onTimeout + type: string + default: 'reject' + values: + - resume + - reject + +jobs: + - job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + pool: + type: agentless + timeoutInMinutes: 4320 # job times out in 3 days + steps: + - task: ManualValidation@0 + displayName: ${{ parameters.displayName }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + inputs: + instructions: ${{ parameters.instructions }} + onTimeout: ${{ parameters.onTimeout }} diff --git a/.pipelines/templates/InstallSnapd.yml b/.pipelines/templates/InstallSnapd.yml new file mode 100644 index 0000000..a8ed4f5 --- /dev/null +++ b/.pipelines/templates/InstallSnapd.yml @@ -0,0 +1,11 @@ +steps: + + - bash: | + sudo apt update + displayName: Apt Update + condition: succeeded() + + - bash: | + sudo apt-get -y install snapd + displayName: Install snapd + condition: succeeded() diff --git a/.pipelines/templates/createOutputDirectory-linux.yml b/.pipelines/templates/createOutputDirectory-linux.yml new file mode 100644 index 0000000..6f00b62 --- /dev/null +++ b/.pipelines/templates/createOutputDirectory-linux.yml @@ -0,0 +1,6 @@ +steps: + + - bash: | + mkdir -p -m a=rwx $(ob_outputDirectory) + displayName: Create $(ob_outputDirectory) + condition: succeeded() diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml new file mode 100644 index 0000000..bcd3151 --- /dev/null +++ b/.pipelines/templates/pushJob.yml @@ -0,0 +1,111 @@ +parameters: + - name: channel + type: string + - name: release + default: 'private' + +jobs: +- job: push + displayName: Push to ${{ parameters.release }} + + pool: + type: linux + isCustom: true + name: $(ubuntuPool) + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure + + variables: + - name: ReleaseTag + value: edge/default + - group: SnapLogin + - name: channel + value: ${{ parameters.channel }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + - name: DOWNLOAD_DIRECTORY + value: '$(Build.ArtifactStagingDirectory)/down' + + steps: + - checkout: self + lfs: false + + - template: ./createOutputDirectory-linux.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download build files' + inputs: + targetPath: $(DOWNLOAD_DIRECTORY) + artifact: drop_build_$(channel)_build_$(channel) + + - pwsh: | + $snaps = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.snap -recurse -File + if($snaps.Count -gt 1) + { + $snaps | out-string -width 100 | Write-Verbose -verbose + Write-Error "***More than one snap found***" -errorAction stop + } + displayName: Verify that only one snap was downloaded + + - pwsh: | + [string]$Branch=$env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + if('${{ parameters.release }}' -eq 'private') { + if($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + { + Write-verbose "release branch: $branchOnly" -verbose + $generatedBranch = ([guid]::NewGuid()).ToString().Replace('-','') + $releaseTag = "edge/$generatedBranch" + } + else + { + Write-verbose "non-release branch" -verbose + # Branch is named <previewname> + $releaseTag = "edge/$branchOnly" + $releaseTag += ([guid]::NewGuid()).ToString().Replace('-','') + } + } + else { + $releaseTag = "${{ parameters.release }}" + } + + $vstsCommandString = "vso[task.setvariable variable=ReleaseTag]$releaseTag" + Write-Verbose -Message "setting ReleaseTag to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set ReleaseTag Variable + + - pwsh: | + sudo chown root:root / + displayName: 'Make sure root owns root' + condition: succeeded() + + - template: ./InstallSnapd.yml + + - pwsh: | + $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel + Write-Verbose -Verbose -Message "using Channel $channel" + sudo snap install snapcraft --classic "--channel=$channel" + condition: succeeded() + displayName: 'Install snapcraft' + retryCountOnTaskFailure: 2 + + - pwsh: | + $track = 'latest' + if('$(channel)' -eq 'lts') + { + $track = 'lts' + } + + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File | Select-Object -ExpandProperty FullName + + foreach($snap in $snaps) + { + Write-Verbose -Verbose -Message "Uploading $snap to $track/$(ReleaseTag)" + snapcraft upload --release "$track/$(ReleaseTag)" $snap + } + displayName: 'snapcraft upload' + retryCountOnTaskFailure: 1 + env: + SNAPCRAFT_STORE_CREDENTIALS: $(SNAPCRAFT_STORE_CREDENTIALS) diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml new file mode 100644 index 0000000..203bb3d --- /dev/null +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -0,0 +1,31 @@ +parameters: + - name: channel + default: '' + - name: release + default: 'private' + +stages: +- stage: build_${{ parameters.channel }} + displayName: Build - ${{ parameters.channel }} + dependsOn: [] + jobs: + - template: ./releaseBuildJob.yml + parameters: + channel: ${{ parameters.channel }} + +- stage: scan_${{ parameters.channel }} + displayName: SDL - ${{ parameters.channel }} + dependsOn: build_${{ parameters.channel }} + jobs: + - template: ./scanJob.yml + parameters: + channel: ${{ parameters.channel }} + +- stage: Push_${{ parameters.channel }} + displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} + dependsOn: scan_${{ parameters.channel }} + jobs: + - template: ./pushJob.yml + parameters: + channel: ${{ parameters.channel }} + release: ${{ parameters.release }} diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml new file mode 100644 index 0000000..c3d81a4 --- /dev/null +++ b/.pipelines/templates/releaseBuildJob.yml @@ -0,0 +1,168 @@ +parameters: + - name: channel + default: 'stable' + - name: test + default: Yes + +jobs: + +- job: build_${{ parameters.channel }} + displayName: Build ${{ parameters.channel }} + variables: + - name: channel + value: ${{ parameters.channel }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + - name: SNAP_MOUNT + value: $(Agent.TempDirectory)/pwshSnap + - name: SNAP_COPY + value: $(Agent.TempDirectory)/pwshSnapCopy + - name: SNAP_TARGZ + value: $(Agent.TempDirectory)/pwshSnap.tar.gz + + pool: + type: linux + isCustom: true + name: $(ubuntuPool) + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure + + steps: + - checkout: self + lfs: false + + - template: ./createOutputDirectory-linux.yml + + - powershell: | + sudo chown root:root / + displayName: 'Make sure root owns root' + condition: succeeded() + + - template: ./InstallSnapd.yml + + - powershell: 'dir env:' + displayName: 'Capture Environment' + + - powershell: | + $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel + Write-Verbose -Verbose -Message "using Channel $channel" + sudo snap install snapcraft --classic "--channel=$channel" + displayName: Install snapcraft + condition: succeeded() + retryCountOnTaskFailure: 2 + + - powershell: | + switch('$(channel)') { + 'stable' { + cd stable + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" + Write-Host $message + Write-Host "##$message" + } + 'lts' { + cd lts + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" + Write-Host $message + Write-Host "##$message" + } + 'preview' { + cd preview + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell-preview" + Write-Host $message + Write-Host "##$message" + } + default { + throw "Unknown channel: '$(channel)'" + } + } + + Write-Host "Building snap..." + # destructive-mode leaves files and packages on the machine + # This is okay because Azure DevOps Hosted machines are ephemeral. + Write-Verbose "snapcraft version: $(snapcraft --version)" -Verbose + snapcraft snap --debug --destructive-mode + Write-Host "finding snap..." + $snap = Get-ChildItem "*.snap" -Recurse -ErrorAction SilentlyContinue | Select-Object -ExpandProperty fullname -ErrorAction SilentlyContinue + if(!$snap) + { + throw "snap build failed" + } + else + { + $message = "vso[task.setvariable variable=PS_SNAP_PATH;]$snap" + Write-Host $message + Write-Host "##$message" + } + displayName: Build snap $(channel) + condition: and(succeeded(), ne(variables['$(channel)'],'no')) + + - powershell: | + Get-ChildItem env: + displayName: Capture env + condition: always() + + - powershell: | + Get-ChildItem $env:HOME/.local/state/snapcraft/log/*.log | foreach-object { + $name = $_.fullname + Write-Verbose -Verbose "uploading '$name'" + Write-Host "##vso[artifact.upload containerfolder=$(channel)-logs;artifactname=$(channel)-log]$name" + } + displayName: 'Upload $(channel) build log' + condition: always() + + - powershell: | + sudo snap install $(PS_SNAP_PATH) --classic --dangerous + displayName: 'Install $(channel) snap' + condition: succeeded() + + - powershell: | + &"$(PS_SNAP_EXEC)" -nologo -c '$psversiontable' + displayName: 'Test $(channel) snap' + condition: and(succeeded(), eq('${{ parameters.test }}','Yes')) + + # This is what we would need to do if this wasn't a custom job + - powershell: | + Copy-Item $(PS_SNAP_PATH) -Destination $(ob_outputDirectory)/ -verbose + $artifactName = "drop_build_$(channel)_build_$(channel)" + Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]$(ob_outputDirectory)" + displayName: Copy Artifact to OneBranch Job output directory and publish - $(channel) + condition: succeeded() + continueOnError: true + + - pwsh: | + $snap = '$(PS_SNAP_PATH)' + $mount = $env:SNAP_MOUNT + $null = new-item -Path $mount -ItemType Directory -Force + Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" + sudo mount -t squashfs -o ro $snap $mount + Write-Verbose -Verbose "get-childitem -recurse $mount" + get-childitem -recurse $mount + displayName: Unsquash Snap and capture contents + + # Copy to read write filesystem because scan tries to write back to this folder. + - pwsh: | + $mount = $env:SNAP_MOUNT + $readWriteCopy = $env:SNAP_COPY + Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" + $null = new-item -Path $readWriteCopy -ItemType Directory -Force + $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint + $exclude = @() + $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } + Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose + Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" + get-childitem -recurse $readWriteCopy + displayName: Copy to read/write fs + + - pwsh: | + Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose + tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} + copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose + displayName: tar the snap contents from the r/w copy + + # Uploads any packages as an artifact + - powershell: | + $artifactName = "drop_build_$(channel)_build_$(channel)_targz" + Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]${env:SNAP_TARGZ}" + displayName: Publish tar.gz Artifact - $(channel) + condition: succeeded() + continueOnError: true diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml new file mode 100644 index 0000000..d40e3d3 --- /dev/null +++ b/.pipelines/templates/scanJob.yml @@ -0,0 +1,39 @@ +parameters: + - name: channel + type: string + +jobs: + +- job: scan + displayName: Scan ${{ parameters.channel }} + + pool: + type: linux + + variables: + - name: channel + value: ${{ parameters.channel }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + - name: DOWNLOAD_DIRECTORY + value: '$(Build.ArtifactStagingDirectory)/down' + + steps: + - template: ./createOutputDirectory-linux.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download targz files' + inputs: + targetPath: $(DOWNLOAD_DIRECTORY) + artifact: drop_build_$(channel)_build_$(channel)_targz + + - pwsh: | + $targzs = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.tar.gz -recurse -File | Select-Object -ExpandProperty FullName + + foreach($targz in $targzs) + { + Write-Verbose -Verbose "Extracting $targz" + $target = '$(ob_outputDirectory)' + tar -xvf $targz -C $target + } + displayName: 'Extract Tar.Gz'