Batch generation

.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

@@ -0,0 +1,82 @@
+param (
+    [string]$MatrixKey,
+    [string]$RepoRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+
+$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
+$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
+$moduleGroup = $generateTargets.$MatrixKey
+$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
+
+$AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
+New-Item -ItemType Directory -Force -Path $AutorestOutputDir
+
+$sourceDirectory = Join-Path $RepoRoot "src"
+$generatedDirectory = Join-Path $RepoRoot "generated"
+$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
+Import-Module $buildScriptsModulePath -Force
+
+$results = @()
+
+foreach ($moduleName in $sortedModuleNames) {
+    Write-Host "=============================================================="
+    Write-Host "Regenerating Module: $moduleName"
+    $moduleStartTime = Get-Date
+    $moduleResult = @{
+        Module = $moduleName
+        DurationSeconds = 0
+        SubModules = @()
+    }
+
+    $subModuleNames = $moduleGroup.$moduleName
+    foreach ($subModuleName in $subModuleNames) {
+        Write-Host "Regenerating SubModule: $subModuleName"
+        $subModuleStartTime = Get-Date
+        $subModuleResult = @{
+            SubModule = $subModuleName
+            Status = "Success"
+            DurationSeconds = 0
+            Error = ""
+        }
+
+        try {
+            $generateLog = Join-Path $AutorestOutputDir $moduleName "$subModuleName.log"
+            if (Test-Path $generateLog) {
+                Remove-Item -Path $generateLog -Recurse -Force
+            }
+            New-Item -ItemType File -Force -Path $generateLog
+
+            if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
+                Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+                Write-Warning "log can be found at $generateLog"
+                $subModuleResult.Status = "Failed"
+                $subModuleResult.Error = "Update-GeneratedSubModule function returned false."
+            }
+
+        } catch {
+            Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+            $subModuleResult.Status = "Failed"
+            $subModuleResult.Error = $_.Exception.Message
+        } finally {
+            $subModuleEndTime = Get-Date
+            $subModuleResult.DurationSeconds = ($subModuleEndTime - $subModuleStartTime).TotalSeconds
+            $moduleResult.SubModules += $subModuleResult
+        }
+    }
+    $moduleEndTime = Get-Date
+    $moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
+    $results += $moduleResult
+}
+
+$ArtifactOutputDir = Join-Path $RepoRoot "artifacts"
+
+git add .
+$patchPath = Join-Path $ArtifactOutputDir "changed-$MatrixKey.patch"
+git diff --cached > $patchPath
+
+$reportPath = Join-Path $ArtifactOutputDir "GenerationReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
+
+Write-Host "Build report written to $reportPath"

.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1

@@ -0,0 +1,40 @@
+param (
+    [string]$MatrixKey,
+    [string]$RepoRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+
+$buildTargetsOutputFile = Join-Path $RepoRoot "artifacts" "buildTargets.json"
+$buildTargets = Get-Content -Path $buildTargetsOutputFile -Raw | ConvertFrom-Json
+$moduleGroup = $buildTargets.$MatrixKey
+$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
+
+$results = @()  
+foreach ($moduleName in $moduleGroup) {
+    Write-Host "=============================================================="
+    Write-Host "Building Module: $moduleName"
+
+    $startTime = Get-Date
+    $result = @{
+        Module = $moduleName
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+    }
+
+    try {
+        & $buildModulesPath -TargetModule $moduleName -InvokedByPipeline
+    } catch {
+        Write-Warning "Failed to build module: $moduleName"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    } finally {
+        $endTine = Get-Date
+        $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+$reportPath = Join-Path $RepoRoot "artifacts" "BuildReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8

.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1

@@ -0,0 +1,22 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$Owner,
+    [string]$Repo,
+    [string]$BaseBranch,
+    [string]$NewBranch,
+    [string]$Token
+)
+
+$headers = @{ Authorization = "Bearer $Token"; "User-Agent" = "ADO-Pipeline" }
+$branchInfo = Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/ref/heads/$BaseBranch" -Headers $headers
+$sha = $branchInfo.object.sha
+
+$body = @{
+    ref = "refs/heads/$NewBranch"
+    sha = $sha
+} | ConvertTo-Json
+
+Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
+                -Method Post -Headers $headers -Body $body -ContentType "application/json"
+
+Write-Host "Created branch '$NewBranch' from '$BBaseBranch'"

.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

@@ -0,0 +1,57 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [int]$MaxParallelBuildJobs = 3,
+    [int]$MaxParallelAnalyzeJobs = 3,
+    [int]$MaxParallelTestWindowsJobs = 3,
+    [int]$MaxParallelTestLinuxJobs = 3,
+    [int]$MaxParallelTestMacJobs = 3,
+    [string[]]$ChangedFiles,
+    [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+
+$changedModulesDict = @{}
+$changedSubModulesDict = @{}
+for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+    if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+        $parent = $Matches[2]
+        $child = $Matches[3]
+        $key = "$parent/$child"
+
+        $changedModulesDict[$parent] = $true
+        $changedSubModulesDict[$key] = $true
+    }
+}
+$changedModules = $changedModulesDict.Keys | Sort-Object
+$changedSubModules = $changedSubModulesDict.Keys | Sort-Object
+
+Write-Host "##[group]Changed modules: $($changedModules.Count)"
+foreach ($module in $changedModules) {
+    Write-Host $module
+}
+Write-Host "##[endgroup]"
+    Write-Host
+
+Write-Host "##[group]Changed sub modules: $($changedSubModules.Count)"
+foreach ($subModule in $changedSubModules) {
+    Write-Host $subModule
+}
+Write-Host "##[endgroup]"
+Write-Host
+
+$groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelBuildJobs
+Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
+
+# $groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelAnalyzeJobs
+# Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
+
+# $groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestWindowsJobs
+# Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
+
+# $groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestLinuxJobs
+# Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
+
+# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestMacJobs
+# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules

.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

@@ -0,0 +1,116 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [int]$MaxParallelJobs = 3
+)
+
+$srcPath = Join-Path $RepoRoot 'src'
+function Get-SubModuleWithAutorestV4 {
+    param (
+        [string]$srcPath
+    )
+
+    $result = @{}
+
+    Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
+        $module = $_
+
+        Get-ChildItem -Path $module.FullName -Directory | Where-Object { 
+            $_.Name -like '*.autorest'
+        } | ForEach-Object {
+            $subModule = $_
+
+            $readmePath = Join-Path $subModule.FullName 'README.md'
+
+            if (Test-Path $readmePath) {
+                $readmeContent = Get-Content -Path $readmePath -Raw
+
+                if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
+                    if ($result.ContainsKey($module.Name)) {
+                        $result[$module.Name] += $subModule.Name
+                    } else {
+                        $result[$module.Name] = @($subModule.Name) 
+                    }
+                }
+            }
+        }
+    }
+
+    return $result
+}
+# TODO(Bernard): Use real function after test
+# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
+$modules = @{
+    "DeviceRegistry" = @("DeviceRegistry.Autorest")
+    "ArcGateway" = @("ArcGateway.Autorest")
+    "Chaos" = @("Chaos.Autorest")
+    "Cdn" = @("Cdn.Autorest")
+    "Communication" = @("EmailService.Autorest", "EmailServicedata.Autorest")
+    "Astro" = @("Astro.Autorest")
+    "ImageBuilder" = @("ImageBuilder.Autorest")
+}
+$modules = $modules.GetEnumerator() | ForEach-Object {
+    [PSCustomObject]@{
+        ModuleName = $_.Key
+        SubModules = ($_.Value | Sort-Object)
+    }
+} | Sort-Object -Property ModuleName
+
+Write-Host "Total matched modules: $($modules.Count)"
+
+function Group-Modules {
+    param (
+        [array]$modules,
+        [int]$maxParallelJobs
+    )
+
+    $count = $modules.Count
+    $n = [Math]::Min($count, $maxParallelJobs)
+    if ($n -eq 0) {
+        return @()
+    }
+
+    $result = @()
+    $sizePerGroup = [Math]::Ceiling($count / $n)
+
+    for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
+        $group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+        $result += ,$group
+    }
+
+    return $result
+}
+
+$groupedModules = Group-Modules -modules $modules -maxParallelJobs $MaxParallelJobs
+Write-Host "Total module groups: $($groupedModules.Count)"
+
+$index = 0
+$generateTargets = @{}
+foreach ($moduleGroup in $groupedModules) {
+    Write-Host "##[group]Prepareing module group $($index + 1)"
+    $mergedModules = @{}
+    foreach ($moduleObj in $moduleGroup) {
+        Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
+        $mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
+        $subIndex++
+    }
+
+    $key = ($index + 1).ToString() + "-" + $moduleGroup.Count
+    $generateTargets[$key] = $mergedModules
+    $MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
+    Write-Host "##[endgroup]"
+    Write-Host
+    $index++
+}
+
+$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
+if (-not (Test-Path -Path $generateTargetsOutputDir)) {
+    New-Item -ItemType Directory -Path $generateTargetsOutputDir
+}
+$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
+$generateTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
+
+if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+    $MatrixStr = $MatrixStr.Substring(1)
+}
+Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"

.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1

@@ -0,0 +1,45 @@
+param (
+    [string]$MatrixKey,
+    [string]$Target,
+    [string]$TestEnvName,
+    [string]$RepoRoot,
+    [string]$ArtifactRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+Write-Host "Test $($TestEnvName): $Target"
+
+$modules = $Target -split ','
+$results = @()  
+
+foreach ($module in $modules) {
+    $startTime = Get-Date
+    $result = @{
+        Module = $module
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+    }
+
+    try {
+        Write-Host "Testing module: $module"
+        $subModulePath = Join-Path $RepoRoot 'src' $module
+        # TODO(Bernard) Remove log after test
+        Write-Host "Sub module path: $subModulePath"
+        Set-Location -Path $subModulePath
+        & ".\test-module.ps1"  
+    }
+    catch {
+        Write-Warning "Failed to test module: $module"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    }
+    finally {
+        $endTime = Get-Date
+        $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+$reportPath = Join-Path $ArtifactRoot "Test$($TestEnvName)Report-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8