Skip to content

Commit abb0093

Browse files
Pan-QiPan-Qi
committed
batch generate
1 parent 7a89243 commit abb0093

File tree

6 files changed

+423
-143
lines changed

6 files changed

+423
-143
lines changed

‎.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
param (
2+
[string]$MatrixKey,
3+
[string]$RepoRoot
4+
)
5+
6+
Write-Host "Matrix Key: $MatrixKey"
7+
8+
$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
9+
$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
10+
$moduleGroup = $generateTargets.$MatrixKey
11+
$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
12+
13+
$AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
14+
New-Item -ItemType Directory -Force -Path $AutorestOutputDir
15+
16+
$sourceDirectory = Join-Path $RepoRoot "src"
17+
$generatedDirectory = Join-Path $RepoRoot "generated"
18+
$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
19+
Import-Module $buildScriptsModulePath
20+
21+
$results = @()
22+
23+
foreach ($moduleName in $sortedModuleNames) {
24+
Write-Host "=============================================================="
25+
Write-Host "Regenerating Module: $moduleName"
26+
$moduleStartTime = Get-Date
27+
$moduleResult = @{
28+
Module = $moduleName
29+
DurationSeconds = 0
30+
SubModules = @()
31+
}
32+
33+
$subModuleNames = $moduleGroup.$moduleName
34+
foreach ($subModuleName in $subModuleNames) {
35+
Write-Host "Regenerating SubModule: $subModuleName"
36+
$subModuleStartTime = Get-Date
37+
$subModuleResult = @{
38+
SubModule = $subModuleName
39+
Status = "Success"
40+
DurationSeconds = 0
41+
Error = ""
42+
}
43+
44+
try {
45+
46+
throw "Module '$moduleName' is not allowed for this operation."
47+
$moduleRootSource = Join-Path $sourceDirectory $moduleName
48+
$moduleRootGenerated = Join-Path $generatedDirectory $moduleName
49+
50+
$generateLog = Join-Path $AutorestOutputDir $moduleName "$subModuleName.log"
51+
if (Test-Path $generateLog) {
52+
Remove-Item -Path $generateLog -Recurse -Force
53+
}
54+
New-Item -ItemType File -Force -Path $generateLog
55+
56+
if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
57+
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
58+
Write-Warning "log can be found at $generateLog"
59+
$subModuleResult.Status = "Failed"
60+
$subModuleResult.Error = "Update-GeneratedSubModule function returned false."
61+
}
62+
63+
} catch {
64+
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
65+
$subModuleResult.Status = "Failed"
66+
$subModuleResult.Error = $_.Exception.Message
67+
} finally {
68+
$subModuleEndTime = Get-Date
69+
$subModuleResult.DurationSeconds = ($subModuleEndTime - $subModuleStartTime).TotalSeconds
70+
$moduleResult.SubModules += $subModuleResult
71+
}
72+
}
73+
$moduleEndTime = Get-Date
74+
$moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
75+
$results += $moduleResult
76+
}
77+
78+
$ArtifactOutputDir = Join-Path $RepoRoot "artifacts"
79+
80+
git add .
81+
$patchPath = Join-Path $ArtifactOutputDir "changed-$MatrixKey.patch"
82+
git diff --cached > $patchPath
83+
84+
$reportPath = Join-Path $ArtifactOutputDir "GenerationReport-$MatrixKey.json"
85+
$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
86+
87+
Write-Host "Build report written to $reportPath"

‎.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

+52-28
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
[CmdletBinding(DefaultParameterSetName="AllSet")]
22
param (
3-
[int]$MaxParallelJobs = 3,
3+
[int]$MaxParalleAnalyzeJobs = 3,
4+
[int]$MaxParalleTestWindowsJobs = 3,
5+
[int]$MaxParalleTestLinuxJobs = 3,
6+
[int]$MaxParalleTestMacJobs = 3,
47
[string[]]$ChangedFiles
58
)
69

710
$autorestFolders = @{}
8-
9-
foreach ($file in $changedFiles) {
10-
if ($file -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
11+
for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
12+
if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
1113
$parent = $Matches[2]
1214
$child = $Matches[3]
1315
$key = "$parent/$child"
@@ -16,8 +18,13 @@ foreach ($file in $changedFiles) {
1618
}
1719
}
1820

19-
$subModules = $autorestFolders.Keys
20-
Write-Host "Outer Group ${subModules}:"
21+
$changedSubModules = $autorestFolders.Keys
22+
# TODO(Bernard) Remove test data after test
23+
# $changedSubModules = @("A", "B", "C", "D", "E", "F", "G")
24+
Write-Host "Chagned sub modules: "
25+
foreach ($subModule in $changedSubModules) {
26+
Write-Host $subModule
27+
}
2128

2229
function Split-List {
2330
param (
@@ -27,39 +34,56 @@ function Split-List {
2734

2835
$count = $subModules.Count
2936
$n = [Math]::Min($count, $maxParallelJobs)
37+
3038
if ($n -eq 0) {
3139
return @()
3240
}
3341

3442
$result = @()
35-
$sizePerGroup = [Math]::Ceiling($count / $n)
3643

37-
for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
38-
$group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
39-
$result += ,$group
44+
for ($i = 0; $i -lt $n; $i++) {
45+
$result += ,@()
46+
}
47+
48+
for ($i = 0; $i -lt $count; $i++) {
49+
$groupIndex = $i % $n
50+
$result[$groupIndex] += $subModules[$i]
4051
}
4152

42-
return $result
53+
return ,$result
4354
}
4455

45-
$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
56+
function Write-Matrix {
57+
param (
58+
[string]$variableName,
59+
[array]$groupedSubModules
60+
)
4661

47-
$index = 0
48-
foreach ($subModules in $devidedSubModules) {
49-
Write-Host "Outer Group ${index}:"
50-
$subIndex = 0
51-
foreach ($subModule in $subModules) {
52-
Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
53-
$subIndex++
62+
$index = 0
63+
foreach ($subModules in $groupedSubModules) {
64+
$moduleNamesStr = $subModules -join ','
65+
$key = ($index + 1).ToString() + "-" + $subModules.Count
66+
$MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
67+
$index++
5468
}
5569

56-
$moduleNamesStr = $subModules -join ','
57-
$key = ($index + 1).ToString() + "-" + $subModules.Count
58-
$MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
59-
$index++
60-
}
70+
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
71+
$MatrixStr = $MatrixStr.Substring(1)
72+
}
73+
Write-Host "##vso[task.setVariable variable=$variableName;isOutput=true]{$MatrixStr}"
74+
Write-Host "variable=$variableName; value=$MatrixStr"
75+
}
6176

62-
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
63-
$MatrixStr = $MatrixStr.Substring(1)
64-
}
65-
Write-Host "##vso[task.setVariable variable=analyzeTargets;isOutput=true]{$MatrixStr}"
77+
$groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleAnalyzeJobs
78+
Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
79+
80+
$groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestWindowsJobs
81+
Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
82+
83+
$groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestLinuxJobs
84+
Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
85+
86+
# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestMacJobs
87+
# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules
88+
89+
Write-Host "##vso[task.setVariable variable=TestMacTargets;isOutput=true]{}"

‎.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

+76-43
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,67 @@ param (
44
[int]$MaxParallelJobs = 3
55
)
66

7-
$moduleRoot = Join-Path $RepoRoot 'src'
8-
$subModules = @()
9-
10-
Get-ChildItem -Path $moduleRoot -Directory | ForEach-Object {
11-
$module = $_
12-
Get-ChildItem -Path $module.FullName -Directory | Where-Object {
13-
$_.Name -like '*.autorest'
14-
} | ForEach-Object {
15-
$sub_module = $_
16-
$subModules += ,@($module.Name, $sub_module.Name)
17-
}
18-
}
7+
$srcPath = Join-Path $RepoRoot 'src'
8+
function Get-SubModuleWithAutorestV4 {
9+
param (
10+
[string]$srcPath
11+
)
12+
13+
$result = @{}
1914

20-
$subModules = @(
21-
# V3
22-
@("Cdn","Cdn.Autorest"),
23-
@("ImageBuilder", "ImageBuilder.Autorest"),
15+
Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
16+
$module = $_
2417

25-
# V4
26-
@("Chaos", "Chaos.Autorest"),
27-
@("DeviceRegistry", "DeviceRegistry.Autorest"),
28-
@("Astro", "Astro.Autorest"),
29-
30-
# V4 Multi sub-modules
31-
@("Communication","EmailService.Autorest"),
32-
@("Communication", "EmailServicedata.Autorest")
33-
)
18+
Get-ChildItem -Path $module.FullName -Directory | Where-Object {
19+
$_.Name -like '*.autorest'
20+
} | ForEach-Object {
21+
$subModule = $_
22+
23+
$readmePath = Join-Path $subModule.FullName 'README.md'
24+
25+
if (Test-Path $readmePath) {
26+
$readmeContent = Get-Content -Path $readmePath -Raw
27+
28+
if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
29+
if ($result.ContainsKey($module.Name)) {
30+
$result[$module.Name] += $subModule.Name
31+
} else {
32+
$result[$module.Name] = @($subModule.Name)
33+
}
34+
}
35+
}
36+
}
37+
}
3438

35-
Write-Host "Total matched sub modules: $($subModules.Count)"
39+
return $result
40+
}
41+
# TODO(Bernard): Use real function after test
42+
# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
43+
$modules = @{
44+
"DeviceRegistry" = @("DeviceRegistry.Autorest")
45+
"ArcGateway" = @("ArcGateway.Autorest")
46+
"Chaos" = @("Chaos.Autorest")
47+
"Cdn" = @("Cdn.Autorest")
48+
"Communication" = @("EmailService.Autorest", "EmailServicedata.Autorest")
49+
"Astro" = @("Astro.Autorest")
50+
"ImageBuilder" = @("ImageBuilder.Autorest")
51+
}
52+
$modules = $modules.GetEnumerator() | ForEach-Object {
53+
[PSCustomObject]@{
54+
ModuleName = $_.Key
55+
SubModules = ($_.Value | Sort-Object)
56+
}
57+
} | Sort-Object -Property ModuleName
3658

37-
function Split-List {
59+
Write-Host "Total matched modules: $($modules.Count)"
60+
61+
function Group-List {
3862
param (
39-
[array]$subModules,
63+
[array]$modules,
4064
[int]$maxParallelJobs
4165
)
4266

43-
$count = $subModules.Count
67+
$count = $modules.Count
4468
$n = [Math]::Min($count, $maxParallelJobs)
4569
if ($n -eq 0) {
4670
return @()
@@ -50,34 +74,43 @@ function Split-List {
5074
$sizePerGroup = [Math]::Ceiling($count / $n)
5175

5276
for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
53-
$group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
77+
$group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
5478
$result += ,$group
5579
}
5680

5781
return $result
5882
}
5983

60-
$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
61-
62-
Write-Host "Total matched devides: $($devidedSubModules.Count)"
84+
$groupedModules = Group-List -modules $modules -maxParallelJobs $MaxParallelJobs
85+
Write-Host "Total module groups: $($groupedModules.Count)"
6386

6487
$index = 0
65-
foreach ($subModules in $devidedSubModules) {
66-
Write-Host "Outer Group ${index}:"
67-
$subIndex = 0
68-
foreach ($subModule in $subModules) {
69-
Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
88+
$generateTargets = @{}
89+
foreach ($moduleGroup in $groupedModules) {
90+
Write-Host "##[group]Prepareing module group $($index + 1)"
91+
$mergedModules = @{}
92+
foreach ($moduleObj in $moduleGroup) {
93+
Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
94+
$mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
7095
$subIndex++
7196
}
7297

73-
$moduleNames = $subModules | ForEach-Object { $_[0] }
74-
$moduleNamesStr = $moduleNames -join ','
75-
$key = ($index + 1).ToString() + "-" + $subModules.Count
76-
$MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
98+
$key = ($index + 1).ToString() + "-" + $moduleGroup.Count
99+
$generateTargets[$key] = $mergedModules
100+
$MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
101+
Write-Host "##[endgroup]"
102+
Write-Host
77103
$index++
78104
}
79105

106+
$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
107+
if (-not (Test-Path -Path $generateTargetsOutputDir)) {
108+
New-Item -ItemType Directory -Path $generateTargetsOutputDir
109+
}
110+
$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
111+
$generateTargets | ConvertTo-Json -Depth 10 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
112+
80113
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
81114
$MatrixStr = $MatrixStr.Substring(1)
82115
}
83-
Write-Host "##vso[task.setVariable variable=buildTargets;isOutput=true]{$MatrixStr}"
116+
Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"

0 commit comments

Comments
 (0)