Add auto discovery of parameter files when scanning a directory

README.md

@@ -37,7 +37,7 @@ Argument | Description
 **(Optional)** `-v` or `--verbose` | Shows details about the analysis
 **(Optional)** `--include-non-security-rules` | Run all the rules against the templates, including non-security rules
 
- The Template BPA runs the [configured rules](#understanding-and-customizing-rules) against the provided template and its corresponding [template parameters](https://docs.microsoft.com/azure/azure-resource-manager/templates/parameter-files), if specified. If no template parameters are specified, then the Template BPA generates the minimum number of placeholder parameters to properly evaluate [template functions](https://docs.microsoft.com/azure/azure-resource-manager/templates/template-functions) in the template.
+ The Template BPA runs the [configured rules](#understanding-and-customizing-rules) against the provided template and its corresponding [template parameters](https://docs.microsoft.com/azure/azure-resource-manager/templates/parameter-files), if specified. If no template parameters are specified, then the Template BPA will check if templates with the [general naming standards defined by Microsoft](https://learn.microsoft.com/azure/azure-resource-manager/templates/parameter-files#file-name) are present in the same folder, otherwise it generates the minimum number of placeholder parameters to properly evaluate [template functions](https://docs.microsoft.com/azure/azure-resource-manager/templates/template-functions) in the template.
 
 **Note**: Providing the Template BPA with template parameter values will result in more accurate results as it will more accurately represent your deployments. The values provided to parameters may affect the evaluation of the Template BPA rule, altering its results. That said, **DO NOT** save sensitive data (passwords, connection strings, etc.) in parameter files in your repositories. Instead, [retrieve these values from your template from Azure Key Vault](https://docs.microsoft.com/azure/azure-resource-manager/templates/key-vault-parameter?tabs=azure-cli#reference-secrets-with-static-id).
 

src/Analyzer.Cli.FunctionalTests/CommandLineParserTests.cs

@@ -4,12 +4,12 @@
 using System;
 using System.Linq;
 using System.IO;
+using System.Text.RegularExpressions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Microsoft.Azure.Templates.Analyzer.Cli;
 using Microsoft.Azure.Templates.Analyzer.Types;
 using Newtonsoft.Json.Linq;
 
-
 namespace Analyzer.Cli.FunctionalTests
 {
     [TestClass]
@@ -99,6 +99,22 @@ public void AnalyzeTemplate_IncludesOrNotNonSecurityRules_ReturnsExpectedExitCod
             Assert.AreEqual((int)ExitCode.Violation, result.Result);
         }
 
+        [TestMethod]
+        public void AnalyzeTemplate_ValidInputValues_AnalyzesUsingAutoDetectedParameters()
+        {
+            var templatePath = GetFilePath(Path.Combine("ToTestSeparateParametersFile", "TemplateWithSeparateParametersFile.bicep"));
+
+            var args = new string[] { "analyze-template", templatePath };
+
+            using StringWriter outputWriter = new();
+            Console.SetOut(outputWriter);
+
+            var result = _commandLineParser.InvokeCommandLineAPIAsync(args);
+
+            Assert.AreEqual((int)ExitCode.Success, result.Result);
+            StringAssert.Contains(outputWriter.ToString(), "Parameters File: " + Path.Combine(Directory.GetCurrentDirectory(), "Tests", "ToTestSeparateParametersFile", "TemplateWithSeparateParametersFile.parameters.json"));
+        }
+
         [TestMethod]
         public void AnalyzeDirectory_ValidInputValues_AnalyzesExpectedNumberOfFiles()
         {
@@ -110,7 +126,26 @@ public void AnalyzeDirectory_ValidInputValues_AnalyzesExpectedNumberOfFiles()
             var result = _commandLineParser.InvokeCommandLineAPIAsync(args);
 
             Assert.AreEqual((int)ExitCode.ErrorAndViolation, result.Result);
-            StringAssert.Contains(outputWriter.ToString(), "Analyzed 10 files");
+            StringAssert.Contains(outputWriter.ToString(), "Analyzed 14 files");
+        }
+
+        [TestMethod]
+        public void AnalyzeDirectory_ValidInputValues_AnalyzesExpectedNumberOfFilesWithAutoDetectedParameters()
+        {
+            var args = new string[] { "analyze-directory", Path.Combine(Directory.GetCurrentDirectory(), "Tests", "ToTestSeparateParametersFile") };
+
+            using StringWriter outputWriter = new();
+            Console.SetOut(outputWriter);
+
+            var result = _commandLineParser.InvokeCommandLineAPIAsync(args);
+
+            Assert.AreEqual((int)ExitCode.Success, result.Result);
+
+            StringAssert.Contains(outputWriter.ToString(), "Analyzed 4 files");
+            StringAssert.Contains(outputWriter.ToString(), "Parameters File: " + Path.Combine(Directory.GetCurrentDirectory(), "Tests", "ToTestSeparateParametersFile", "TemplateWithSeparateParametersFile.parameters.json"));
+            StringAssert.Contains(outputWriter.ToString(), "Parameters File: " + Path.Combine(Directory.GetCurrentDirectory(), "Tests", "ToTestSeparateParametersFile", "TemplateWithSeparateParametersFile.parameters-dev.json"));
+            Assert.AreEqual(2, Regex.Matches(outputWriter.ToString(), "TemplateWithSeparateParametersFile.bicep").Count);
+            Assert.AreEqual(2, Regex.Matches(outputWriter.ToString(), "TemplateWithSeparateParametersFile.json").Count);
         }
 
         [DataTestMethod]
@@ -143,24 +178,14 @@ public void AnalyzeDirectory_DirectoryWithInvalidTemplates_LogsExpectedErrorInSa
             var sarifOutput = JObject.Parse(File.ReadAllText(outputFilePath));
             var toolNotifications = sarifOutput["runs"][0]["invocations"][0]["toolExecutionNotifications"];
 
-            var templateErrorMessage = "An exception occurred while analyzing a template";
-            Assert.AreEqual(templateErrorMessage, toolNotifications[0]["message"]["text"]);
-            Assert.AreEqual(templateErrorMessage, toolNotifications[1]["message"]["text"]);
+            Assert.AreEqual(toolNotifications[0]["message"]["text"].ToString(), $"An exception occurred while analyzing template {Path.Combine(directoryToAnalyze, "AnInvalidTemplate.json")}");
+            Assert.AreEqual(toolNotifications[1]["message"]["text"].ToString(), $"An exception occurred while analyzing template {Path.Combine(directoryToAnalyze, "AnInvalidTemplate.bicep")}");
 
-            var nonJsonFilePath1 = Path.Combine(directoryToAnalyze, "AnInvalidTemplate.json");
-            var nonJsonFilePath2 = Path.Combine(directoryToAnalyze, "AnInvalidTemplate.bicep");
-            var thirdNotificationMessageText = toolNotifications[2]["message"]["text"].ToString();
-            // Both orders have to be considered for Windows and Linux:
-            Assert.IsTrue($"Unable to analyze 2 files: {nonJsonFilePath1}, {nonJsonFilePath2}" == thirdNotificationMessageText ||
-                $"Unable to analyze 2 files: {nonJsonFilePath2}, {nonJsonFilePath1}" == thirdNotificationMessageText);
-
             Assert.AreEqual("error", toolNotifications[0]["level"]);
             Assert.AreEqual("error", toolNotifications[1]["level"]);
-            Assert.AreEqual("error", toolNotifications[2]["level"]);
 
             Assert.AreNotEqual(null, toolNotifications[0]["exception"]);
             Assert.AreNotEqual(null, toolNotifications[1]["exception"]);
-            Assert.AreEqual(null, toolNotifications[2]["exception"]);
         }
 
         [DataTestMethod]
@@ -179,12 +204,24 @@ public void AnalyzeDirectory_ExecutionWithErrorAndWarning_PrintsExpectedMessages
             }
 
             var warningMessage = "An exception occurred when processing the template language expressions";
-            var errorMessage = "An exception occurred while analyzing a template";
+            var errorMessage1 = $"An exception occurred while analyzing template {Path.Combine(directoryToAnalyze, "ReportsError.json")}";
+            var errorMessage2 = $"An exception occurred while analyzing template {Path.Combine(directoryToAnalyze, "ReportsError2.json")}";
 
-            expectedLogSummary += ($"{Environment.NewLine}{Environment.NewLine}\tSummary of the warnings:" +
-                $"{Environment.NewLine}\t\t1 instance of: {warningMessage}{Environment.NewLine}") +
-                $"{Environment.NewLine}\tSummary of the errors:" +
-                $"{Environment.NewLine}\t\t{(multipleErrors ? "2 instances" : "1 instance")} of: {errorMessage}";
+            if (!multipleErrors)
+            {
+                expectedLogSummary += ($"{Environment.NewLine}{Environment.NewLine}\tSummary of the warnings:" +
+                    $"{Environment.NewLine}\t\t1 instance of: {warningMessage}{Environment.NewLine}") +
+                    $"{Environment.NewLine}\tSummary of the errors:" +
+                    $"{Environment.NewLine}\t\t1 instance of: {errorMessage1}";
+            }
+            else
+            {
+                expectedLogSummary += ($"{Environment.NewLine}{Environment.NewLine}\tSummary of the warnings:" +
+                    $"{Environment.NewLine}\t\t1 instance of: {warningMessage}{Environment.NewLine}") +
+                    $"{Environment.NewLine}\tSummary of the errors:" +
+                    $"{Environment.NewLine}\t\t1 instance of: {errorMessage1}" +
+                    $"{Environment.NewLine}\t\t1 instance of: {errorMessage2}";
+            }
 
             expectedLogSummary += ($"{Environment.NewLine}{Environment.NewLine}\t1 Warning" +
                 $"{Environment.NewLine}\t{(multipleErrors ? "2 Errors" : "1 Error")}{Environment.NewLine}");
@@ -214,7 +251,7 @@ public void AnalyzeDirectory_ExecutionWithErrorAndWarning_PrintsExpectedMessages
                 var indexOfLogSummary = cliConsoleOutput.IndexOf("Execution summary:");
                 Assert.IsTrue(indexOfLogSummary >= 0, $"Expected log message not found in CLI output. Found:{Environment.NewLine}{cliConsoleOutput}");
 
-                var errorLog = $"Error: {errorMessage}";
+                var errorLog = $"Error: {errorMessage1}";
                 var warningLog = $"Warning: {warningMessage}";
                 if (usesVerboseMode)
                 {

src/Analyzer.Cli.FunctionalTests/Tests/ToTestSeparateParametersFile/TemplateWithSeparateParametersFile.bicep

@@ -0,0 +1,7 @@
+param infra object
+
+resource applicationInsight 'microsoft.insights/components@2020-02-02' = {
+  name: 'test'
+  location: infra.environment.location
+  kind: 'other'
+}

src/Analyzer.Cli.FunctionalTests/Tests/ToTestSeparateParametersFile/TemplateWithSeparateParametersFile.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "infra": {
+      "type": "object",
+      "metadata": {
+        "description": "Location for the resource."
+      }
+    }
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Insights/components",
+      "apiVersion": "2020-02-02",
+      "name": "test",
+      "location": "[parameters('infra').environment.location]",
+      "kind": "other"
+    }
+  ]
+}

src/Analyzer.Cli.FunctionalTests/Tests/ToTestSeparateParametersFile/TemplateWithSeparateParametersFile.parameters-dev.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "infra": {
+      "value": {
+        "environment": {
+          "location": "West Europe"
+        }
+      }
+    }
+  }
+}

src/Analyzer.Cli.FunctionalTests/Tests/ToTestSeparateParametersFile/TemplateWithSeparateParametersFile.parameters.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "infra": {
+      "value": {
+        "environment": {
+          "location": "West Europe"
+        }
+      }
+    }
+  }
+}

src/Analyzer.Cli/CommandLineParser.cs

@@ -197,10 +197,33 @@ private int AnalyzeTemplateCommandHandler(
                 return (int)ExitCode.ErrorInvalidARMTemplate;
             }
 
-            var analysisResult = AnalyzeTemplate(templateFilePath, parametersFilePath);
+            ExitCode exitCode;
+            if (parametersFilePath != null)
+            {
+                exitCode = AnalyzeTemplate(templateFilePath, parametersFilePath);
+            }
+            else
+            {
+                var parametersFiles = FindParameterFileForTemplate(templateFilePath);
+                var exitCodes = new List<ExitCode>();
+
+                if (parametersFiles.Count() > 0)
+                {
+                    foreach (FileInfo parametersFile in parametersFiles)
+                    {
+                        exitCodes.Add(AnalyzeTemplate(templateFilePath, parametersFile));
+                    }
+
+                    exitCode = AnalyzeExitCodes(exitCodes);
+                }
+                else
+                {
+                    exitCode = AnalyzeTemplate(templateFilePath, null);
+                }
+            }
 
             FinishAnalysis();
-            return (int)analysisResult;
+            return (int)exitCode;
         }
 
         // Note: argument names must match command arguments/options (without "-" characters)
@@ -230,36 +253,27 @@ private int AnalyzeDirectoryCommandHandler(
             // Log root directory info to be analyzed
             Console.WriteLine(Environment.NewLine + Environment.NewLine + $"Directory: {directoryPath}");
 
-            int numOfFilesAnalyzed = 0;
-            bool issueReported = false;
-            var filesFailed = new List<FileInfo>();
+            var exitCodes = new List<ExitCode>();
             foreach (FileInfo file in filesToAnalyze)
             {
-                ExitCode res = AnalyzeTemplate(file, null);
-
-                if (res == ExitCode.Success || res == ExitCode.Violation)
+                var parametersFiles = FindParameterFileForTemplate(file);
+                if (parametersFiles.Count() > 0)
                 {
-                    numOfFilesAnalyzed++;
-                    issueReported |= res == ExitCode.Violation;
+                    foreach (FileInfo parametersFile in parametersFiles)
+                    {
+                        exitCodes.Add(AnalyzeTemplate(file, parametersFile));
+                    }
                 }
-                else if (res == ExitCode.ErrorAnalysis || res == ExitCode.ErrorInvalidBicepTemplate)
+                else
                 {
-                    filesFailed.Add(file);
+                    exitCodes.Add(AnalyzeTemplate(file, null));
                 }
             }
 
+            int numOfFilesAnalyzed = exitCodes.Where(x => x == ExitCode.Success || x == ExitCode.Violation).Count();
             Console.WriteLine(Environment.NewLine + $"Analyzed {numOfFilesAnalyzed} {(numOfFilesAnalyzed == 1 ? "file" : "files")}.");
 
-            ExitCode exitCode;
-            if (filesFailed.Count > 0)
-            {
-                logger.LogError($"Unable to analyze {filesFailed.Count} {(filesFailed.Count == 1 ? "file" : "files")}: {string.Join(", ", filesFailed)}");
-                exitCode = issueReported ? ExitCode.ErrorAndViolation : ExitCode.ErrorAnalysis;
-            }
-            else
-            {
-                exitCode = issueReported ? ExitCode.Violation : ExitCode.Success;
-            }
+            var exitCode = AnalyzeExitCodes(exitCodes);
 
             FinishAnalysis();
 
@@ -281,7 +295,14 @@ private ExitCode AnalyzeTemplate(FileInfo templateFilePath, FileInfo parametersF
             }
             catch (Exception exception)
             {
-                logger.LogError(exception, "An exception occurred while analyzing a template");
+                if (parametersFilePath != null)
+                {
+                    logger.LogError(exception, $"An exception occurred while analyzing template {templateFilePath.FullName} with parameters file {parametersFilePath.FullName}");
+                }                
+                else
+                {
+                    logger.LogError(exception, $"An exception occurred while analyzing template {templateFilePath.FullName}");
+                }
 
                 return (exception.Message == TemplateAnalyzer.BicepCompileErrorMessage)
                     ? ExitCode.ErrorInvalidBicepTemplate
@@ -339,7 +360,7 @@ private IEnumerable<FileInfo> FindTemplateFilesInDirectory(DirectoryInfo directo
                     MatchCasing = MatchCasing.CaseInsensitive,
                     RecurseSubdirectories = true
                 }
-            ).Where(IsValidTemplate);
+            ).Where(s => !s.Name.Contains(".parameters")).Where(IsValidTemplate);
 
             var bicepTemplates = directoryPath.GetFiles(
                 "*.bicep",
@@ -352,6 +373,20 @@ private IEnumerable<FileInfo> FindTemplateFilesInDirectory(DirectoryInfo directo
             return armTemplates.Concat(bicepTemplates);
         }
 
+        // Check if parameters*.json files are present according to naming standards here https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/parameter-files#file-name, and if so use it as the parametersFile input
+        private IEnumerable<FileInfo> FindParameterFileForTemplate(FileInfo template)
+        {
+            var parametersFiles = template.Directory.GetFiles(
+                Path.GetFileNameWithoutExtension(template.Name) + ".parameters*.json",
+                new EnumerationOptions
+                {
+                    MatchCasing = MatchCasing.CaseInsensitive,
+                    RecurseSubdirectories = false
+                });
+
+            return parametersFiles;
+        }
+
         private bool IsValidTemplate(FileInfo file)
         {
             // assume bicep files are valid, they are compiled/verified later
@@ -489,5 +524,23 @@ private bool TryReadConfigurationFile(FileInfo configurationFile, out Configurat
                 return false;
             }
         }
+
+        private ExitCode AnalyzeExitCodes(List<ExitCode> exitCodes)
+        {
+            ExitCode exitCode;
+            bool issueReported = exitCodes.Where(x => x == ExitCode.Violation).Count() > 0;
+            bool filesFailed = exitCodes.Where(x => x == ExitCode.ErrorAnalysis || x == ExitCode.ErrorInvalidBicepTemplate).Count() > 0;
+
+            if (filesFailed)
+            {
+                exitCode = issueReported ? ExitCode.ErrorAndViolation : ExitCode.ErrorAnalysis;
+            }
+            else
+            {
+                exitCode = issueReported ? ExitCode.Violation : ExitCode.Success;
+            }
+
+            return exitCode;
+        }
     }
 }