Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validate deployment files #3884

Draft
wants to merge 9 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions src/cfnlint/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from cfnlint.decode.decode import decode_str
from cfnlint.helpers import REGION_PRIMARY, REGIONS
from cfnlint.rules import Match, RulesCollection
from cfnlint.runner import Runner, TemplateRunner
from cfnlint.runner import Runner, run_template_by_data

Matches = List[Match]

Expand Down Expand Up @@ -56,11 +56,16 @@ def lint(
config_mixin = ConfigMixIn(**config)

if isinstance(rules, RulesCollection):
template_runner = TemplateRunner(None, template, config_mixin, rules) # type: ignore # noqa: E501
return list(template_runner.run())
return list(
run_template_by_data(
template,
config_mixin,
rules, # type: ignore
)
)

runner = Runner(config_mixin)
return list(runner.validate_template(None, template))
return list(runner.validate_template(template))


def lint_all(s: str) -> list[Match]:
Expand Down
188 changes: 140 additions & 48 deletions src/cfnlint/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
import logging
import os
import sys
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict, Sequence, TypedDict

from typing_extensions import Unpack

import cfnlint.decode.cfn_yaml
from cfnlint.context.parameters import ParameterSet
from cfnlint.helpers import REGIONS, format_json_string
from cfnlint.jsonschema import StandardValidator
from cfnlint.version import __version__
Expand Down Expand Up @@ -316,6 +318,64 @@
parser.exit(1)


class ExtendKeyValuePairs(argparse.Action):
def __init__(
self,
option_strings,
dest,
nargs=None,
const=None,
default=None,
type=None,
choices=None,
required=False,
help=None,
metavar=None,
): # pylint: disable=W0622
super().__init__(
option_strings=option_strings,
dest=dest,
nargs=nargs,
const=const,
default=default,
type=type,
choices=choices,
required=required,
help=help,
metavar=metavar,
)

def __call__(self, parser, namespace, values, option_string=None):
try:
items = {}
for value in values:
# split it into key and value
key, value = value.split("=", 1)
items[key.strip()] = value.strip()

result = getattr(namespace, self.dest) + [items]
setattr(namespace, self.dest, result)
except Exception: # pylint: disable=W0703
parser.print_help()
parser.exit(1)


class ExtendAction(argparse.Action):
"""Support argument types that are lists and can
be specified multiple times.
"""

def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest)
items = [] if items is None else items
for value in values:
if isinstance(value, list):
items.extend(value)
else:
items.append(value)
setattr(namespace, self.dest, items)


class CliArgs:
"""Base Args class"""

Expand All @@ -333,21 +393,6 @@
self.print_help(sys.stderr)
self.exit(1, f"{self.prog}: error: {message}\n")

class ExtendAction(argparse.Action):
"""Support argument types that are lists and can
be specified multiple times.
"""

def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest)
items = [] if items is None else items
for value in values:
if isinstance(value, list):
items.extend(value)
else:
items.append(value)
setattr(namespace, self.dest, items)

usage = (
"\nBasic: cfn-lint test.yaml\n"
"Ignore a rule: cfn-lint -i E3012 -- test.yaml\n"
Expand All @@ -357,18 +402,23 @@

parser = ArgumentParser(description="CloudFormation Linter", usage=usage)
parser.register("action", "extend", ExtendAction)
parser.register("action", "rule_configuration", RuleConfigurationAction)
parser.register("action", "extend_key_value", ExtendKeyValuePairs)

standard = parser.add_argument_group("Standard")
advanced = parser.add_argument_group("Advanced / Debugging")

validation_group = standard.add_mutually_exclusive_group()
parameter_group = standard.add_mutually_exclusive_group()

# Allow the template to be passes as an optional or a positional argument
standard.add_argument(
"templates",
metavar="TEMPLATE",
nargs="*",
help="The CloudFormation template to be linted",
)
standard.add_argument(
validation_group.add_argument(
"-t",
"--template",
metavar="TEMPLATE",
Expand All @@ -392,6 +442,22 @@
default=[],
action="extend",
)
validation_group.add_argument(
"--deployment-files",
dest="deployment_files",
help="Deployment files",
nargs="+",
default=[],
action="extend",
)
parameter_group.add_argument(
"--parameters",
dest="parameters",
nargs="+",
default=[],
action="extend_key_value",
help="only check rules whose id do not match these values",
)
advanced.add_argument(
"-D", "--debug", help="Enable debug logging", action="store_true"
)
Expand Down Expand Up @@ -480,7 +546,7 @@
dest="configure_rules",
nargs="+",
default={},
action=RuleConfigurationAction,
action="rule_configuration",
help=(
"Provide configuration for a rule. Format RuleId:key=value. Example:"
" E3012:strict=true"
Expand Down Expand Up @@ -585,15 +651,15 @@

if isinstance(configs, dict):
for key, value in {
"ignore_checks": (list),
"regions": (list),
"append_rules": (list),
"override_spec": (str),
"configure_rules": (dict),
"custom_rules": (str),
"ignore_bad_template": (bool),
"ignore_checks": (list),
"include_checks": (list),
"configure_rules": (dict),
"include_experimental": (bool),
"override_spec": (str),
"regions": (list),
}.items():
if key in configs:
if isinstance(configs[key], value):
Expand All @@ -606,24 +672,26 @@

class ManualArgs(TypedDict, total=False):
configure_rules: dict[str, dict[str, Any]]
include_checks: list[str]
ignore_checks: list[str]
mandatory_checks: list[str]
include_experimental: bool
deployment_files: list[str]
ignore_bad_template: bool
ignore_checks: list[str]
ignore_templates: list
include_checks: list[str]
include_experimental: bool
mandatory_checks: list[str]
merge_configs: bool
non_zero_exit_code: str
output_file: str
regions: list
parameters: list[ParameterSet]
templates: list[str]


# pylint: disable=too-many-public-methods
class ConfigMixIn(TemplateArgs, CliArgs, ConfigFileArgs):

def __init__(self, cli_args: list[str] | None = None, **kwargs: Unpack[ManualArgs]):
self._manual_args = kwargs or ManualArgs()
self._templates_to_process = False
CliArgs.__init__(self, cli_args)
# configure debug as soon as we can
TemplateArgs.__init__(self, {})
Expand All @@ -634,26 +702,33 @@
def __repr__(self):
return format_json_string(
{
"append_rules": self.append_rules,
"config_file": self.config_file,
"configure_rules": self.configure_rules,
"custom_rules": self.custom_rules,
"debug": self.debug,
"deployment_files": self.deployment_files,
"format": self.format,
"ignore_bad_template": self.ignore_bad_template,
"ignore_checks": self.ignore_checks,
"include_checks": self.include_checks,
"mandatory_checks": self.mandatory_checks,
"include_experimental": self.include_experimental,
"configure_rules": self.configure_rules,
"regions": self.regions,
"ignore_bad_template": self.ignore_bad_template,
"debug": self.debug,
"info": self.info,
"format": self.format,
"templates": self.templates,
"append_rules": self.append_rules,
"override_spec": self.override_spec,
"custom_rules": self.custom_rules,
"config_file": self.config_file,
"mandatory_checks": self.mandatory_checks,
"merge_configs": self.merge_configs,
"non_zero_exit_code": self.non_zero_exit_code,
"override_spec": self.override_spec,
"parameters": self.parameters,
"regions": self.regions,
"templates": self.templates,
}
)

def __eq__(self, value):
if not isinstance(value, ConfigMixIn):
return False

Check warning on line 729 in src/cfnlint/config.py

View check run for this annotation

Codecov / codecov/patch

src/cfnlint/config.py#L729

Added line #L729 was not covered by tests
return str(self) == str(value)

def _get_argument_value(self, arg_name, is_template, is_config_file):
cli_value = getattr(self.cli_args, arg_name)
template_value = self.template_args.get(arg_name)
Expand Down Expand Up @@ -748,7 +823,9 @@
file_args = self._get_argument_value("templates", False, True)
cli_args = self._get_argument_value("templates", False, False)

if cli_alt_args:
if "templates" in self._manual_args:
filenames = self._manual_args["templates"]
elif cli_alt_args:
filenames = cli_alt_args
elif file_args:
filenames = file_args
Expand All @@ -758,10 +835,6 @@
# No filenames found, could be piped in or be using the api.
return None

# If we're still haven't returned, we've got templates to lint.
# Build up list of templates to lint.
self.templates_to_process = True

if isinstance(filenames, str):
filenames = [filenames]

Expand Down Expand Up @@ -811,6 +884,23 @@
"append_rules", False, True
)

@property
def parameters(self) -> list[ParameterSet]:
parameter_sets = self._get_argument_value("parameters", True, True)
results: list[ParameterSet] = []
for parameter_set in parameter_sets:
if isinstance(parameter_set, ParameterSet):
results.append(parameter_set)
else:
results.append(
ParameterSet(
source=None,
parameters=parameter_set,
)
)

return results

@property
def override_spec(self):
return self._get_argument_value("override_spec", False, True)
Expand Down Expand Up @@ -844,6 +934,10 @@
def configure_rules(self):
return self._get_argument_value("configure_rules", True, True)

@property
def deployment_files(self):
return self._get_argument_value("deployment_files", False, True)

@property
def config_file(self):
return self._get_argument_value("config_file", False, False)
Expand Down Expand Up @@ -872,10 +966,8 @@
def force(self):
return self._get_argument_value("force", False, False)

@property
def templates_to_process(self):
return self._templates_to_process
def evolve(self, **kwargs: Unpack[ManualArgs]) -> "ConfigMixIn":

@templates_to_process.setter
def templates_to_process(self, value: bool):
self._templates_to_process = value
config = deepcopy(self)
config._manual_args.update(kwargs)
return config
9 changes: 7 additions & 2 deletions src/cfnlint/context/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
__all__ = ["Context", "create_context_for_template"]
__all__ = ["Context", "create_context_for_template", "ParameterSet"]

from cfnlint.context.context import Context, Path, create_context_for_template
from cfnlint.context.context import (
Context,
ParameterSet,
Path,
create_context_for_template,
)
Loading
Loading