From 75cef6083f2f177996cbfc855b716640384160a4 Mon Sep 17 00:00:00 2001 From: Chien-An Lai Date: Fri, 10 Jan 2020 13:39:52 -0800 Subject: [PATCH] paasta validate in GO --- cmd/validate/schemas/adhoc_schema.json | 103 ++++ cmd/validate/schemas/kubernetes_schema.json | 529 ++++++++++++++++++++ cmd/validate/schemas/marathon_schema.json | 365 ++++++++++++++ cmd/validate/schemas/tron_schema.json | 308 ++++++++++++ cmd/validate/validate.go | 214 ++++++++ go.mod | 3 + go.sum | 18 + 7 files changed, 1540 insertions(+) create mode 100644 cmd/validate/schemas/adhoc_schema.json create mode 100644 cmd/validate/schemas/kubernetes_schema.json create mode 100644 cmd/validate/schemas/marathon_schema.json create mode 100644 cmd/validate/schemas/tron_schema.json create mode 100644 cmd/validate/validate.go diff --git a/cmd/validate/schemas/adhoc_schema.json b/cmd/validate/schemas/adhoc_schema.json new file mode 100644 index 0000000..17a3ac5 --- /dev/null +++ b/cmd/validate/schemas/adhoc_schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#adhoc-clustername-yaml", + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "patternProperties": { + "^_.*$": { + "type": "object", + "additionalProperties": true + }, + "^([a-z0-9]|[a-z0-9][a-z0-9_-]*[a-z0-9])*$": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "cpus": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true, + "default": 1 + }, + "mem": { + "type": "number", + "minimum": 32, + "exclusiveMinimum": true, + "default": 1024 + }, + "disk": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true, + "default": 1024 + }, + "gpus": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "cmd": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "type": "object", + "patternProperties": { + "^[a-zA-Z_]+[a-zA-Z0-9_]*$": { + "type": "string" + } + }, + "additionalProperties": false + }, + "extra_volumes": { + "type": "array", + "items": { + "type": "object" + }, + "uniqueItems": true + }, + "deploy_group": { + "type": "string" + }, + "net": { + "type": "string" + }, + "cap_add": { + "type": "array", + "items": { + "type": "string" + } + }, + "cfs_period_us": { + "type": "integer", + "minimum": 1000, + "maximum": 1000000, + "exclusiveMinimum": false + }, + "cpu_burst_add": { + "type": "number", + "minimum": 0.0, + "exclusiveMinimum": false + }, + "extra_docker_args": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "pool": { + "type": "string" + }, + "role": { + "type": "string" + } + } + } + } +} diff --git a/cmd/validate/schemas/kubernetes_schema.json b/cmd/validate/schemas/kubernetes_schema.json new file mode 100644 index 0000000..a884729 --- /dev/null +++ b/cmd/validate/schemas/kubernetes_schema.json @@ -0,0 +1,529 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#kubernetes-clustername-yaml", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + "^_.*$": { + "type": "object", + "additionalProperties": true + }, + "^([a-z0-9]|[a-z0-9][a-z0-9_-]*[a-z0-9])*$": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "allOf": [ + { + "oneOf": [ + { + "properties": { + "healthcheck_mode": { + "enum": [ + "tcp", + "http", + "https" + ] + } + } + }, + { + "properties": { + "healthcheck_mode": { + "enum": [ + "cmd" + ] + }, + "healthcheck_cmd": { + "type": "string" + } + }, + "required": [ + "healthcheck_cmd" + ] + } + ] + }, + { + "oneOf": [ + { + "properties": { + "drain_method": { + "enum": [ + "noop", + "hacheck", + "test" + ] + } + } + }, + { + "properties": { + "drain_method": { + "enum": [ + "http" + ] + }, + "drain_method_params": { + "type": "object", + "properties": { + "drain": { + "type": "object" + }, + "stop_draining": { + "type": "object" + }, + "is_draining": { + "type": "object" + }, + "is_safe_to_kill": { + "type": "object" + } + }, + "required": [ + "drain", + "stop_draining", + "is_draining", + "is_safe_to_kill" + ] + } + }, + "required": [ + "drain_method_params" + ] + } + ] + } + ], + "properties": { + "cpus": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true, + "default": 0.25 + }, + "mem": { + "type": "number", + "minimum": 32, + "exclusiveMinimum": true, + "default": 1024 + }, + "disk": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true, + "default": 1024 + }, + "gpus": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "instances": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "min_instances": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "max_instances": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "backoff_factor": { + "type": "integer", + "default": 2 + }, + "max_launch_delay_seconds": { + "type": "integer", + "default": 300 + }, + "registrations": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "bounce_method": { + "enum": [ + "crossover", + "brutal", + "downthenup" + ] + }, + "bounce_health_params": { + "type": "object", + "properties": { + "check_haproxy": { + "type": "boolean", + "default": true + }, + "min_task_uptime": { + "type": "number" + }, + "haproxy_min_fraction_up": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "exclusiveMinimum": true, + "exclusiveMaximum": false + } + } + }, + "bounce_margin_factor": { + "type": "number", + "default": 1, + "minimum": 0, + "maximum": 1, + "exclusiveMinimum": true, + "exclusiveMaximum": false + }, + "bounce_priority": { + "type": "integer" + }, + "deploy_group": { + "type": "string" + }, + "autoscaling": { + "type": "object" + }, + "horizontal_autoscaling": { + "type": "object", + "additionalProperties": false, + "properties": { + "min_replicas": { + "type": "integer", + "default": 1 + }, + "max_replicas": { + "type": "integer" + }, + "cpu": { + "type": "object", + "additionalProperties": false, + "properties": { + "target_average_value": { + "type": "integer", + "default": 70, + "minimum": 0, + "maximum": 100, + "exclusiveMinimum": true, + "exclusiveMaximum": false + } + }, + "required": [ + "target_average_value" + ] + }, + "memory": { + "type": "object", + "additionalProperties": false, + "properties": { + "target_average_value": { + "type": "integer", + "default": 70, + "minimum": 0, + "maximum": 100, + "exclusiveMinimum": true, + "exclusiveMaximum": false + } + }, + "required": [ + "target_average_value" + ] + }, + "uwsgi": { + "type": "object", + "additionalProperties": false, + "properties": { + "target_average_value": { + "type": "integer", + "default": 70, + "minimum": 0, + "maximum": 100, + "exclusiveMinimum": true, + "exclusiveMaximum": false + }, + "dimensions": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^(?!(sf_|gcp_|azure_|aws_))[a-zA-Z]([-_a-zA-Z0-9])*$": { + "type": "string" + } + } + } + }, + "required": [ + "target_average_value" + ] + }, + "http": { + "type": "object", + "additionalProperties": false, + "properties": { + "target_average_value": { + "type": "number", + "default": 70, + "minimum": 0, + "maximum": 100, + "exclusiveMinimum": true, + "exclusiveMaximum": false + }, + "dimensions": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^(?!(sf_|gcp_|azure_|aws_))[a-zA-Z]([-_a-zA-Z0-9])*$": { + "type": "string" + } + } + } + }, + "required": [ + "target_average_value" + ] + } + }, + "patternProperties": { + "(?!(^cpu|memory|http|uwsgi)$)(^[a-z]([-a-z0-9]*[a-z0-9])?$)": { + "type": "object", + "additionalProperties": false, + "properties": { + "target_value": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "signalflow_metrics_query": { + "type": "string" + } + }, + "required": [ + "signalflow_metrics_query", + "target_value" + ] + } + }, + "required": [ + "max_replicas" + ] + }, + "sfn_autoscaling": { + "type": "object" + }, + "service_account_name": { + "type": "string" + }, + "drain_method": { + "enum": [ + "noop", + "hacheck", + "http", + "test" + ], + "default": "noop" + }, + "drain_method_params": { + "type": "object" + }, + "constraints": { + "type": "array", + "items": { + "type": "array" + }, + "uniqueItems": true + }, + "extra_constraints": { + "type": "array", + "items": { + "type": "array" + }, + "uniqueItems": true + }, + "pool": { + "type": "string" + }, + "cmd": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + } + ] + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "type": "object", + "patternProperties": { + "^[a-zA-Z_]+[a-zA-Z0-9_]*$": { + "type": "string" + } + }, + "additionalProperties": false + }, + "cap_add": { + "type": "array", + "items": { + "type": "string" + } + }, + "extra_volumes": { + "type": "array", + "items": { + "type": "object" + }, + "uniqueItems": true + }, + "monitoring": { + "type": "object", + "properties": { + "team": { + "type": "string" + }, + "page": { + "type": "boolean" + } + }, + "additionalProperties": true + }, + "marathon_shard": { + "type": "integer", + "minimum": 0 + }, + "previous_marathon_shards": { + "type": "array" + }, + "aws_ebs_volumes": { + "type": "array", + "items": { + "type": "object" + }, + "uniqueItems": true + }, + "persistent_volumes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "container_path": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "mode": { + "type": "string" + } + } + }, + "uniqueItems": true + }, + "replication_threshold": { + "type": "integer", + "minimum": 0 + }, + "cfs_period_us": { + "type": "integer", + "minimum": 1000, + "maximum": 1000000, + "exclusiveMinimum": false + }, + "net": { + "type": "string" + }, + "container_port": { + "type": "number" + }, + "deploy_blacklist": { + "type": "array" + }, + "deploy_whitelist": { + "type": "array" + }, + "monitoring_blacklist": { + "type": "array" + }, + "iam_role": { + "type": "string" + }, + "healthcheck_mode": { + "enum": [ + "cmd", + "tcp", + "http", + "https" + ] + }, + "healthcheck_cmd": { + "type": "string", + "default": "/bin/true" + }, + "healthcheck_grace_period_seconds": { + "type": "number", + "default": 60 + }, + "healthcheck_interval_seconds": { + "type": "number", + "default": 10 + }, + "healthcheck_timeout_seconds": { + "type": "number", + "default": 10 + }, + "healthcheck_max_consecutive_failures": { + "type": "integer", + "default": 6 + }, + "healthcheck_uri": { + "type": "string", + "default": "/status" + }, + "cpu_burst_add": { + "type": "number", + "minimum": 0.0, + "exclusiveMinimum": false + }, + "host_port": { + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 65535, + "exclusiveMinimum": false + }, + "dependencies_reference": { + "type": "string" + }, + "extra_docker_args": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "security": { + "type": "object", + "properties": { + "outbound_firewall": { + "enum": [ + "block", + "monitor" + ] + } + } + } + } + } + } +} diff --git a/cmd/validate/schemas/marathon_schema.json b/cmd/validate/schemas/marathon_schema.json new file mode 100644 index 0000000..aca5c79 --- /dev/null +++ b/cmd/validate/schemas/marathon_schema.json @@ -0,0 +1,365 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#marathon-clustername-yaml", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + "^_.*$": { + "type": "object", + "additionalProperties": true + }, + "^([a-z0-9]|[a-z0-9][a-z0-9_-]*[a-z0-9])*$": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "allOf": [ + { + "oneOf": [ + { + "properties": { + "healthcheck_mode": { + "enum": [ + "tcp", + "http", + "https" + ] + } + } + }, + { + "properties": { + "healthcheck_mode": { + "enum": [ + "cmd" + ] + }, + "healthcheck_cmd": { + "type": "string" + } + }, + "required": [ + "healthcheck_cmd" + ] + } + ] + }, + { + "oneOf": [ + { + "properties": { + "drain_method": { + "enum": [ + "noop", + "hacheck", + "test" + ] + } + } + }, + { + "properties": { + "drain_method": { + "enum": [ + "http" + ] + }, + "drain_method_params": { + "type": "object", + "properties": { + "drain": { + "type": "object" + }, + "stop_draining": { + "type": "object" + }, + "is_draining": { + "type": "object" + }, + "is_safe_to_kill": { + "type": "object" + } + }, + "required": [ + "drain", + "stop_draining", + "is_draining", + "is_safe_to_kill" + ] + } + }, + "required": [ + "drain_method_params" + ] + } + ] + } + ], + "properties": { + "cpus": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true, + "default": 0.25 + }, + "mem": { + "type": "number", + "minimum": 32, + "exclusiveMinimum": true, + "default": 1024 + }, + "disk": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true, + "default": 1024 + }, + "gpus": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "instances": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "min_instances": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "max_instances": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "backoff_factor": { + "type": "integer", + "default": 2 + }, + "max_launch_delay_seconds": { + "type": "integer", + "default": 300 + }, + "registrations": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "bounce_method": { + "type": "string" + }, + "bounce_health_params": { + "type": "object", + "properties": { + "check_haproxy": { + "type": "boolean", + "default": true + }, + "min_task_uptime": { + "type": "number" + }, + "haproxy_min_fraction_up": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "exclusiveMinimum": true, + "exclusiveMaximum": false + } + } + }, + "bounce_margin_factor": { + "type": "number", + "default": 1, + "minimum": 0, + "maximum": 1, + "exclusiveMinimum": true, + "exclusiveMaximum": false + }, + "bounce_priority": { + "type": "integer" + }, + "bounce_start_deadline": { + "type": "number" + }, + "deploy_group": { + "type": "string" + }, + "autoscaling": { + "type": "object" + }, + "sfn_autoscaling": { + "type": "object" + }, + "drain_method": { + "enum": [ + "noop", + "hacheck", + "http", + "test" + ], + "default": "noop" + }, + "drain_method_params": { + "type": "object" + }, + "constraints": { + "type": "array", + "items": { + "type": "array" + }, + "uniqueItems": true + }, + "extra_constraints": { + "type": "array", + "items": { + "type": "array" + }, + "uniqueItems": true + }, + "pool": { + "type": "string" + }, + "cmd": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "type": "object", + "patternProperties": { + "^[a-zA-Z_]+[a-zA-Z0-9_]*$": { + "type": "string" + } + }, + "additionalProperties": false + }, + "extra_volumes": { + "type": "array", + "items": { + "type": "object" + }, + "uniqueItems": true + }, + "monitoring": { + "type": "object", + "properties": { + "team": { + "type": "string" + }, + "page": { + "type": "boolean" + } + }, + "additionalProperties": true + }, + "net": { + "type": "string" + }, + "container_port": { + "type": "number" + }, + "deploy_blacklist": { + "type": "array" + }, + "deploy_whitelist": { + "type": "array" + }, + "healthcheck_mode": { + "enum": [ + "cmd", + "tcp", + "http", + "https" + ] + }, + "healthcheck_cmd": { + "type": "string", + "default": "/bin/true" + }, + "healthcheck_grace_period_seconds": { + "type": "number", + "default": 60 + }, + "healthcheck_interval_seconds": { + "type": "number", + "default": 10 + }, + "healthcheck_timeout_seconds": { + "type": "number", + "default": 10 + }, + "healthcheck_max_consecutive_failures": { + "type": "integer", + "default": 6 + }, + "healthcheck_uri": { + "type": "string", + "default": "/status" + }, + "marathon_shard": { + "type": "integer", + "minimum": 0 + }, + "previous_marathon_shards": { + "type": "array" + }, + "replication_threshold": { + "type": "integer", + "minimum": 0 + }, + "cap_add": { + "type": "array", + "items": { + "type": "string" + } + }, + "cfs_period_us": { + "type": "integer", + "minimum": 1000, + "maximum": 1000000, + "exclusiveMinimum": false + }, + "cpu_burst_add": { + "type": "number", + "minimum": 0.0, + "exclusiveMinimum": false + }, + "host_port": { + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 65535, + "exclusiveMinimum": false + }, + "dependencies_reference": { + "type": "string" + }, + "extra_docker_args": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "security": { + "type": "object", + "properties": { + "outbound_firewall": { + "enum": [ + "block", + "monitor" + ] + } + } + } + } + } + } +} diff --git a/cmd/validate/schemas/tron_schema.json b/cmd/validate/schemas/tron_schema.json new file mode 100644 index 0000000..91b4ca2 --- /dev/null +++ b/cmd/validate/schemas/tron_schema.json @@ -0,0 +1,308 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "tron on paasta yaml (docs todo)", + "type": "object", + "definitions": { + "name": { + "type": "string", + "pattern": "^[A-Za-z_][\\w\\-]{0,254}$" + }, + "time_delta": { + "type": "string", + "pattern": "^\\d+\\s*[a-z]+$" + }, + "action": { + "type": "object", + "additionalProperties": false, + "required": [ + "command" + ], + "properties": { + "name": { + "$ref": "#definitions/name" + }, + "command": { + "type": "string" + }, + "node": { + "$ref": "#definitions/name" + }, + "requires": { + "type": "array", + "items": { + "type": "string" + } + }, + "retries": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": false + }, + "retries_delay": { + "$ref": "#definitions/time_delta" + }, + "executor": { + "enum": [ + "ssh", + "paasta" + ] + }, + "cpus": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "cpu_burst_add": { + "type": "number", + "minimum": 0.0, + "exclusiveMinimum": false + }, + "cap_add": { + "type": "array", + "items": { + "type": "string" + } + }, + "mem": { + "type": "number", + "minimum": 32, + "exclusiveMinimum": true + }, + "disk": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "constraints": { + "type": "array", + "items": { + "type": "array" + }, + "uniqueItems": true + }, + "extra_constraints": { + "type": "array", + "items": { + "type": "array" + }, + "uniqueItems": true + }, + "service": { + "type": "string" + }, + "deploy_group": { + "type": "string" + }, + "pool": { + "type": "string" + }, + "env": { + "type": "object", + "patternProperties": { + "^[a-zA-Z_]+[a-zA-Z0-9_]*$": { + "type": "string" + } + }, + "additionalProperties": false + }, + "extra_volumes": { + "type": "array", + "items": { + "type": "object" + }, + "uniqueItems": true + }, + "cluster": { + "type": "string" + }, + "expected_runtime": { + "$ref": "#definitions/time_delta" + }, + "triggered_by": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "trigger_downstreams": { + "type": [ + "object", + "boolean" + ], + "additionalProperties": { + "type": "string" + } + }, + "on_upstream_rerun": { + "type": "string" + }, + "trigger_timeout": { + "$ref": "#definitions/time_delta" + } + } + }, + "job": { + "type": "object", + "required": [ + "node", + "schedule", + "actions" + ], + "additionalProperties": false, + "properties": { + "name": { + "$ref": "#definitions/name" + }, + "node": { + "$ref": "#definitions/name" + }, + "schedule": { + "type": [ + "string", + "object" + ] + }, + "actions": { + "type": [ + "array", + "object" + ], + "items": { + "$ref": "#definitions/action" + }, + "patternProperties": { + ".+": { + "$ref": "#definitions/action" + } + } + }, + "cluster": { + "type": "string" + }, + "monitoring": { + "type": "object", + "properties": { + "team": { + "type": "string" + }, + "runbook": { + "type": "string" + }, + "page": { + "type": "boolean" + }, + "tip": { + "type": "string" + }, + "notification_email": { + "type": [ + "string", + "boolean", + "null" + ] + }, + "realert_every": { + "type": "integer", + "minimum": -1, + "exclusiveMinimum": false + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + } + }, + "irc_channels": { + "type": "array", + "items": { + "type": "string" + } + }, + "slack_channels": { + "type": "array", + "items": { + "type": "string" + } + }, + "ticket": { + "type": "boolean" + }, + "project": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "component": { + "type": [ + "string", + "array" + ] + }, + "description": { + "type": "string" + }, + "alert_after": { + "type": "string" + }, + "check_that_every_day_has_a_successful_run": { + "type": "boolean" + }, + "priority": { + "type": "string" + } + }, + "additionalProperties": false + }, + "queueing": { + "type": "boolean" + }, + "allow_overlap": { + "type": "boolean" + }, + "run_limit": { + "type": "integer", + "minimum": 0, + "exclusiveMinimum": true + }, + "all_nodes": { + "type": "boolean" + }, + "cleanup_action": { + "$ref": "#definitions/action" + }, + "enabled": { + "type": "boolean" + }, + "max_runtime": { + "$ref": "#definitions/time_delta" + }, + "expected_runtime": { + "$ref": "#definitions/time_delta" + }, + "time_zone": { + "type": "string" + }, + "service": { + "type": "string" + }, + "deploy_group": { + "type": "string" + } + } + } + }, + "patternProperties": { + "^_.*$": { + "type": "object", + "additionalProperties": true + }, + "^[^_].*$": { + "$ref": "#definitions/job" + } + } +} diff --git a/cmd/validate/validate.go b/cmd/validate/validate.go new file mode 100644 index 0000000..96fb388 --- /dev/null +++ b/cmd/validate/validate.go @@ -0,0 +1,214 @@ +package main + +import ( + "fmt" + "flag" + "errors" + "os" + "path" + "path/filepath" + "strings" + "github.com/xeipuuv/gojsonschema" + "github.com/ghodss/yaml" + "io/ioutil" + "github.com/fatih/color" +) + +type ValidateOptions struct { + Service string + SOAConfigPath string +} + +func parseFlags(opts *ValidateOptions) error { + flag.StringVar(&opts.Service, "service", "", "service to validate") + flag.StringVar(&opts.SOAConfigPath, "yelpsoa-config-root", "", "yelpsoa-configs path") + flag.Parse() + return nil +} + +func sanitiseKubernetesName(service string) string { + name := strings.ReplaceAll(service, "_", "--") + if strings.HasPrefix(name, "--") { + name = strings.Replace(name, "--", "underscore-", 1) + } + return strings.ToLower(name) +} + +// getServicePathDetermines the path of the directory containing the conf files +func getServicePath(service string, soa_dir string) (string, error) { + if service != "" { + return filepath.Join(soa_dir, service), nil + } + + current_path, _ := os.Getwd() + if soa_dir == current_path { + return soa_dir, nil + } + return soa_dir, errors.New("Unknown service") +} + +// guessServiceName deduces the service name from the pwd +func guessServiceName(service string) string { + if service != "" { + return service + } + current_path, _ := os.Getwd() + return filepath.Base(current_path) + +} + +// validateAutoscalingConfigs hasn't been implemented yet. +func validateAutoscalingConfigs(service_path string) bool { + return true +} + +// validateUniqueInstanceNames hasn't been implemented yet. +func validateUniqueInstanceNames(service_path string) bool { + return true +} + +// validatePaastaObjects hasn't been implemented yet. +func validatePaastaObjects(service_path string) bool { + return true +} + +// validateTron hasn't been implemented yet. +func validateTron(service_path string) bool { + return true +} + +// validateSchema checks if the specified config file has a valid schema. +// file_path is the path to file to validate. +// file_type is what schema type should we validate against +func validateSchema(file_name string, file_type string) bool { + fileContents, err := ioutil.ReadFile(file_name) + if err != nil { + fmt.Printf("Failed to load config file: %s\n", err) + return false + } + fileContentsJSON, err := yaml.YAMLToJSON(fileContents) + if err != nil { + fmt.Printf("Failed to convert yaml to json: %s\n", err) + return false + } + + schemaLoader := gojsonschema.NewReferenceLoader("file://schemas/" + file_type + "_schema.json") + fileLoader := gojsonschema.NewStringLoader(string(fileContentsJSON)) + result, err := gojsonschema.Validate(schemaLoader, fileLoader) + if err != nil { + fmt.Printf("Schema invalid: %s\n", err) + return false + } + if result.Valid() { + green := color.New(color.FgGreen).SprintFunc() + fmt.Printf("%s Successfully validated Schema: %s\n", green("Yes"), file_name) + return true + } else { + red := color.New(color.FgRed).SprintFunc() + blue := color.New(color.FgBlue).SprintFunc() + paasta_document_link := "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html" + fmt.Printf("%s Failed to validate schema. More info: %s: %s\n", red("No"), blue(paasta_document_link), file_name) + fmt.Printf(" Validation Message:\n") + for _, desc := range result.Errors() { + fmt.Printf(" - %s\n", desc) + } + return false + } +} + +// validateAllSchemas Finds all recognized config files in service directory,and validates their schema. +// service_path is the path to location of configuration files. +func validateAllSchemas(service_path string) bool { + matches, _ := filepath.Glob(service_path + "/*.yaml") + return_code := true + for _, file_name := range matches { + file_info, _ := os.Lstat(file_name) + if file_info.Mode() & os.ModeSymlink != 0 { + continue + } + basename := path.Base(file_name) + // This should be file_types := [4]string{"marathon", "adhoc", "tron", "kubernetes"} + // But there is still some issues need to solve. For example, + // https://groups.google.com/forum/#!topic/golang-nuts/7qgSDWPIh_E + file_types := [2]string{"marathon", "adhoc"} + for _, file_type := range file_types { + if strings.HasPrefix(basename, file_type) == true { + if validateSchema(file_name, file_type) == false { + return_code = false + } + } + } + } + return return_code +} + +func validateServiceName(service string) bool { + sanitise_name := sanitiseKubernetesName(service) + if len(sanitise_name) > 63 { + fmt.Printf("Length of service name %s should be no more than 63.\n", sanitise_name) + return false + } + return true +} + +// checkServicePath heck that the specified path exists and has yaml files. +// service_path is the path to directory that should contain yaml files +func checkServicePath(service_path string) bool { + if _, err := os.Stat(service_path); err != nil { + fmt.Printf("%s is not a directory", service_path) + return false + } + if matches, err := filepath.Glob(service_path + "/*.yaml"); len(matches) == 0 || err != nil { + fmt.Printf("%s does not contain any .yaml files\n", service_path) + return false + } + return true +} + +func paastaValidateSoaConfigs(service string, service_path string) bool { + if checkServicePath(service_path) == false { + return false + } + if validateServiceName(service) == false { + return false + } + if validateAllSchemas(service_path) == false { + return false + } + if validateTron(service_path) == false { + return false + } + if validatePaastaObjects(service_path) == false { + return false + } + if validateUniqueInstanceNames(service_path) == false { + return false + } + if validateAutoscalingConfigs(service_path) == false { + return false + } + + return true +} + +func main() { + options := &ValidateOptions{} + err := parseFlags(options) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + service_path, err := getServicePath(options.Service, options.SOAConfigPath) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + service := guessServiceName(options.Service) + + result := paastaValidateSoaConfigs(service, service_path) + if result == false { + os.Exit(1) + } +} diff --git a/go.mod b/go.mod index 6c9d2fa..e993570 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,9 @@ module github.com/Yelp/paasta-tools-go require ( github.com/dlespiau/kube-test-harness v0.0.0-20190930170435-ec3f93e1a754 + github.com/fatih/color v1.9.0 // indirect github.com/fatih/structs v1.1.0 + github.com/ghodss/yaml v1.0.0 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/googleapis/gnostic v0.3.1 // indirect github.com/imdario/mergo v0.3.8 // indirect @@ -13,6 +15,7 @@ require ( github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.4.0 github.com/subosito/gotenv v1.2.0 + github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect diff --git a/go.sum b/go.sum index f5eee1d..f822976 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,12 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dlespiau/kube-test-harness v0.0.0-20190930170435-ec3f93e1a754 h1:Qd6djMDE2KFeLwuKQ4B316XYt101DFoymbgoxFHeKCY= github.com/dlespiau/kube-test-harness v0.0.0-20190930170435-ec3f93e1a754/go.mod h1:rTr8X4qZPRmQKsyAjhECPi+zPnmlcmv5W9s1F11oBSo= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -22,6 +26,11 @@ github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62F github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -43,6 +52,12 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -58,8 +73,11 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=