From adc3659b438152defbe2ef0a0af7e977e48d5155 Mon Sep 17 00:00:00 2001 From: santoshkal Date: Wed, 1 Nov 2023 22:01:29 +0530 Subject: [PATCH] Fix typos, Add Tekton pipeline and ArgoCD schemas, Add example workflow document and link it in README Signed-off-by: santoshkal --- README.md | 37 +++------ cmd/cueval/example.md | 82 +++++++++++++++++++ deployment.yaml | 53 ------------ pkg/utils/utils.go | 2 +- templates/defaultpolicies/cue/application.cue | 54 ++++++++++++ .../defaultpolicies/cue/applicationset.cue | 5 -- templates/defaultpolicies/cue/tekton.cue | 39 ++++++++- templates/inputs/cue/pipeline.json | 74 +++++++++++++++++ 8 files changed, 259 insertions(+), 87 deletions(-) create mode 100644 cmd/cueval/example.md delete mode 100644 deployment.yaml create mode 100644 templates/defaultpolicies/cue/application.cue delete mode 100644 templates/defaultpolicies/cue/applicationset.cue create mode 100644 templates/inputs/cue/pipeline.json diff --git a/README.md b/README.md index 140c6da..c077431 100644 --- a/README.md +++ b/README.md @@ -90,28 +90,19 @@ Genval's release process signs binaries using Cosign's keyless signing mode. To # Example commands to verify a binary -wget https://github.com/intelops/genval/releases/download/v0.0.1/checksums.txt - -wget https://github.com/intelops/genval/releases/download/v0.0.1/checksums.txt.pem - -wget https://github.com/intelops/genval/releases/download/v0.0.1/checksums.txt.sig +$ wget https://github.com/intelops/genval/releases/download/v0.0.1/checksums.txt +$ wget https://github.com/intelops/genval/releases/download/v0.0.1/checksums.txt.pem +$ wget https://github.com/intelops/genval/releases/download/v0.0.1/checksums.txt.sig cosign verify-blob \ - --certificate-identity "https://github.com/intelops/genval/.github/workflows/release.yaml@refs/tags/v0.0.1" \ - --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ - --cert ./checksums.txt.pem \ - --signature ./checksums.txt.sig \ - ./checksums.txt - ``` - If verification is successful, you'll see "**Verified OK.**" @@ -127,11 +118,9 @@ If verification is successful, you'll see "**Verified OK.**" ## Quick Start + - - -For a quick start, pre-built templates for Dockerfile generation for popular languages can be found in the /templates/dockerFile-sample directory - +For a quick start, pre-built templates for Dockerfile generation for popular languages can be found in the `./templates/inputs/dockerfile_input` folder. ## Building from Source @@ -154,11 +143,8 @@ To build genval from source: The generated binary, genval, will be available in the current working directory. You can move it to your PATH or use it from the current directory. - - Genval offers two modes: - - `container` for Dockerfile validation and generation @@ -169,7 +155,6 @@ Genval offers two modes: ### Dockerfile Validation and Generation: - Run Genval with the --mode container flag, providing the path to your input JSON or YAML file using the `--reqinput` flag and specifying the desired output path for the generated Dockerfile along with `--inputpolicy` and `--outputpolicy` for validating the input JSON and the generated Dockerfile respectively. Genval will take care of the rest. @@ -188,25 +173,23 @@ $ genval --mode container --reqinput ./templates/inputs/dockerfile_input/golang_ -**Review Feedback**: Genal provides feedback based on best practice validation. Use this feedback to refine your Dockerfile. +**Review Feedback**: Genval provides feedback based on best practice validation. Use this feedback to refine your Dockerfile. ### Validation and Generation of Kubernetes configurations -The validation and generation of Kubernetes and CRD manifests are facilitated through the use of [cuelang](https://cuelang.org/docs/). When using Genval for validating and generating Kubernetes and related manifests, make use of the Genval tool in `cue` mode. This mode necessitates JSON input provided via the `--reqinput` flag. Furthermore, you should specify a `resource` flag, indicating the Kubernetes or CRD `Kind` that requires validation. Additionally, attach the `.cue schema definitions` to the `--policy` flag. These policy files can be provided wither store in the users local file system or from a remote URL, like a Git repository. +The validation and generation of Kubernetes and CRD manifests are facilitated through the use of [cuelang](https://cuelang.org/docs/). When using Genval for validating and generating Kubernetes and related manifests, make use of the Genval tool in `cue` mode. This mode necessitates JSON input provided via the `--reqinput` flag. Furthermore, you should specify a `resource` flag, indicating the Kubernetes or CRD `Kind` that requires validation. Additionally, attach the `.cue schema definitions` to the `--policy` flag. These policy files can be provided from the users local file system or from a remote URL, like a Git repository. -You have the flexibility to employ multiple `--policy` flags, allowing you to supply distinct `.cue` definitions as needed. For instance, your DevSecOps team can furnish a schema that enforces security best practices for a specific environment, encompassing all the pertinent mandatory fields. This approach leaves room for custom fields like `metadata`, `image`, `replicas`, specific to a **Deployment**, to be provided by the development teams. In the `cue` mode, development teams can then contribute their customized policies for validation and generation, tailoring the configurations to suit their particular environments. +You have the flexibility to employ multiple `--policy` flags, allowing you to supply distinct `.cue` definitions as needed. For instance, your DevSecOps/Platform engineering team can furnish a schema that enforces security best practices for a specific environment, encompassing all the pertinent mandatory fields. This approach leaves room for custom fields like `metadata`, `image`, `replicas`, specific to a **Deployment**, to be provided by the development teams. In the `cue` mode, development teams can then contribute their customized policies for validation and generation, tailoring the configurations to suit their particular environments. Example: - ```shell - $ genval --mode cue --reqinput ./templates/inputs/cue/deploy,json \ --resource Deployment \ --policy ./templates/defaultpolicies/cue/deployment.cue \ @@ -218,6 +201,8 @@ The above command will validate a Deployment manifests using the provided `.cue` > The `--resource` flag in `cue` mode needs a valid Kind, like in above example "Deployment" or StatefulSet, DaemonSet etc. +For a detailed workflow illustrating the capabilities of Cue and Genval for validating and generating Kubernetes configurations, you can refer to [this document](./cmd/cueval/example.md). + ### Templates -The `./templates` folder holds some sample files to be used in Genval. the `./templates/inputs` holds JSON input templates for both generating Dockerfiles in container mode and Kubernetes manifests in cue mode. Similarly, all the default policies for both the modes are stored in `./templates/defaultpolices` directory. User can use these template files to start with and as they go along they can customize these files to suite their specific use cases. \ No newline at end of file +The `./templates` folder holds some sample files to be used in Genval. the `./templates/inputs` holds JSON input templates for both generating Dockerfiles in `container` mode and Kubernetes manifests in `cue` mode. Similarly, all the default policies for both the modes are stored in `./templates/defaultpolices` directory. User can use these template files to start with and as they go along they can customize these files to suite their specific use cases. \ No newline at end of file diff --git a/cmd/cueval/example.md b/cmd/cueval/example.md new file mode 100644 index 0000000..2a5a15b --- /dev/null +++ b/cmd/cueval/example.md @@ -0,0 +1,82 @@ +# Validation and generation of Kubernetes Deployment using Genval + +## Introduction + +In this guide, we'll explore the usage of the Genval tool in Cue mode for validating and generating Kubernetes YAML manifests, specifically focusing on Kubernetes Deployments. Genval simplifies the process of ensuring your Kubernetes configurations adhere to security best practices and industry standards. While we use Kubernetes Deployments as an example, the same principles apply to other Kubernetes resources, ArgoCD, TektonCD, Crossplane, and ClusterAPI resources. + +## What is Cue + +Cue aka [Cuelang](https://cuelang.org/docs) is a versatile configuration language and runtime environment designed to fulfill various tasks, including defining and validating data schemas, configuring files, and generating configuration artifacts. It presents a succinct and expressive syntax tailored for efficient configuration management while providing advanced features like type checking, constraints, and code generation. + + +One of Cue's standout features is its robust [type system](https://cuelang.org/docs/tutorials/tour/types/types/), which empowers developers to specify and enforce data types within their configuration files. This type checking capability greatly enhances the reliability and stability of configurations. Additionally, Cue allows developers to establish [constraints](https://cuelang.org/docs/tutorials/tour/intro/constraints/) or rules that configuration data must follow. Enforcing these constraints ensures that configurations align with specific business requirements or predefined rules, elevating the overall quality and correctness of configurations. + +## Flow diagram fo Cue mode + +

+ +

+ + + + +## Genval in action + +The core idea behind using Genval in Cue mode is as follows: + +- **Cue Schema**: DevOps or platform engineers create a Cue schema that abstracts the complex requirements of a Kubernetes Deployment resource. This schema defines various aspects of the Deployment, such as: + + - `securityContext:` + - Resource limits and requests + - any constraints in container images, like no use of `latest` image tags + - Probes like `livenessProbe` and/or `readinessProbe` + - And more + +An example Cue schema can be found in `./templates/defaultpolicies/cue/deployment.json`. The Cue format closely resembles JSON. Importing the Kubernetes APIs helps map fields/structs in the `deployment.cue` schema to upstream Kubernetes APIs. For instance: + + +```cue +package k8s + +import ( + apps "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" +) + +#Deployment: apps.#Deployment & { + apiVersion: "apps/v1" + kind: "Deployment" + + // Removed for brevity + ... +} +``` + +In this example, we map the #Deployment definition to the apps.#Deployment object defined in our cue.mod, which is created by importing the Go APIs into Cue types. + +- **Cue Definitions**: Definitions in Cue are marked with a `#` symbol, indicating they are used for validation and not included as data output. The `&` symbol unifies nested fields, mapping them to `#Deployment`. +- **Validation with Input JSON**: Developers provide a minimal input JSON for a Deployment, without worrying about the complex requirements. A sample input JSON for a Deployment is located at `./templates/inputs/cue/deploy.json`. +- **Validation and Generation**: With both the input in JSON format and the Cue schema in place, Genval leverages Cue's powerful validation and generation capabilities. Run the following command: + +```shell +$ genval --mode cue --reqinput ./templates/inputs/cue/deploy.json \ # input to be validated + --resource Deployment # a Kind of resource we are trying to validate and generate, needs a valid Kind + --policy ./templates/defaultpolicies/cue/deployment.cue # Our Cue schema/policy for Deployment + --policy ./templates/defaultpolicies/cue/metadata.cue # We can pass multiple policies/schemas referring them in the main policy `deployment.cue`. +``` + - `--reqinput`: Specifies the input to be validated. + - `--resource`: Defines the Kubernetes resource kind to validate and generate. It must correspond to a valid Kind. + - `--policy`: Points to the Cue schema/policy for Deployment. Multiple policies/schemas can be used by referencing them in the main policy `deployment.cue`. + +- **Validation Result**: If the input passes validation, Genval responds with: + +```shell +$ INFO[0000] validation for Deployment succeeded, generated manifest: deployment.yaml +``` + +The complete, validated deployment.yaml is generated and written to the current working directory. + +- **Validation-Only Mode**: If you only want to validate input against a policy without generating a manifest, set the `--verify` flag to true (it's false by default). + +By following these steps, you can efficiently validate and generate Kubernetes YAML manifests using Genval in Cue mode, ensuring your configurations meet security requirements and best practices. + diff --git a/deployment.yaml b/deployment.yaml deleted file mode 100644 index b2512d4..0000000 --- a/deployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: cueTest - env: mytest - name: genval - namespace: websites -spec: - minReadySeconds: 5 - replicas: 3 - revisionHistoryLimit: 5 - selector: - matchLabels: - app: cueTest - template: - metadata: - labels: - app: cueTest - version: changeMe - spec: - containers: - - image: nginx:1.20 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - httpGet: - port: 80 - initialDelaySeconds: 6 - periodSeconds: 10 - name: website - ports: - - containerPort: 80 - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - port: 80 - initialDelaySeconds: 6 - periodSeconds: 10 - resources: - limits: - cpu: 100m - memory: 256Mi - requests: - cpu: 100m - memory: 256Mi - securityContext: - allowPrivilegeEscalation: false - privileged: false - runAsGroup: 1001 - runAsNonRoot: true - runAsUser: 1001 diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index ede17a2..ab3e739 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -172,7 +172,7 @@ func fetchFileWithCURL(urlStr string) (string, error) { log.Println("Error checking directory:", err) } - cmd := exec.Command("curl", "-s", "-o", filepath.Join(dir, filename), urlStr) + cmd := exec.Command("curl", "-s", "-O", filepath.Join(dir, filename), urlStr) if err := cmd.Run(); err != nil { log.Errorf("failed fetching content using curl: %v", err) return "", err diff --git a/templates/defaultpolicies/cue/application.cue b/templates/defaultpolicies/cue/application.cue new file mode 100644 index 0000000..e2a0bc1 --- /dev/null +++ b/templates/defaultpolicies/cue/application.cue @@ -0,0 +1,54 @@ +package argo + +import "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + +#Application: v1alpha1.#Application & { + apiVersion: string | *"argoproj.io/v1alpha1" + kind: string | *"Application" + metadata: _Metadata + spec: v1alpha.#AppicationSpec & { + source: { + repoURL: string // requires a URL to your manifest repo + targetRevision: string // requires to track the commit/branch/tag + path: string // requires the path to the manifest in the repo + chart: string // requires if your app uses Helm + helm: { + // All your Helm file values go here + ... + } + // If your app uses Kustomize overlays, they go here + kustomize: { + ... + } + directory: { + ... + } + plugin: { + ... + } + } + + syncPolicy: { + automated: { + ... + } + syncOptions: [...string] + retry: { + ... + } + ... + } + ... + revisionHistoryLimit: int + } +} + +_Metadata: { + name: *"genval" | string + namespace: *"genval" | string + labels: { + app: string | *"genval" + env: *"mytest" | string + } + ... +} diff --git a/templates/defaultpolicies/cue/applicationset.cue b/templates/defaultpolicies/cue/applicationset.cue deleted file mode 100644 index 1c59961..0000000 --- a/templates/defaultpolicies/cue/applicationset.cue +++ /dev/null @@ -1,5 +0,0 @@ -package argocd - -import v1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - -#ApplicationSet: v1alpha1.#ApplicationSet diff --git a/templates/defaultpolicies/cue/tekton.cue b/templates/defaultpolicies/cue/tekton.cue index d0f00a4..9fa0dea 100644 --- a/templates/defaultpolicies/cue/tekton.cue +++ b/templates/defaultpolicies/cue/tekton.cue @@ -1,5 +1,40 @@ package tekton -import "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" +import "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" -#Pipeline: v1.#Pipeline +#Pipeline: v1beta1.#Pipeline & { + apiVersion: string | *"tekton.dev/v1beta1" + kind: string | *"Pipeline" + metadata: _Metadata + spec: v1beta1.#PipelineSpec & { + params: [...{ + name: string + description: string + ... + }] + tasks: [...{ + name: string + taskRef: name: string + params: [...{ + name: string + value: string + ... + }] + }] + results: [...{ + name: string + description: string + value: string + ... + }] + }} + +_Metadata: { + name: *"genval" | string + namespace: *"genval" | string + labels: { + app: string | *"genval" + env: *"mytest" | string + } + ... +} diff --git a/templates/inputs/cue/pipeline.json b/templates/inputs/cue/pipeline.json new file mode 100644 index 0000000..b817492 --- /dev/null +++ b/templates/inputs/cue/pipeline.json @@ -0,0 +1,74 @@ +{ + "apiVersion": "tekton.dev/v1beta1", + "kind": "Pipeline", + "metadata": { + "name": "add-three-numbers" + }, + "spec": { + "params": [ + { + "name": "first", + "description": "the first operand" + }, + { + "name": "second", + "description": "the second operand" + }, + { + "name": "third", + "description": "the third operand" + } + ], + "tasks": [ + { + "name": "first-add", + "taskRef": { + "name": "add-task" + }, + "params": [ + { + "name": "first", + "value": "$(params.first)" + }, + { + "name": "second", + "value": "$(params.second)" + } + ] + }, + { + "name": "second-add", + "taskRef": { + "name": "add-task" + }, + "params": [ + { + "name": "first", + "value": "$(tasks.first-add.results.sum)" + }, + { + "name": "second", + "value": "$(params.third)" + } + ] + } + ], + "results": [ + { + "name": "sum", + "description": "the sum of all three operands", + "value": "$(tasks.second-add.results.sum)" + }, + { + "name": "partial-sum", + "description": "the sum of first two operands", + "value": "$(tasks.first-add.results.sum)" + }, + { + "name": "all-sum", + "description": "the sum of everything", + "value": "$(tasks.second-add.results.sum)-$(tasks.first-add.results.sum)" + } + ] + } + } \ No newline at end of file