Prune pipelines / pipeline runs

Closes #153.

CHANGELOG.md

@@ -27,6 +27,7 @@ listed in the changelog.
 - Create and use one PVC per repository ([#160](https://github.com/opendevstack/ods-pipeline/issues/160))
 - Leaner NodeJS 16 Typescript image task, removed cypress and its dependencies ([#426](https://github.com/opendevstack/ods-pipeline/issues/426))
 - Update skopeo (from 1.4 to 1.5) and buildah (from 1.22 to 1.23) ([#430](https://github.com/opendevstack/ods-pipeline/issues/430))
+- Prune pipelines and pipeline runs ([#153](https://github.com/opendevstack/ods-pipeline/issues/153))
 
 ### Fixed
 - Cannot enable debug mode in some tasks ([#377](https://github.com/opendevstack/ods-pipeline/issues/377))

cmd/start/main.go

@@ -200,7 +200,7 @@ func main() {
 		}
 	}
 
-	if len(ctxt.Environment) > 0 {
+	if ctxt.Environment != "" {
 		env, err := odsConfig.Environment(ctxt.Environment)
 		if err != nil {
 			log.Fatal(fmt.Sprintf("err during namespace extraction: %s", err))

cmd/webhook-interceptor/main.go

@@ -7,13 +7,15 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/opendevstack/pipeline/internal/interceptor"
 	kubernetesClient "github.com/opendevstack/pipeline/internal/kubernetes"
 	tektonClient "github.com/opendevstack/pipeline/internal/tekton"
 	"github.com/opendevstack/pipeline/pkg/bitbucket"
+	"github.com/opendevstack/pipeline/pkg/logging"
 )
 
 const (
@@ -29,6 +31,10 @@ const (
 	storageClassNameDefault  = "standard"
 	storageSizeEnvVar        = "ODS_STORAGE_SIZE"
 	storageSizeDefault       = "2Gi"
+	pruneMinKeepHoursEnvVar  = "ODS_PRUNE_MIN_KEEP_HOURS"
+	pruneMinKeepHoursDefault = 48
+	pruneMaxKeepRunsEnvVar   = "ODS_PRUNE_MAX_KEEP_RUNS"
+	pruneMaxKeepRunsDefault  = 20
 )
 
 func init() {
@@ -55,55 +61,27 @@ func serve() error {
 		return fmt.Errorf("%s must be set", tokenEnvVar)
 	}
 
-	taskKind := os.Getenv(taskKindEnvVar)
-	if taskKind == "" {
-		taskKind = taskKindDefault
-		log.Println(
-			"INFO:",
-			taskKindEnvVar,
-			"not set, using default value:",
-			taskKindDefault,
-		)
-	}
+	taskKind := readStringFromEnvVar(taskKindEnvVar, taskKindDefault)
 
-	taskSuffix := os.Getenv(taskSuffixEnvVar)
-	if taskSuffix == "" {
-		log.Println(
-			"INFO:",
-			taskSuffixEnvVar,
-			"not set, using no suffix",
-		)
-	}
+	taskSuffix := readStringFromEnvVar(taskSuffixEnvVar, "")
 
-	storageProvisioner := os.Getenv(storageProvisionerEnvVar)
-	if storageProvisioner == "" {
-		log.Println(
-			"INFO:",
-			storageProvisionerEnvVar,
-			"not set, using no storage provisioner",
-		)
-	}
+	storageProvisioner := readStringFromEnvVar(storageProvisionerEnvVar, "")
 
-	storageClassName := os.Getenv(storageClassNameEnvVar)
-	if storageClassName == "" {
-		storageClassName = storageClassNameDefault
-		log.Println(
-			"INFO:",
-			storageClassNameEnvVar,
-			"not set, using default value:",
-			storageClassNameDefault,
-		)
-	}
+	storageClassName := readStringFromEnvVar(storageClassNameEnvVar, storageClassNameDefault)
 
-	storageSize := os.Getenv(storageSizeEnvVar)
-	if storageSize == "" {
-		storageSize = storageSizeDefault
-		log.Println(
-			"INFO:",
-			storageSizeEnvVar,
-			"not set, using default value:",
-			storageSizeDefault,
-		)
+	storageSize := readStringFromEnvVar(storageSizeEnvVar, storageSizeDefault)
+
+	pruneMinKeepHours, err := readIntFromEnvVar(
+		pruneMinKeepHoursEnvVar, pruneMinKeepHoursDefault,
+	)
+	if err != nil {
+		return err
+	}
+	pruneMaxKeepRuns, err := readIntFromEnvVar(
+		pruneMaxKeepRunsEnvVar, pruneMaxKeepRunsDefault,
+	)
+	if err != nil {
+		return err
 	}
 
 	namespace, err := getFileContent(namespaceFile)
@@ -135,6 +113,21 @@ func serve() error {
 		BaseURL:  strings.TrimSuffix(repoBase, "/scm"),
 	})
 
+	// TODO: Use this logger in the interceptor as well, not just in the pruner.
+	var logger logging.LeveledLoggerInterface
+	if os.Getenv("DEBUG") == "true" {
+		logger = &logging.LeveledLogger{Level: logging.LevelDebug}
+	} else {
+		logger = &logging.LeveledLogger{Level: logging.LevelInfo}
+	}
+
+	pruner := interceptor.NewPipelineRunPrunerByStage(
+		tClient,
+		logger,
+		pruneMinKeepHours,
+		pruneMaxKeepRuns,
+	)
+
 	server, err := interceptor.NewServer(interceptor.ServerConfig{
 		Namespace:  namespace,
 		Project:    project,
@@ -147,9 +140,10 @@ func serve() error {
 			ClassName:   storageClassName,
 			Size:        storageSize,
 		},
-		KubernetesClient: kClient,
-		TektonClient:     tClient,
-		BitbucketClient:  bitbucketClient,
+		KubernetesClient:  kClient,
+		TektonClient:      tClient,
+		BitbucketClient:   bitbucketClient,
+		PipelineRunPruner: pruner,
 	})
 	if err != nil {
 		return err
@@ -178,3 +172,32 @@ func getFileContent(filename string) (string, error) {
 	}
 	return string(content), nil
 }
+
+func readIntFromEnvVar(envVar string, fallback int) (int, error) {
+	var val int
+	valString := os.Getenv(envVar)
+	if valString == "" {
+		val = fallback
+		log.Println(
+			"INFO:", envVar, "not set, using default value:", fallback,
+		)
+	} else {
+		i, err := strconv.Atoi(valString)
+		if err != nil {
+			return 0, fmt.Errorf("could not read value of %s: %s", envVar, err)
+		}
+		val = i
+	}
+	return val, nil
+}
+
+func readStringFromEnvVar(envVar, fallback string) string {
+	val := os.Getenv(envVar)
+	if val == "" {
+		val = fallback
+		log.Printf(
+			"INFO: %s not set, using default value: '%s'", envVar, fallback,
+		)
+	}
+	return val
+}

deploy/cd-namespace/chart/templates/deployment-interceptor.yaml

@@ -37,12 +37,21 @@ spec:
                 secretKeyRef:
                   key: password
                   name: ods-bitbucket-auth
+            - name: DEBUG
+              valueFrom:
+                configMapKeyRef:
+                  key: debug
+                  name: ods-pipeline
             - name: ODS_STORAGE_PROVISIONER
               value: '{{.Values.interceptor.storageProvisioner}}'
             - name: ODS_STORAGE_CLASS_NAME
               value: '{{.Values.interceptor.storageClassName}}'
             - name: ODS_STORAGE_SIZE
               value: '{{.Values.interceptor.storageSize}}'
+            - name: ODS_PRUNE_MIN_KEEP_HOURS
+              value: '{{int .Values.pipelineRunMinKeepHours}}'
+            - name: ODS_PRUNE_MAX_KEEP_RUNS
+              value: '{{int .Values.pipelineRunMaxKeepRuns}}'
           readinessProbe:
             httpGet:
               path: /health

deploy/cd-namespace/chart/values.yaml

@@ -95,6 +95,12 @@ notification:
       ]
     }
 
+# Pipeline(Run) Pruning
+# Minimum hours to keep a pipeline run. Has precendence over pipelineRunMaxKeepRuns.
+pipelineRunMinKeepHours: '48'
+# Maximum number of pipeline runs to keep per stage (stages: DEV, QA, PROD).
+pipelineRunMaxKeepRuns: '20'
+
 # Webhook Interceptor
 interceptor:
   # PVC (used for the pipeline workspace)

docs/design/software-design-specification.adoc

@@ -371,6 +371,8 @@ For Git commits which message instructs to skip CI, no pipelines are triggererd.
 A pipeline is created or updated corresponding to the Git branch received in the webhook request. The pipeline name is made out of the component and the sanitized branch. A maximum of 63 characters is respected. Tasks (including `finally` tasks) of the pipline are read from the ODS config file in the repository.
 
 A PVC is created per repository unless it exists already. The name is equal to `ods-workspace-<component>` (shortened to 63 characters if longer). This PVC is then used in the pipeline as a shared workspace.
+
+Pipelines and pipeline runs are pruned when a webhook trigger is received. Pipeline runs that are newer than the configured time window are protected from pruning. Older pipeline runs are cleaned up to not grow beyond the configured maximum amount. If all pipeline runs of one pipeline can be pruned, the whole pipeline is pruned. The pruning strategy is applied per repository and stage (DEV, QA, PROD) to avoid aggressive pruning of QA and PROD pipeline runs.
 |===
 
 ===== Artifact Download

docs/design/software-requirements-specification.adoc

@@ -63,6 +63,9 @@ The tasks shall create artifacts of their work. Those artifacts shall be stored
 
 | SRS-INTERCEPTOR-4
 | The interceptor shall create a PVC for use as a pipeline workspace per repository.
+
+| SRS-INTERCEPTOR-5
+| The interceptor shall prune pipelines and pipeline runs per repository and stage.
 |===
 
 === Task Requirements