From 3baa5df22fa86e55c3f073d808c935bf6c218d5d Mon Sep 17 00:00:00 2001 From: Kim Oliver Drechsel Date: Sun, 15 Sep 2024 14:44:03 +0200 Subject: [PATCH] feat: add support to deploy multiple stacks (#140) * feat: add support to deploy multiple stacks * test: adjust expected response strings * refactor: adjust log * refactor: adjust jobLog to stack context * refactor: adjust jobLog to stack context * refactor: move stack deployment logic to separate function * test: adjust expected response strings * refactor: format code * fix: adjust http status codes * refactor: change func args to pointers * fix: remove superfluous response.WriteHeader call * fix: remove superfluous response.WriteHeader call * refactor: adjust error log * refactor: adjust error log --- cmd/doco-cd/http_handler.go | 170 +++++++++++++------------- cmd/doco-cd/http_handler_test.go | 2 +- cmd/doco-cd/main_test.go | 6 +- internal/config/deploy_config.go | 33 ++--- internal/config/deploy_config_test.go | 28 ++++- internal/config/utils.go | 34 ++++-- internal/docker/compose_test.go | 10 +- 7 files changed, 164 insertions(+), 119 deletions(-) diff --git a/cmd/doco-cd/http_handler.go b/cmd/doco-cd/http_handler.go index 5efc9f3..385c955 100644 --- a/cmd/doco-cd/http_handler.go +++ b/cmd/doco-cd/http_handler.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "log/slog" "net/http" "os" @@ -29,7 +30,7 @@ type handlerData struct { func HandleEvent(ctx context.Context, jobLog *slog.Logger, w http.ResponseWriter, c *config.AppConfig, p webhook.ParsedPayload, jobID string, dockerCli command.Cli) { jobLog = jobLog.With(slog.String("repository", p.FullName)) - jobLog.Info("preparing project deployment") + jobLog.Info("preparing stack deployment") // Clone the repository jobLog.Debug( @@ -85,8 +86,9 @@ func HandleEvent(ctx context.Context, jobLog *slog.Logger, w http.ResponseWriter } fs := worktree.Filesystem + rootDir := fs.Root() - jobLog.Debug("repository cloned", slog.String("path", fs.Root())) + jobLog.Debug("repository cloned", slog.String("path", rootDir)) // Defer removal of the repository defer func(workDir string) { @@ -102,12 +104,12 @@ func HandleEvent(ctx context.Context, jobLog *slog.Logger, w http.ResponseWriter jobID, http.StatusInternalServerError) } - }(fs.Root()) + }(rootDir) jobLog.Debug("retrieving deployment configuration") - // Get the deployment config from the repository - deployConfig, err := config.GetDeployConfig(fs.Root(), p.Name) + // Get the deployment configs from the repository + deployConfigs, err := config.GetDeployConfigs(rootDir, p.Name) if err != nil { if errors.Is(err, config.ErrDeprecatedConfig) { jobLog.Warn(err.Error()) @@ -124,89 +126,17 @@ func HandleEvent(ctx context.Context, jobLog *slog.Logger, w http.ResponseWriter } } - jobLog = jobLog.With(slog.String("reference", deployConfig.Reference)) - - jobLog.Debug("deployment configuration retrieved", slog.Any("config", deployConfig)) - - workingDir := path.Join(fs.Root(), deployConfig.WorkingDirectory) - - err = os.Chdir(workingDir) - if err != nil { - errMsg = "failed to change working directory" - jobLog.Error(errMsg, logger.ErrAttr(err), slog.String("path", workingDir)) - JSONError(w, - errMsg, - err.Error(), - jobID, - http.StatusInternalServerError) - - return - } - - // Check if the default compose files are used - if reflect.DeepEqual(deployConfig.ComposeFiles, cli.DefaultFileNames) { - var tmpComposeFiles []string - - jobLog.Debug("checking for default compose files") - - // Check if the default compose files exist - for _, f := range deployConfig.ComposeFiles { - if _, err = os.Stat(path.Join(workingDir, f)); errors.Is(err, os.ErrNotExist) { - continue - } - - tmpComposeFiles = append(tmpComposeFiles, f) - } - - if len(tmpComposeFiles) == 0 { - errMsg = "no compose files found" - jobLog.Error(errMsg, - slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles))) - JSONError(w, - errMsg, - err.Error(), - jobID, - http.StatusInternalServerError) - + for _, deployConfig := range deployConfigs { + err = deployStack(jobLog, jobID, rootDir, &w, &ctx, &dockerCli, &p, deployConfig) + if err != nil { + msg := "deployment failed" + jobLog.Error(msg) + JSONError(w, err, msg, jobID, http.StatusInternalServerError) return } - - deployConfig.ComposeFiles = tmpComposeFiles } - project, err := docker.LoadCompose(ctx, workingDir, deployConfig.Name, deployConfig.ComposeFiles) - if err != nil { - errMsg = "failed to load project" - jobLog.Error(errMsg, - logger.ErrAttr(err), - slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles))) - JSONError(w, - errMsg, - err.Error(), - jobID, - http.StatusInternalServerError) - - return - } - - jobLog.Info("deploying project") - - err = docker.DeployCompose(ctx, dockerCli, project, deployConfig, p) - if err != nil { - errMsg = "failed to deploy project" - jobLog.Error(errMsg, - logger.ErrAttr(err), - slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles))) - JSONError(w, - errMsg, - err.Error(), - jobID, - http.StatusInternalServerError) - - return - } - - msg := "project deployment successful" + msg := "deployment successful" jobLog.Info(msg) JSONResponse(w, msg, jobID, http.StatusCreated) } @@ -266,3 +196,75 @@ func (h *handlerData) HealthCheckHandler(w http.ResponseWriter, _ *http.Request) h.log.Debug("health check successful") JSONResponse(w, "healthy", "", http.StatusOK) } + +func deployStack( + jobLog *slog.Logger, jobID, rootDir string, + w *http.ResponseWriter, ctx *context.Context, + dockerCli *command.Cli, p *webhook.ParsedPayload, deployConfig *config.DeployConfig, +) error { + stackLog := jobLog. + With(slog.String("stack", deployConfig.Name)). + With(slog.String("reference", deployConfig.Reference)) + + stackLog.Debug("deployment configuration retrieved", slog.Any("config", deployConfig)) + + workingDir := path.Join(rootDir, deployConfig.WorkingDirectory) + + err := os.Chdir(workingDir) + if err != nil { + errMsg = "failed to change working directory" + jobLog.Error(errMsg, logger.ErrAttr(err), slog.String("path", workingDir)) + + return fmt.Errorf("%s: %w", errMsg, err) + } + + // Check if the default compose files are used + if reflect.DeepEqual(deployConfig.ComposeFiles, cli.DefaultFileNames) { + var tmpComposeFiles []string + + jobLog.Debug("checking for default compose files") + + // Check if the default compose files exist + for _, f := range deployConfig.ComposeFiles { + if _, err = os.Stat(path.Join(workingDir, f)); errors.Is(err, os.ErrNotExist) { + continue + } + + tmpComposeFiles = append(tmpComposeFiles, f) + } + + if len(tmpComposeFiles) == 0 { + errMsg = "no compose files found" + stackLog.Error(errMsg, + slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles))) + + return fmt.Errorf("%s: %w", errMsg, err) + } + + deployConfig.ComposeFiles = tmpComposeFiles + } + + project, err := docker.LoadCompose(*ctx, workingDir, deployConfig.Name, deployConfig.ComposeFiles) + if err != nil { + errMsg = "failed to load compose config" + stackLog.Error(errMsg, + logger.ErrAttr(err), + slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles))) + + return fmt.Errorf("%s: %w", errMsg, err) + } + + stackLog.Info("deploying stack") + + err = docker.DeployCompose(*ctx, *dockerCli, project, deployConfig, *p) + if err != nil { + errMsg = "failed to deploy stack" + stackLog.Error(errMsg, + logger.ErrAttr(err), + slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles))) + + return fmt.Errorf("%s: %w", errMsg, err) + } + + return nil +} diff --git a/cmd/doco-cd/http_handler_test.go b/cmd/doco-cd/http_handler_test.go index 0ec4011..ee43fec 100644 --- a/cmd/doco-cd/http_handler_test.go +++ b/cmd/doco-cd/http_handler_test.go @@ -72,7 +72,7 @@ func TestHandlerData_HealthCheckHandler(t *testing.T) { } func TestHandlerData_WebhookHandler(t *testing.T) { - expectedResponse := `{"details":"project deployment successful","job_id":"[a-f0-9-]{36}"}` + expectedResponse := `{"details":"deployment successful","job_id":"[a-f0-9-]{36}"}` expectedStatusCode := http.StatusCreated payload, err := os.ReadFile(githubPayloadFile) diff --git a/cmd/doco-cd/main_test.go b/cmd/doco-cd/main_test.go index c6c590a..ced3026 100644 --- a/cmd/doco-cd/main_test.go +++ b/cmd/doco-cd/main_test.go @@ -46,7 +46,7 @@ func TestHandleEvent(t *testing.T) { Private: false, }, expectedStatusCode: http.StatusCreated, - expectedResponseBody: `{"details":"project deployment successful","job_id":"%s"}`, + expectedResponseBody: `{"details":"deployment successful","job_id":"%s"}`, overrideEnv: nil, }, { @@ -74,7 +74,7 @@ func TestHandleEvent(t *testing.T) { Private: true, }, expectedStatusCode: http.StatusCreated, - expectedResponseBody: `{"details":"project deployment successful","job_id":"%s"}`, + expectedResponseBody: `{"details":"deployment successful","job_id":"%s"}`, overrideEnv: nil, }, { @@ -104,7 +104,7 @@ func TestHandleEvent(t *testing.T) { Private: false, }, expectedStatusCode: http.StatusInternalServerError, - expectedResponseBody: `{"error":"no compose files found","details":"stat ` + filepath.Join(os.TempDir(), "kimdre/kimdre/docker-compose.yaml") + `: no such file or directory","job_id":"%s"}`, + expectedResponseBody: `{"error":"no compose files found: stat ` + filepath.Join(os.TempDir(), "kimdre/kimdre/docker-compose.yaml") + `: no such file or directory","details":"deployment failed","job_id":"%[1]s"}`, overrideEnv: nil, }, } diff --git a/internal/config/deploy_config.go b/internal/config/deploy_config.go index a8115b0..e32b28c 100644 --- a/internal/config/deploy_config.go +++ b/internal/config/deploy_config.go @@ -68,8 +68,8 @@ func (c *DeployConfig) validateConfig() error { return nil } -// GetDeployConfig returns either the deployment configuration from the repository or the default configuration -func GetDeployConfig(repoDir, name string) (*DeployConfig, error) { +// GetDeployConfigs returns either the deployment configuration from the repository or the default configuration +func GetDeployConfigs(repoDir, name string) ([]*DeployConfig, error) { files, err := os.ReadDir(repoDir) if err != nil { return nil, err @@ -79,7 +79,7 @@ func GetDeployConfig(repoDir, name string) (*DeployConfig, error) { DeploymentConfigFileNames := append(DefaultDeploymentConfigFileNames, DeprecatedDeploymentConfigFileNames...) for _, configFile := range DeploymentConfigFileNames { - config, err := getDeployConfigFile(repoDir, files, configFile) + configs, err := getDeployConfigsFromFile(repoDir, files, configFile) if err != nil { if errors.Is(err, ErrConfigFileNotFound) { continue @@ -88,27 +88,27 @@ func GetDeployConfig(repoDir, name string) (*DeployConfig, error) { } } - if config != nil { - if err := validator.Validate(config); err != nil { + if configs != nil { + if err := validator.Validate(configs); err != nil { return nil, err } // Check if the config file name is deprecated for _, deprecatedConfigFile := range DeprecatedDeploymentConfigFileNames { if configFile == deprecatedConfigFile { - return config, fmt.Errorf("%w: %s", ErrDeprecatedConfig, configFile) + return configs, fmt.Errorf("%w: %s", ErrDeprecatedConfig, configFile) } } - return config, nil + return configs, nil } } - return DefaultDeployConfig(name), nil + return []*DeployConfig{DefaultDeployConfig(name)}, nil } -// getDeployConfigFile returns the deployment configuration from the repository or nil if not found -func getDeployConfigFile(dir string, files []os.DirEntry, configFile string) (*DeployConfig, error) { +// getDeployConfigsFromFile returns the deployment configurations from the repository or nil if not found +func getDeployConfigsFromFile(dir string, files []os.DirEntry, configFile string) ([]*DeployConfig, error) { for _, f := range files { if f.IsDir() { continue @@ -116,17 +116,20 @@ func getDeployConfigFile(dir string, files []os.DirEntry, configFile string) (*D if f.Name() == configFile { // Get contents of deploy config file - c, err := FromYAML(path.Join(dir, f.Name())) + configs, err := FromYAML(path.Join(dir, f.Name())) if err != nil { return nil, err } - if err = c.validateConfig(); err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + // Validate all deploy configs + for _, c := range configs { + if err = c.validateConfig(); err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + } } - if c != nil { - return c, nil + if configs != nil { + return configs, nil } } } diff --git a/internal/config/deploy_config_test.go b/internal/config/deploy_config_test.go index f7c6cca..2436f17 100644 --- a/internal/config/deploy_config_test.go +++ b/internal/config/deploy_config_test.go @@ -29,7 +29,7 @@ func createTmpDir(t *testing.T) string { return dirName } -func TestGetDeployConfig(t *testing.T) { +func TestGetDeployConfigs(t *testing.T) { t.Run("Valid Config", func(t *testing.T) { fileName := ".doco-cd.yaml" reference := "refs/heads/test" @@ -58,11 +58,17 @@ compose_files: t.Fatal(err) } - config, err := GetDeployConfig(dirName, projectName) + configs, err := GetDeployConfigs(dirName, projectName) if err != nil { t.Fatal(err) } + if len(configs) != 1 { + t.Fatalf("expected 1 config, got %d", len(configs)) + } + + config := configs[0] + if config.Name != projectName { t.Errorf("expected name to be %v, got %s", projectName, config.Name) } @@ -108,11 +114,17 @@ compose_files: t.Fatal(err) } - config, err := GetDeployConfig(dirName, projectName) + configs, err := GetDeployConfigs(dirName, projectName) if err == nil || !errors.Is(err, ErrDeprecatedConfig) { t.Fatalf("expected deprecated config error, got %v", err) } + if len(configs) != 1 { + t.Fatalf("expected 1 config, got %d", len(configs)) + } + + config := configs[0] + if config == nil { t.Fatal("expected config to be returned, got nil") return @@ -136,7 +148,7 @@ compose_files: }) } -func TestGetDeployConfig_DefaultValues(t *testing.T) { +func TestGetDeployConfigs_DefaultValues(t *testing.T) { defaultConfig := DefaultDeployConfig(projectName) dirName := createTmpDir(t) @@ -147,11 +159,17 @@ func TestGetDeployConfig_DefaultValues(t *testing.T) { } }) - config, err := GetDeployConfig(dirName, projectName) + configs, err := GetDeployConfigs(dirName, projectName) if err != nil { t.Fatal(err) } + if len(configs) != 1 { + t.Fatalf("expected 1 config, got %d", len(configs)) + } + + config := configs[0] + if config.Name != projectName { t.Errorf("expected name to be %v, got %s", projectName, config.Name) } diff --git a/internal/config/utils.go b/internal/config/utils.go index d626252..9f1a110 100644 --- a/internal/config/utils.go +++ b/internal/config/utils.go @@ -1,7 +1,10 @@ package config import ( + "bytes" + "errors" "fmt" + "io" "os" "github.com/creasty/defaults" @@ -24,18 +27,35 @@ func (c *DeployConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } -func FromYAML(f string) (*DeployConfig, error) { - var c DeployConfig - +func FromYAML(f string) ([]*DeployConfig, error) { b, err := os.ReadFile(f) if err != nil { return nil, fmt.Errorf("failed to read file: %v", err) } - err = yaml.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal yaml: %v", err) + // Read all yaml documents in the file and unmarshal them into a slice of DeployConfig structs + dec := yaml.NewDecoder(bytes.NewReader(b)) + + var configs []*DeployConfig + + for { + var c DeployConfig + + err = dec.Decode(&c) + if err != nil { + if err == io.EOF { + break + } + + return nil, fmt.Errorf("failed to decode yaml: %v", err) + } + + configs = append(configs, &c) + } + + if len(configs) == 0 { + return nil, errors.New("no yaml documents found in file") } - return &c, nil + return configs, nil } diff --git a/internal/docker/compose_test.go b/internal/docker/compose_test.go index c825fb3..838fe48 100644 --- a/internal/docker/compose_test.go +++ b/internal/docker/compose_test.go @@ -163,7 +163,7 @@ compose_files: t.Fatal(err) } - deployConf, err := config.GetDeployConfig(dirName, projectName) + deployConfigs, err := config.GetDeployConfigs(dirName, projectName) if err != nil { t.Fatal(err) } @@ -186,8 +186,10 @@ compose_files: } }) - err = DeployCompose(ctx, dockerCli, project, deployConf, p) - if err != nil { - t.Fatal(err) + for _, deployConf := range deployConfigs { + err = DeployCompose(ctx, dockerCli, project, deployConf, p) + if err != nil { + t.Fatal(err) + } } }