diff --git a/components/automate-cli/cmd/chef-automate/config.go b/components/automate-cli/cmd/chef-automate/config.go index 74bf552f9aa..1784d0cf046 100644 --- a/components/automate-cli/cmd/chef-automate/config.go +++ b/components/automate-cli/cmd/chef-automate/config.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "reflect" "regexp" "strings" @@ -11,6 +12,8 @@ import ( "path/filepath" "github.com/fatih/color" + ptoml "github.com/pelletier/go-toml" + "github.com/pkg/errors" "github.com/spf13/cobra" @@ -462,6 +465,23 @@ func runPatchCommand(cmd *cobra.Command, args []string) error { if configCmdFlags.waitTimeout < DEFAULT_TIMEOUT { return errors.Errorf("The operation timeout duration for each individual node during the config patch process should be set to a value greater than %v seconds.", DEFAULT_TIMEOUT) } + isLoggerConfig, err := checkIfRequestedConfigHasCentrailisedLogging(args) + if err != nil { + return err + } + if (configCmdFlags.opensearch || configCmdFlags.postgresql) && isLoggerConfig { + modifiedConfig, err := patchAndRemoveCentralisedLoggingForBackend(args, infra) + if err != nil { + return err + } + checkIfConfigHasOnlyLogConfig, err := checkUserConfigHasOnlyCentrailisedLogConfig(modifiedConfig) + if err != nil { + return err + } + if checkIfConfigHasOnlyLogConfig { + return nil + } + } configFile := args[0] configFileName := stringutils.GetFileName(configFile) @@ -561,6 +581,9 @@ func runPatchCommand(cmd *cobra.Command, args []string) error { } else { writer.Println(cmd.UsageString()) } + if (configCmdFlags.opensearch || configCmdFlags.postgresql) && isLoggerConfig { + os.Remove(configFile) + } } else { cfg, err := dc.LoadUserOverrideConfigFile(args[0]) @@ -592,6 +615,55 @@ func runPatchCommand(cmd *cobra.Command, args []string) error { return nil } +func patchAndRemoveCentralisedLoggingForBackend(args []string, infra *AutomateHAInfraDetails) (string, error) { + var inputfile string + sshconfig := &SSHConfig{} + sshconfig.sshUser = infra.Outputs.SSHUser.Value + sshconfig.sshKeyFile = infra.Outputs.SSHKeyFile.Value + sshconfig.sshPort = infra.Outputs.SSHPort.Value + sshUtil := NewSSHUtil(sshconfig) + + err := patchCentralisedLoggingForBackend(args, sshUtil, infra) + if err != nil { + return "", err + } + writer.Success("Centralised logging configuration is patched. \n") + if configCmdFlags.postgresql { + inputfile, err = removeCentralisedLogsFromUserConfigForPg(args[0]) + if err != nil { + return "", err + } + } + if configCmdFlags.opensearch { + inputfile, err = removeCentralisedLogsFromUserConfigForOs(args[0]) + if err != nil { + return "", err + } + } + args[0] = inputfile + return inputfile, nil +} + +func checkUserConfigHasOnlyCentrailisedLogConfig(inputfile string) (bool, error) { + file, err := os.Open(inputfile) + if err != nil { + return false, err + } + defer file.Close() + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(file) + if err != nil { + return false, err + } + + content := buf.String() + if content == "" { + os.Remove(inputfile) + return true, nil + } + return false, nil +} + // prePatchCheckForFrontendNodes patches the configuration for front end nodes in Automate HA func prePatchCheckForFrontendNodes(inputs *CmdInputs, sshUtil SSHUtil, infra *AutomateHAInfraDetails, remoteService string, writer *cli.Writer) error { srcPath, err := parseAndRemoveRestrictedKeysFromSrcFile(inputs.Args[0]) @@ -604,6 +676,25 @@ func prePatchCheckForFrontendNodes(inputs *CmdInputs, sshUtil SSHUtil, infra *Au return nil } +func patchCentralisedLoggingForBackend(args []string, sshUtil SSHUtil, infra *AutomateHAInfraDetails) error { + var remoteService string + var remoteIps []string + if configCmdFlags.postgresql { + remoteService = "postgresql" + remoteIps = infra.Outputs.PostgresqlPrivateIps.Value + } + if configCmdFlags.opensearch { + remoteService = "opensearch" + remoteIps = infra.Outputs.OpensearchPrivateIps.Value + } + // checking for log configuration + err := enableCentralizedLogConfigForHA(args, remoteService, sshUtil, remoteIps) + if err != nil { + return err + } + return nil +} + // prePatchCheckForPostgresqlNodes patches the config for postgresql nodes in Automate HA func prePatchCheckForPostgresqlNodes(inputs *CmdInputs, sshUtil SSHUtil, infra *AutomateHAInfraDetails, remoteService string, writer *cli.Writer) error { if isManagedServicesOn() { @@ -617,14 +708,8 @@ func prePatchCheckForPostgresqlNodes(inputs *CmdInputs, sshUtil SSHUtil, infra * args := inputs.Args - //checking for log configuration - err := enableCentralizedLogConfigForHA(args, remoteService, sshUtil, infra.Outputs.PostgresqlPrivateIps.Value) - if err != nil { - return err - } - //checking database configuration - existConfig, reqConfig, err := getExistingAndRequestedConfigForPostgres(args, infra, GET_APPLIED_CONFIG, sshUtil) + existConfig, reqConfig, err := getExistingAndRequestedConfigForPostgres(args, infra, GET_APPLIED_CONFIG, sshUtil, inputs) if err != nil { return err } @@ -664,12 +749,6 @@ func prePatchCheckForOpensearch(inputs *CmdInputs, sshUtil SSHUtil, infra *Autom args := inputs.Args - //checking for log configuration - err := enableCentralizedLogConfigForHA(args, remoteService, sshUtil, infra.Outputs.OpensearchPrivateIps.Value) - if err != nil { - return err - } - //checking database configuration existConfig, reqConfig, err := getExistingAndRequestedConfigForOpenSearch(args, infra, GET_APPLIED_CONFIG, sshUtil) if err != nil { @@ -905,7 +984,7 @@ func setConfigForPostgresqlNodes(args []string, remoteService string, sshUtil SS } //Getting Requested Config - reqConfigInterface, err := getConfigForArgsPostgresqlOrOpenSearch(args, postgresql) + reqConfigInterface, err := getConfigInterfaceForPostgresqlOrOpenSearch(args, postgresql) if err != nil { return err } @@ -939,7 +1018,7 @@ func setConfigForOpensearch(args []string, remoteService string, sshUtil SSHUtil } //Getting Requested Config - reqConfigInterface, err := getConfigForArgsPostgresqlOrOpenSearch(args, opensearch_const) + reqConfigInterface, err := getConfigInterfaceForPostgresqlOrOpenSearch(args, opensearch_const) if err != nil { return err } @@ -1130,8 +1209,8 @@ func getDecodedConfig(input string, remoteService string) (interface{}, error) { return nil, nil } -// getConfigForArgsPostgresqlAndOpenSearch gets the requested config from the args provided for postgresql or opensearch -func getConfigForArgsPostgresqlOrOpenSearch(args []string, remoteService string) (interface{}, error) { +// getConfigInterfaceForPostgresqlOrOpenSearch gets the requested config from the args provided for postgresql or opensearch +func getConfigInterfaceForPostgresqlOrOpenSearch(args []string, remoteService string) (interface{}, error) { pemBytes, err := os.ReadFile(args[0]) if err != nil { return nil, err @@ -1157,33 +1236,54 @@ func isConfigChanged(src interface{}, dest interface{}) bool { return !reflect.DeepEqual(src, dest) } -// getExistingAndRequestedConfigForPostgres get requested and existing config for postgresql -func getExistingAndRequestedConfigForPostgres(args []string, infra *AutomateHAInfraDetails, config string, sshUtil SSHUtil) (PostgresqlConfig, PostgresqlConfig, error) { +// getExistingAndRequestedConfigForPostgres get user updating config and existing *applied* config for postgresql +func getExistingAndRequestedConfigForPostgres(args []string, infra *AutomateHAInfraDetails, config string, sshUtil SSHUtil, inputs *CmdInputs) (PostgresqlConfig, PostgresqlConfig, error) { + //Getting Existing config from server - var existingConfig PostgresqlConfig + var existingAppliedConfig PostgresqlConfig var reqConfig PostgresqlConfig var emptyConfig PostgresqlConfig + + // getting applied remote config from pg node srcInputString, err := getConfigFromRemoteServer(infra, postgresql, config, sshUtil) if err != nil { - return existingConfig, reqConfig, errors.Wrapf(err, "Unable to get config from the server with error") + return existingAppliedConfig, reqConfig, errors.Wrapf(err, "Unable to get config from the server with error") } - existingConfigInterface, err := getDecodedConfig(srcInputString, postgresql) + existingAppliedConfigInterface, err := getDecodedConfig(srcInputString, postgresql) if err != nil { - return existingConfig, reqConfig, err + return existingAppliedConfig, reqConfig, err } - existingConfig = existingConfigInterface.(PostgresqlConfig) + existingAppliedConfig = existingAppliedConfigInterface.(PostgresqlConfig) //Getting Requested Config - reqConfigInterface, err := getConfigForArgsPostgresqlOrOpenSearch(args, postgresql) + reqConfigInterface, err := getConfigInterfaceForPostgresqlOrOpenSearch(args, postgresql) if err != nil { - return existingConfig, reqConfig, err + return existingAppliedConfig, reqConfig, err } reqConfig = reqConfigInterface.(PostgresqlConfig) if !isConfigChanged(emptyConfig, reqConfig) { - return existingConfig, reqConfig, status.Annotate(errors.New("Incorrect Config"), status.ConfigError) + return existingAppliedConfig, reqConfig, status.Annotate(errors.New("Incorrect Config"), status.ConfigError) } - mergo.Merge(&reqConfig, existingConfig) - return existingConfig, reqConfig, nil + mergo.Merge(&reqConfig, existingAppliedConfig) + return existingAppliedConfig, reqConfig, nil +} + +func checkIfRequestedConfigHasCentrailisedLogging(args []string) (bool, error) { + config, err := ptoml.LoadFile(args[0]) + if err != nil { + writer.Println(err.Error()) + return false, err + } + if config.Get("global.v1.log") != nil { + isconfig := config.Get("global.v1.log").(*ptoml.Tree) + if isconfig != nil { + val := isconfig.Get("redirect_sys_log").(bool) + if val == true { + return true, nil + } + } + } + return false, nil } // getExistingAndRequestedConfigForOpenSearch gets existed and requested config for opensearch @@ -1203,7 +1303,7 @@ func getExistingAndRequestedConfigForOpenSearch(args []string, infra *AutomateHA existingConfig = existingConfigInterface.(OpensearchConfig) //Getting Requested Config - reqConfigInterface, err := getConfigForArgsPostgresqlOrOpenSearch(args, opensearch_const) + reqConfigInterface, err := getConfigInterfaceForPostgresqlOrOpenSearch(args, opensearch_const) if err != nil { return existingConfig, reqConfig, err } @@ -1266,6 +1366,40 @@ func parseAndRemoveRestrictedKeysFromSrcFile(srcString string) (string, error) { } } +func removeCentralisedLogsFromUserConfigForPg(srcString string) (string, error) { + tomlbyt, _ := os.ReadFile(srcString) + destString := string(tomlbyt) + var dest PostgresqlConfig + if _, err := toml.Decode(destString, &dest); err != nil { + fmt.Println(err) + return "", err + } + timestamp := time.Now().Format("20060102150405") + tomlFile := srcString + timestamp + newSrcString, err := createTomlFileFromConfig(dest, tomlFile) + if err != nil { + return "", err + } + return newSrcString, nil +} + +func removeCentralisedLogsFromUserConfigForOs(srcString string) (string, error) { + tomlbyt, _ := os.ReadFile(srcString) + destString := string(tomlbyt) + var dest OpensearchConfig + if _, err := toml.Decode(destString, &dest); err != nil { + fmt.Println(err) + return "", err + } + timestamp := time.Now().Format("20060102150405") + tomlFile := srcString + timestamp + newSrcString, err := createTomlFileFromConfig(dest, tomlFile) + if err != nil { + return "", err + } + return newSrcString, nil +} + // If the output contains the word "error" then return error func checkOutputForError(output string) error { if strings.Contains(strings.ToUpper(strings.TrimSpace(output)), "ERROR") { diff --git a/components/automate-cli/cmd/chef-automate/config_test.go b/components/automate-cli/cmd/chef-automate/config_test.go index ca3a53cfad5..b1b43f51ddf 100644 --- a/components/automate-cli/cmd/chef-automate/config_test.go +++ b/components/automate-cli/cmd/chef-automate/config_test.go @@ -1,6 +1,7 @@ package main import ( + "io/ioutil" "os" "testing" @@ -8,7 +9,9 @@ import ( "github.com/chef/automate/api/config/shared" w "github.com/chef/automate/api/config/shared/wrappers" "github.com/pkg/errors" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var fqdn = "a2.test.com" @@ -103,6 +106,30 @@ func TestTomlFileCreateFromReqConfigLog(t *testing.T) { os.Remove(fileName) } +func TestCheckIfRequestedIsCentrailisedLogging(t *testing.T) { + file := []string{"../../pkg/testfiles/aws/centralised_log.toml"} + val, _ := checkIfRequestedConfigHasCentrailisedLogging(file) + assert.True(t, val) +} + +func TestCheckIfRequestedIsCentrailisedLoggingWithPgConfig(t *testing.T) { + file := []string{"../../pkg/testfiles/aws/pg.toml"} + val, _ := checkIfRequestedConfigHasCentrailisedLogging(file) + assert.Equal(t, val, false) +} + +func TestCheckIfRequestedIsCentrailisedLoggingWithInvalidFile(t *testing.T) { + file := []string{"../../pkg/testfiles/aws/centralised.toml"} + _, err := checkIfRequestedConfigHasCentrailisedLogging(file) + assert.Error(t, err) +} + +func TestCheckIfRequestedIsCentrailisedLoggingwitherror(t *testing.T) { + file := []string{"../../pkg/testfiles/aws/centralised_log.toml"} + val, _ := checkIfRequestedConfigHasCentrailisedLogging(file) + assert.True(t, val) +} + func TestErrorOnSelfManaged(t *testing.T) { testCases := []struct { isPostgresql bool @@ -248,6 +275,279 @@ func TestSetConfigForFrontEndNodes(t *testing.T) { } } +func TestPatchConfigCommand(t *testing.T) { + infra := getMockInfra() + file := "../../pkg/testfiles/aws/centralised_log.toml" + tests := []struct { + testName string + infra *AutomateHAInfraDetails + sshUtil SSHUtil + args []string + isAutomate bool + isChefServer bool + iPostgresql bool + isOpenSearch bool + wantErr bool + }{ + { + "patch for pg", + infra, + getMockSSHUtil(&SSHConfig{}, nil, "", nil), + []string{file}, + false, + false, + true, + false, + true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + if err := runPatchCommand(&cobra.Command{}, tt.args); (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} +func TestCheckUserConfigHasOnlyCentrailisedLogConfig(t *testing.T) { + t.Run("Returning false as the file is not empty", func(t *testing.T) { + fileContent := `[global.v1.log] + compress_rotated_logs = true + max_number_rotated_logs = 10 + max_size_rotate_logs = "10M" + redirect_log_file_path = "/var/tmp/" + redirect_sys_log = true` + tmpfile, err := ioutil.TempFile("", "test-output.toml") + require.NoError(t, err) + + n, err := tmpfile.Write([]byte(fileContent)) + require.NoError(t, err) + require.NotZero(t, n) + + err = tmpfile.Close() + require.NoError(t, err) + + isOnlyCentralisedLog, err := checkUserConfigHasOnlyCentrailisedLogConfig(tmpfile.Name()) + require.NoError(t, err) + require.Equal(t, isOnlyCentralisedLog, false) + require.NoError(t, err) + err = os.Remove(tmpfile.Name()) + require.NoError(t, err) + }) + t.Run("Returning true as the file is empty", func(t *testing.T) { + fileContent := `` + tmpfile, err := ioutil.TempFile("", "test-output.toml") + require.NoError(t, err) + + _, err = tmpfile.Write([]byte(fileContent)) + require.NoError(t, err) + + err = tmpfile.Close() + require.NoError(t, err) + + isOnlyCentralisedLog, err := checkUserConfigHasOnlyCentrailisedLogConfig(tmpfile.Name()) + require.NoError(t, err) + require.Equal(t, isOnlyCentralisedLog, true) + require.NoError(t, err) + require.NoError(t, err) + err = os.Remove(tmpfile.Name()) + + }) + t.Run("File not found", func(t *testing.T) { + filePath := "testdata/notfound.json" + _, err := checkUserConfigHasOnlyCentrailisedLogConfig(filePath) + require.Error(t, err) + }) +} + +func TestPatchAndRemoveCentralisedLoggingForBackendForPg(t *testing.T) { + infra := getMockInfra() + file := "../../pkg/testfiles/aws/centralised_log.toml" + tests := []struct { + testName string + infra *AutomateHAInfraDetails + sshUtil SSHUtil + args []string + wantErr bool + }{ + { + "patch and remove centralised log for pg", + infra, + getMockSSHUtil(&SSHConfig{}, nil, "", nil), + []string{file}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + fileName, err := patchAndRemoveCentralisedLoggingForBackend(tt.args, infra) + os.Remove(fileName) + assert.Nilf(t, err, "Unable to created toml file for postgresql toml") + + }) + } + +} + +func TestPatchAndRemoveCentralisedLoggingForBackendWithError(t *testing.T) { + infra := getMockInfra() + file := "" + tests := []struct { + testName string + infra *AutomateHAInfraDetails + sshUtil SSHUtil + args []string + wantErr bool + }{ + { + "Error while patching and remove centralised log for pg", + infra, + getMockSSHUtil(&SSHConfig{}, nil, "", nil), + []string{file}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + fileName, err := patchAndRemoveCentralisedLoggingForBackend(tt.args, infra) + os.Remove(fileName) + assert.Error(t, err) + + }) + } + +} + +func TestPatchCentralisedLoggingForBackendForPg(t *testing.T) { + infra := getMockInfra() + file := "../../pkg/testfiles/aws/centralised_log.toml" + tests := []struct { + testName string + infra *AutomateHAInfraDetails + sshUtil SSHUtil + args []string + wantErr bool + }{ + { + "Patch centralised log for pg", + infra, + getMockSSHUtil(&SSHConfig{}, nil, "", nil), + []string{file}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + configCmdFlags.postgresql = true + err := patchCentralisedLoggingForBackend(tt.args, tt.sshUtil, infra) + assert.Nilf(t, err, "Unable to created toml file for postgresql toml") + + }) + } + +} + +func TestPatchCentralisedLoggingForBackendForPgWithError(t *testing.T) { + infra := getMockInfra() + file := "" + tests := []struct { + testName string + infra *AutomateHAInfraDetails + sshUtil SSHUtil + args []string + wantErr bool + }{ + { + "Error while patching centralised log for pg", + infra, + getMockSSHUtil(&SSHConfig{}, nil, "", nil), + []string{file}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + configCmdFlags.postgresql = true + err := patchCentralisedLoggingForBackend(tt.args, tt.sshUtil, infra) + assert.Error(t, err) + + }) + } + +} + +func TestPatchCentralisedLoggingForBackendForOs(t *testing.T) { + infra := getMockInfra() + file := "../../pkg/testfiles/aws/centralised_log.toml" + tests := []struct { + testName string + infra *AutomateHAInfraDetails + sshUtil SSHUtil + args []string + wantErr bool + }{ + { + "Patch centralised log for os", + infra, + getMockSSHUtil(&SSHConfig{}, nil, "", nil), + []string{file}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + configCmdFlags.opensearch = true + err := patchCentralisedLoggingForBackend(tt.args, tt.sshUtil, infra) + assert.Nilf(t, err, "Unable to created toml file for postgresql toml") + + }) + } + +} + +func TestRemoveCentralisedLogsFromUserConfigForPg(t *testing.T) { + file := "../../pkg/testfiles/aws/centralised_log.toml" + emptyFile := "../../pkg/testfiles/aws/invalid_pg.toml" + t.Run("Remove log config from the file", + func(t *testing.T) { + file, err := removeCentralisedLogsFromUserConfigForPg(file) + os.Remove(file) + assert.NoError(t, err) + + }, + ) + t.Run("Error while removing log config from the file", + func(t *testing.T) { + file, err := removeCentralisedLogsFromUserConfigForPg(emptyFile) + os.Remove(file) + assert.Error(t, err) + + }, + ) +} + +func TestRemoveCentralisedLogsFromUserConfigForOs(t *testing.T) { + file := "../../pkg/testfiles/aws/centralised_log.toml" + emptyFile := "../../pkg/testfiles/aws/invalid_pg.toml" + t.Run("Remove log config from the file", + func(t *testing.T) { + file, err := removeCentralisedLogsFromUserConfigForOs(file) + os.Remove(file) + assert.NoError(t, err) + + }, + ) + t.Run("Error while removing log config from the file", + func(t *testing.T) { + file, err := removeCentralisedLogsFromUserConfigForOs(emptyFile) + os.Remove(file) + assert.Error(t, err) + + }, + ) + +} + func getMockSSHUtil(sshConfig *SSHConfig, CFTRError error, CSECOROutput string, CSECORError error) *MockSSHUtilsImpl { return &MockSSHUtilsImpl{ getSSHConfigFunc: func() *SSHConfig { diff --git a/components/automate-cli/pkg/testfiles/aws/centralised_log.toml b/components/automate-cli/pkg/testfiles/aws/centralised_log.toml new file mode 100644 index 00000000000..54429843870 --- /dev/null +++ b/components/automate-cli/pkg/testfiles/aws/centralised_log.toml @@ -0,0 +1,6 @@ +[global.v1.log] + compress_rotated_logs = true + max_number_rotated_logs = 10 + max_size_rotate_logs = "10M" + redirect_log_file_path = "/var/tmp/" + redirect_sys_log = true \ No newline at end of file diff --git a/components/automate-cli/pkg/testfiles/aws/invalid_pg.toml b/components/automate-cli/pkg/testfiles/aws/invalid_pg.toml new file mode 100644 index 00000000000..455ccd4f8dc --- /dev/null +++ b/components/automate-cli/pkg/testfiles/aws/invalid_pg.toml @@ -0,0 +1,3 @@ +[pg_dump] + enable = *true + path = "/mnt/automate_backups/postgresql/pg_dump \ No newline at end of file diff --git a/components/automate-cli/pkg/testfiles/aws/pg.toml b/components/automate-cli/pkg/testfiles/aws/pg.toml new file mode 100644 index 00000000000..002eb46aa2b --- /dev/null +++ b/components/automate-cli/pkg/testfiles/aws/pg.toml @@ -0,0 +1,3 @@ +[pg_dump] + enable = true + path = "/mnt/automate_backups/postgresql/pg_dump" \ No newline at end of file