diff --git a/.editorconfig b/.editorconfig index e709fe6..1e72057 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,3 +2,7 @@ indent_size=2 insert_final_newline = true indent_style = space + +[*.go] +indent_style = tab +indent_size = 4 diff --git a/cmd/configure.go b/cmd/configure.go index 52916f8..27ef483 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -18,7 +18,6 @@ var configure = &cobra.Command{ Short: "Creates a profile by prompting for input over the command line.", Long: "Will first prompt for a name then a series of variable specific for the user being created", Run: func(cmd *cobra.Command, args []string) { - configPath := profiles.GetConfigFilePath() cfg, err := ini.Load(configPath) if err != nil { diff --git a/cmd/create.go b/cmd/create.go index 0875466..a615105 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -24,7 +24,7 @@ var create = &cobra.Command{ Short: "Creates an entity of a resource.", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - body, err := createInternal(args, crud.AutoFillOnCreate) + body, err := createInternal(context.Background(), args, crud.AutoFillOnCreate) if err != nil { return err @@ -92,7 +92,10 @@ var create = &cobra.Command{ }, } -func createInternal(args []string, autoFillOnCreate bool) (string, error) { +func createInternal(ctx context.Context, args []string, autoFillOnCreate bool) (string, error) { + crud.OutstandingRequestCounter.Add(1) + defer crud.OutstandingRequestCounter.Done() + // Find Resource resource, ok := resources.GetResourceByName(args[0]) if !ok { @@ -135,7 +138,7 @@ func createInternal(args []string, autoFillOnCreate bool) (string, error) { } // Submit request - resp, err = httpclient.DoFileRequest(context.TODO(), resourceURL, byteBuf, contentType) + resp, err = httpclient.DoFileRequest(ctx, resourceURL, byteBuf, contentType) } else { // Assume it's application/json @@ -169,7 +172,7 @@ func createInternal(args []string, autoFillOnCreate bool) (string, error) { } // Submit request - resp, err = httpclient.DoRequest(context.TODO(), "POST", resourceURL, params.Encode(), strings.NewReader(body)) + resp, err = httpclient.DoRequest(ctx, "POST", resourceURL, params.Encode(), strings.NewReader(body)) } diff --git a/cmd/delete-all.go b/cmd/delete-all.go index 16eb161..bf1c2fc 100644 --- a/cmd/delete-all.go +++ b/cmd/delete-all.go @@ -24,7 +24,7 @@ var DeleteAll = &cobra.Command{ Args: cobra.MinimumNArgs(1), Hidden: false, RunE: func(cmd *cobra.Command, args []string) error { - return deleteAllInternal(args) + return deleteAllInternal(context.Background(), args) }, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -39,7 +39,7 @@ var DeleteAll = &cobra.Command{ }, } -func deleteAllInternal(args []string) error { +func deleteAllInternal(ctx context.Context, args []string) error { // Find Resource resource, ok := resources.GetResourceByName(args[0]) if !ok { @@ -54,7 +54,7 @@ func deleteAllInternal(args []string) error { return fmt.Errorf("resource %s doesn't support DELETE", args[0]) } - allParentEntityIds, err := getParentIds(context.Background(), resource) + allParentEntityIds, err := getParentIds(ctx, resource) if err != nil { return fmt.Errorf("could not retrieve parent ids for for resource %s, error: %w", resource.PluralName, err) @@ -78,7 +78,7 @@ func deleteAllInternal(args []string) error { params := url.Values{} params.Add("page[limit]", "25") - resp, err := httpclient.DoRequest(context.Background(), "GET", resourceURL, params.Encode(), nil) + resp, err := httpclient.DoRequest(ctx, "GET", resourceURL, params.Encode(), nil) if err != nil { return err @@ -122,7 +122,7 @@ func deleteAllInternal(args []string) error { } - delPage(resource.DeleteEntityInfo, allIds) + delPage(ctx, resource.DeleteEntityInfo, allIds) } } @@ -160,7 +160,7 @@ func getParentIds(ctx context.Context, resource resources.Resource) ([][]id.Idab var flowsUrlRegex = regexp.MustCompile("^/v2/flows/([^/]+)$") -func delPage(urlInfo *resources.CrudEntityInfo, ids [][]id.IdableAttributes) { +func delPage(ctx context.Context, urlInfo *resources.CrudEntityInfo, ids [][]id.IdableAttributes) { // Create a wait group to run DELETE in parallel wg := sync.WaitGroup{} for _, idAttr := range ids { @@ -177,7 +177,7 @@ func delPage(urlInfo *resources.CrudEntityInfo, ids [][]id.IdableAttributes) { } // Submit request - resp, err := httpclient.DoRequest(context.TODO(), "DELETE", resourceURL, "", nil) + resp, err := httpclient.DoRequest(ctx, "DELETE", resourceURL, "", nil) if err != nil { return } @@ -200,7 +200,7 @@ func delPage(urlInfo *resources.CrudEntityInfo, ids [][]id.IdableAttributes) { id := matches[1] jsonBody := fmt.Sprintf(`{ "data": { "id":"%s", "type": "flow", "slug": "delete-%s" }}`, id, id) - resp2, err := httpclient.DoRequest(context.TODO(), "PUT", resourceURL, "", strings.NewReader(jsonBody)) + resp2, err := httpclient.DoRequest(ctx, "PUT", resourceURL, "", strings.NewReader(jsonBody)) if err != nil { return @@ -212,7 +212,7 @@ func delPage(urlInfo *resources.CrudEntityInfo, ids [][]id.IdableAttributes) { } } - resp3, err := httpclient.DoRequest(context.TODO(), "DELETE", resourceURL, "", nil) + resp3, err := httpclient.DoRequest(ctx, "DELETE", resourceURL, "", nil) if err != nil { return } diff --git a/cmd/delete.go b/cmd/delete.go index 5e50511..132d104 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -23,7 +23,7 @@ var delete = &cobra.Command{ Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - body, err := deleteInternal(args) + body, err := deleteInternal(context.Background(), args) if err != nil { return err } @@ -96,13 +96,16 @@ var delete = &cobra.Command{ }, } -func deleteInternal(args []string) (string, error) { +func deleteInternal(ctx context.Context, args []string) (string, error) { + crud.OutstandingRequestCounter.Add(1) + defer crud.OutstandingRequestCounter.Done() + resource, ok := resources.GetResourceByName(args[0]) if !ok { return "", fmt.Errorf("could not find resource %s", args[0]) } - resp, err := deleteResource(args) + resp, err := deleteResource(ctx, args) if err != nil { return "", err } @@ -132,7 +135,7 @@ func deleteInternal(args []string) (string, error) { } -func deleteResource(args []string) (*http.Response, error) { +func deleteResource(ctx context.Context, args []string) (*http.Response, error) { // Find Resource resource, ok := resources.GetResourceByName(args[0]) if !ok { @@ -186,7 +189,7 @@ func deleteResource(args []string) (*http.Response, error) { } // Submit request - resp, err := httpclient.DoRequest(context.TODO(), "DELETE", resourceURL, params.Encode(), payload) + resp, err := httpclient.DoRequest(ctx, "DELETE", resourceURL, params.Encode(), payload) if err != nil { return nil, fmt.Errorf("got error %s", err.Error()) } diff --git a/cmd/get.go b/cmd/get.go index d65e7bc..59c7d1d 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -22,7 +22,7 @@ var get = &cobra.Command{ Short: "Retrieves a single resource.", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - body, err := getInternal(args) + body, err := getInternal(context.Background(), args) if err != nil { return err } @@ -109,8 +109,8 @@ var get = &cobra.Command{ }, } -func getInternal(args []string) (string, error) { - resp, err := getResource(args) +func getInternal(ctx context.Context, args []string) (string, error) { + resp, err := getResource(ctx, args) if err != nil { return "", err @@ -158,7 +158,10 @@ func getUrl(resource resources.Resource, args []string) (*resources.CrudEntityIn } } -func getResource(args []string) (*http.Response, error) { +func getResource(ctx context.Context, args []string) (*http.Response, error) { + crud.OutstandingRequestCounter.Add(1) + defer crud.OutstandingRequestCounter.Done() + // Find Resource resource, ok := resources.GetResourceByName(args[0]) if !ok { @@ -210,7 +213,7 @@ func getResource(args []string) (*http.Response, error) { } // Submit request - resp, err := httpclient.DoRequest(context.TODO(), "GET", resourceURL, params.Encode(), nil) + resp, err := httpclient.DoRequest(ctx, "GET", resourceURL, params.Encode(), nil) if err != nil { return nil, fmt.Errorf("got error %s", err.Error()) diff --git a/cmd/login.go b/cmd/login.go index 6206c8e..dae3270 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -1,6 +1,7 @@ package cmd import ( + "context" gojson "encoding/json" "fmt" "github.com/elasticpath/epcc-cli/config" @@ -273,11 +274,12 @@ var loginCustomer = &cobra.Command{ }, RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() newArgs := make([]string, 0) newArgs = append(newArgs, "customer-token") newArgs = append(newArgs, args...) - body, err := createInternal(newArgs, crud.AutoFillOnCreate) + body, err := createInternal(ctx, newArgs, crud.AutoFillOnCreate) if err != nil { log.Warnf("Login not completed successfully") @@ -303,7 +305,7 @@ var loginCustomer = &cobra.Command{ if customerTokenResponse != nil { // Get the customer so we have aliases where we need the id. - getCustomerBody, err := getInternal([]string{"customer", customerTokenResponse.Data.CustomerId}) + getCustomerBody, err := getInternal(ctx, []string{"customer", customerTokenResponse.Data.CustomerId}) if err != nil { log.Warnf("Could not retrieve customer") diff --git a/cmd/reset-store.go b/cmd/reset-store.go index d346d57..63125ec 100644 --- a/cmd/reset-store.go +++ b/cmd/reset-store.go @@ -24,7 +24,9 @@ var ResetStore = &cobra.Command{ Long: "This command resets a store to it's initial state. There are some limitations to this as for instance orders cannot be deleted, nor can audit entries.", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - storeId, err := getStoreId(args) + ctx := context.Background() + + storeId, err := getStoreId(ctx, args) if err != nil { return fmt.Errorf("could not determine store id: %w", err) } @@ -49,25 +51,25 @@ var ResetStore = &cobra.Command{ // We would also need locking to go faster. // Get customer and account authentication settings to populate the aliases - _, err = getInternal([]string{"customer-authentication-settings"}) + _, err = getInternal(ctx, []string{"customer-authentication-settings"}) if err != nil { errors = append(errors, err.Error()) } - _, err = getInternal([]string{"account-authentication-settings"}) + _, err = getInternal(ctx, []string{"account-authentication-settings"}) if err != nil { errors = append(errors, err.Error()) } - _, err = getInternal([]string{"merchant-realm-mappings"}) + _, err = getInternal(ctx, []string{"merchant-realm-mappings"}) if err != nil { errors = append(errors, err.Error()) } - _, err = getInternal([]string{"authentication-realms"}) + _, err = getInternal(ctx, []string{"authentication-realms"}) if err != nil { errors = append(errors, err.Error()) @@ -101,7 +103,7 @@ var ResetStore = &cobra.Command{ }, } -func getStoreId(args []string) (string, error) { +func getStoreId(ctx context.Context, args []string) (string, error) { resource, ok := resources.GetResourceByName("settings") if !ok { @@ -116,7 +118,7 @@ func getStoreId(args []string) (string, error) { params := url.Values{} - resp, err := httpclient.DoRequest(context.Background(), "GET", resourceURL, params.Encode(), nil) + resp, err := httpclient.DoRequest(ctx, "GET", resourceURL, params.Encode(), nil) defer resp.Body.Close() diff --git a/cmd/root.go b/cmd/root.go index 952f042..b494e60 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,7 @@ import ( "github.com/elasticpath/epcc-cli/external/httpclient" "github.com/elasticpath/epcc-cli/external/logger" "github.com/elasticpath/epcc-cli/external/profiles" + "github.com/elasticpath/epcc-cli/external/shutdown" "github.com/elasticpath/epcc-cli/external/version" log "github.com/sirupsen/logrus" "github.com/thediveo/enumflag" @@ -69,6 +70,7 @@ func init() { RootCmd.PersistentFlags().Uint16VarP(&rateLimit, "rate-limit", "", 10, "Request limit per second") RootCmd.PersistentFlags().BoolVarP(&httpclient.Retry5xx, "retry-5xx", "", false, "Whether we should retry requests with HTTP 5xx response code") RootCmd.PersistentFlags().BoolVarP(&httpclient.Retry429, "retry-429", "", false, "Whether we should retry requests with HTTP 429 response code") + RootCmd.PersistentFlags().BoolVarP(&httpclient.DontLog2xxs, "silence-2xx", "", false, "Whether we should silence HTTP 2xx response code logging") RootCmd.PersistentFlags().Float32VarP(&requestTimeout, "timeout", "", 10, "Request timeout in seconds (fractional values allowed)") @@ -160,6 +162,7 @@ func Execute() { select { case sig := <-sigs: log.Warnf("Shutting down program due to signal [%v]", sig) + shutdown.ShutdownFlag.Store(true) exit = true case <-normalShutdown: } @@ -168,6 +171,9 @@ func Execute() { shutdownHandlerDone <- true }() + log.Infof("Waiting for all outstanding requests to finish") + crud.OutstandingRequestCounter.Wait() + httpclient.LogStats() aliases.FlushAliases() diff --git a/cmd/runbooks.go b/cmd/runbooks.go index d1cf6cd..8566773 100644 --- a/cmd/runbooks.go +++ b/cmd/runbooks.go @@ -9,11 +9,13 @@ import ( "github.com/elasticpath/epcc-cli/external/resources" "github.com/elasticpath/epcc-cli/external/runbooks" _ "github.com/elasticpath/epcc-cli/external/runbooks" + "github.com/elasticpath/epcc-cli/external/shutdown" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" "strconv" "strings" + "sync/atomic" "time" ) @@ -31,6 +33,8 @@ func initRunbookCommands() { runbookGlobalCmd.AddCommand(initRunbookRunCommands()) } +var AbortRunbookExecution = atomic.Bool{} + func initRunbookShowCommands() *cobra.Command { // epcc runbook show @@ -139,6 +143,10 @@ func initRunbookRunCommands() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { numSteps := len(runbookAction.RawCommands) + parentCtx := context.Background() + + ctx, cancelFunc := context.WithCancel(parentCtx) + for stepIdx, rawCmd := range runbookAction.RawCommands { // Create a copy of loop variables @@ -185,7 +193,7 @@ func initRunbookRunCommands() *cobra.Command { case "get": funcs = append(funcs, func() { - body, err := getInternal(rawCmdArguments[2:]) + body, err := getInternal(ctx, rawCmdArguments[2:]) // We print "get" calls because they don't do anything useful (well I guess they populate aliases) json.PrintJson(body) @@ -198,7 +206,7 @@ func initRunbookRunCommands() *cobra.Command { case "delete": funcs = append(funcs, func() { - _, err := deleteInternal(rawCmdArguments[2:]) + _, err := deleteInternal(ctx, rawCmdArguments[2:]) commandResult.error = err @@ -207,7 +215,7 @@ func initRunbookRunCommands() *cobra.Command { }) case "delete-all": funcs = append(funcs, func() { - err := deleteAllInternal(rawCmdArguments[2:]) + err := deleteAllInternal(ctx, rawCmdArguments[2:]) commandResult.error = err resultChan <- commandResult }) @@ -217,9 +225,9 @@ func initRunbookRunCommands() *cobra.Command { var err error = nil if len(rawCmdArguments) >= 3 && rawCmdArguments[2] == "--auto-fill" { - _, err = createInternal(rawCmdArguments[3:], true) + _, err = createInternal(ctx, rawCmdArguments[3:], true) } else { - _, err = createInternal(rawCmdArguments[2:], false) + _, err = createInternal(ctx, rawCmdArguments[2:], false) } commandResult.error = err @@ -227,7 +235,7 @@ func initRunbookRunCommands() *cobra.Command { }) case "update": funcs = append(funcs, func() { - _, err := updateInternal(rawCmdArguments[2:]) + _, err := updateInternal(ctx, rawCmdArguments[2:]) commandResult.error = err resultChan <- commandResult @@ -255,7 +263,13 @@ func initRunbookRunCommands() *cobra.Command { // Start processing all the functions go func() { - for _, fn := range funcs { + for idx, fn := range funcs { + if shutdown.ShutdownFlag.Load() { + log.Infof("Aborting runbook execution, after %d scheduled executions", idx) + cancelFunc() + break + } + fn := fn semaphore.Acquire(context.TODO(), 1) go func() { @@ -269,11 +283,16 @@ func initRunbookRunCommands() *cobra.Command { for i := 0; i < len(funcs); i++ { select { case result := <-resultChan: - if result.error != nil { - log.Warnf("(Step %d/%d Command %d/%d) %v", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs), fmt.Errorf("error processing command [%s], %w", result.commandLine, err)) - errorCount++ + if !shutdown.ShutdownFlag.Load() { + if result.error != nil { + log.Warnf("(Step %d/%d Command %d/%d) %v", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs), fmt.Errorf("error processing command [%s], %w", result.commandLine, result.error)) + errorCount++ + } else { + log.Debugf("(Step %d/%d Command %d/%d) finished successfully ", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs)) + } } else { - log.Debugf("(Step %d/%d Command %d/%d) finished successfully ", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs)) + log.Tracef("Shutdown flag enabled, completion result %v", result) + cancelFunc() } case <-time.After(time.Duration(*execTimeoutInSeconds) * time.Second): return fmt.Errorf("timeout of %d seconds reached, only %d of %d commands finished of step %d/%d", *execTimeoutInSeconds, i+1, len(funcs), stepIdx+1, numSteps) @@ -289,6 +308,7 @@ func initRunbookRunCommands() *cobra.Command { return fmt.Errorf("error occurred while processing script aborting") } } + defer cancelFunc() return nil }, } diff --git a/cmd/update.go b/cmd/update.go index 80d9f75..3058dde 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -22,7 +22,7 @@ var update = &cobra.Command{ Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - body, err := updateInternal(args) + body, err := updateInternal(context.Background(), args) if err != nil { return err @@ -90,7 +90,10 @@ var update = &cobra.Command{ }, } -func updateInternal(args []string) (string, error) { +func updateInternal(ctx context.Context, args []string) (string, error) { + crud.OutstandingRequestCounter.Add(1) + defer crud.OutstandingRequestCounter.Done() + // Find Resource resource, ok := resources.GetResourceByName(args[0]) if !ok { @@ -137,7 +140,7 @@ func updateInternal(args []string) (string, error) { } // Submit request - resp, err := httpclient.DoRequest(context.TODO(), "PUT", resourceURL, params.Encode(), strings.NewReader(body)) + resp, err := httpclient.DoRequest(ctx, "PUT", resourceURL, params.Encode(), strings.NewReader(body)) if err != nil { return "", fmt.Errorf("got error %s", err.Error()) } else if resp == nil { diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b6254fd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '2.2' +services: + wiremock: + build: + context: ./wiremock + volumes: + - ./wiremock/mappings:/home/wiremock/mappings/ + - ./wiremock/files:/home/wiremock/__files/ + ports: + - 44400:8080 + healthcheck: + test: | + curl -f http://localhost:8080 diff --git a/external/aliases/aliases.go b/external/aliases/aliases.go index f9ad256..9cae507 100644 --- a/external/aliases/aliases.go +++ b/external/aliases/aliases.go @@ -20,12 +20,15 @@ import ( var mutex = &sync.RWMutex{} var aliasDirectoryOverride = "" -var aliases = map[string]map[string]*id.IdableAttributes{} + +var typeToAliasNameToIdMap = map[string]map[string]*id.IdableAttributes{} var dirtyAliases = map[string]bool{} var SkipAliasProcessing = false +var typeToIdToAliasNamesMap = map[string]map[string]map[string]bool{} + func ClearAllAliasesForJsonApiType(jsonApiType string) error { ClearCache(jsonApiType) @@ -65,8 +68,7 @@ func GetAliasesForJsonApiTypeAndAlternates(jsonApiType string, alternateJsonApiT func getAliasesForSingleJsonApiType(jsonApiType string) map[string]*id.IdableAttributes { mutex.RLock() - - aliasMap, ok := aliases[jsonApiType] + aliasMap, ok := typeToAliasNameToIdMap[jsonApiType] if !ok { mutex.RUnlock() @@ -84,7 +86,21 @@ func getAliasesForSingleJsonApiType(jsonApiType string) map[string]*id.IdableAtt } err = yaml.Unmarshal(data, aliasMap) - aliases[jsonApiType] = aliasMap + typeToAliasNameToIdMap[jsonApiType] = aliasMap + + aliasForTypeAndIdMap := map[string]map[string]bool{} + + for aliasName, aliasAttributes := range aliasMap { + + if _, ok := aliasForTypeAndIdMap[aliasAttributes.Id]; !ok { + aliasForTypeAndIdMap[aliasAttributes.Id] = map[string]bool{} + } + + aliasForTypeAndIdMap[aliasAttributes.Id][aliasName] = true + } + + typeToIdToAliasNamesMap[jsonApiType] = aliasForTypeAndIdMap + if err != nil { log.Debugf("Could not unmarshall existing file %s, error %s", data, err) } else { @@ -140,18 +156,27 @@ func SaveAliasesForResources(jsonTxt string) { for resourceType, foundAliases := range results { saveAliasesForResource(resourceType, foundAliases) - log.Tracef("Number of resources for type [%s] is now %d and value is %v", resourceType, len(aliases[resourceType]), aliases[resourceType]) } } func DeleteAliasesById(idStr string, jsonApiType string) { - modifyAliases(jsonApiType, func(m map[string]*id.IdableAttributes) { - for key, value := range m { - if value.Id == idStr { - delete(m, key) + modifyAliases(jsonApiType, func(allAliasesForType map[string]*id.IdableAttributes, aliasesById map[string]map[string]bool) { + if aliasesForId, ok := aliasesById[idStr]; ok { + for aliasForId := range aliasesForId { + if aliasForType, ok := allAliasesForType[aliasForId]; ok { + if aliasForType.Id != idStr { + log.Warnf("Trying to delete all aliases for id %v, including %v, however this alias points to id %v", idStr, aliasesForId, aliasForType.Id) + } else { + delete(allAliasesForType, aliasForId) + } + } + } + + delete(aliasesById, idStr) } + }, ) } @@ -178,13 +203,13 @@ func getAliasFileForJsonApiType(profileDirectory string, resourceType string) st return aliasFile } -func modifyAliases(jsonApiType string, fn func(map[string]*id.IdableAttributes)) { - aliasMap := getAliasesForSingleJsonApiType(jsonApiType) +func modifyAliases(jsonApiType string, fn func(map[string]*id.IdableAttributes, map[string]map[string]bool)) { + aliasesToIdMapForType := getAliasesForSingleJsonApiType(jsonApiType) mutex.Lock() defer mutex.Unlock() - fn(aliasMap) + fn(aliasesToIdMapForType, typeToIdToAliasNamesMap[jsonApiType]) dirtyAliases[jsonApiType] = true } @@ -192,27 +217,53 @@ func modifyAliases(jsonApiType string, fn func(map[string]*id.IdableAttributes)) // This function saves all the aliases for a specific resource. func saveAliasesForResource(jsonApiType string, newAliases map[string]*id.IdableAttributes) { - modifyAliases(jsonApiType, func(aliasMap map[string]*id.IdableAttributes) { - + modifyAliases(jsonApiType, func(aliasMap map[string]*id.IdableAttributes, aliasesById map[string]map[string]bool) { // Aliases have the format KEY=VALUE and this maps to an ID. // This code checks for where two aliases have the same KEY and same ID, and replaces the old value, with the new one. // This happens in cases where we store a name like "name=John_Smith" and then the user renames it to "name=Jane_Doe". // The old alias for the same id name=John_Smith should be removed. for newAliasName, newAliasReferencedId := range newAliases { newAliasKeyName := strings.Split(newAliasName, "=")[0] - for oldAliasName, oldAliasReferencedId := range aliasMap { + + // This step repairs any aliases that map to the new alias id + // (e.g., if we have an alias name=jane, and we are going to set name=john, then name=jane should be deleted) + for oldAliasName := range aliasesById[newAliasReferencedId.Id] { oldAliasKeyName := strings.Split(oldAliasName, "=")[0] - oldAliasValue := strings.Split(oldAliasName, "=")[1] - if oldAliasKeyName == newAliasKeyName && oldAliasReferencedId.Id == newAliasReferencedId.Id { + if oldAliasKeyName == "last_read" { + // Leave these aliases alone + continue + } + + if oldAliasKeyName == newAliasKeyName && oldAliasName != newAliasName { + + if aliases, ok := aliasMap[oldAliasName]; ok { + if aliases.Id != newAliasReferencedId.Id { + log.Warnf("Trying to delete alias %v, but it points to id %v not %v, this is a bug", oldAliasName, aliases.Id, newAliasReferencedId.Id) + } else { + delete(aliasMap, oldAliasName) + } + } - delete(aliasMap, oldAliasKeyName+"="+oldAliasValue) } } } for key, value := range newAliases { + if oldAliasData, ok := aliasMap[key]; ok { + if oldAliasData.Id != value.Id { + oldAliasId := oldAliasData.Id + delete(aliasesById[oldAliasId], key) + } + + } + aliasMap[key] = value + + if _, ok := aliasesById[value.Id]; !ok { + aliasesById[value.Id] = map[string]bool{} + } + aliasesById[value.Id][key] = true } }) } @@ -380,15 +431,17 @@ func InitializeAliasDirectoryForTesting() { func ClearAllCaches() { mutex.Lock() - aliases = map[string]map[string]*id.IdableAttributes{} + typeToAliasNameToIdMap = map[string]map[string]*id.IdableAttributes{} + typeToIdToAliasNamesMap = map[string]map[string]map[string]bool{} dirtyAliases = map[string]bool{} mutex.Unlock() } func ClearCache(jsonApiType string) { mutex.Lock() - delete(aliases, jsonApiType) + delete(typeToAliasNameToIdMap, jsonApiType) delete(dirtyAliases, jsonApiType) + delete(typeToIdToAliasNamesMap, jsonApiType) mutex.Unlock() } @@ -419,13 +472,13 @@ func SyncAliases() int { for jsonApiType, val := range dirtyAliases { if val == false { - log.Errorf("Not expecting a dirty alias to be false, should either exist or not, this is a bug, for type %s", jsonApiType) + log.Warnf("Not expecting a dirty alias to be false, should either exist or not, this is a bug, for type %s", jsonApiType) continue } aliasFile := path.Clean(getAliasFileForJsonApiType(getAliasDataDirectory(), jsonApiType)) - aliasesForType := aliases[jsonApiType] + aliasesForType := typeToAliasNameToIdMap[jsonApiType] // We will write to a temp file and then rename, to prevent data loss. rename's in the same folder are likely atomic in most settings. // Although we should probably sync on the file as well, that might be too much overhead, and I was too lazy to rewrite this @@ -457,6 +510,7 @@ func SyncAliases() int { } log.Debugf("Syncing aliases to disk, %d files changed", syncedFiles) + return syncedFiles } diff --git a/external/aliases/aliases_test.go b/external/aliases/aliases_test.go index b6d0d34..cbe8a8e 100644 --- a/external/aliases/aliases_test.go +++ b/external/aliases/aliases_test.go @@ -15,7 +15,7 @@ func TestSavedAliasIsReturnedInAllAliasesForSingleResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -32,7 +32,7 @@ func TestSavedAliasIsReturnedInAllAliasesForSingleResponse(t *testing.T) { aliases := GetAliasesForJsonApiTypeAndAlternates("foo", []string{}) // Verification - require.Len(t, aliases, 2, "There should be %d aliases in map not %d", 2, len(aliases)) + require.Len(t, aliases, 2, "There should be %d typeToAliasNameToIdMap in map not %d", 2, len(aliases)) require.Contains(t, aliases, "id=123") require.Equal(t, "123", aliases["id=123"].Id) @@ -46,7 +46,7 @@ func TestSavedAliasAppendsAndPreservesPreviousUnrelatedAliases(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -73,7 +73,7 @@ func TestSavedAliasAppendsAndPreservesPreviousUnrelatedAliases(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "id=123") require.Equal(t, "123", aliases["id=123"].Id) @@ -90,7 +90,7 @@ func TestSavedAliasIsReplacedWhenNewEntityHasTheSameAttributeValue(t *testing.T) // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -119,7 +119,7 @@ func TestSavedAliasIsReplacedWhenNewEntityHasTheSameAttributeValue(t *testing.T) // Verification - require.Len(t, aliases, 4, "There should be %d aliases in map not %d", 4, len(aliases)) + require.Len(t, aliases, 4, "There should be %d typeToAliasNameToIdMap in map not %d", 4, len(aliases)) require.Contains(t, aliases, "id=123") require.Equal(t, "123", aliases["id=123"].Id) @@ -139,7 +139,7 @@ func TestSavedAliasIsReplacedWhenSameEntityHasANewValue(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -168,7 +168,7 @@ func TestSavedAliasIsReplacedWhenSameEntityHasANewValue(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "id=123") require.Equal(t, "123", aliases["id=123"].Id) @@ -179,12 +179,117 @@ func TestSavedAliasIsReplacedWhenSameEntityHasANewValue(t *testing.T) { require.Equal(t, "123", aliases["last_read=entity"].Id) } +func TestThatLastReadAliasesAreNotReplacedWhenSeenInADifferentContext(t *testing.T) { + + // Fixture Setup + err := ClearAllAliases() + if err != nil { + t.Fatalf("Could not clear typeToAliasNameToIdMap") + } + + // Execute SUT + SaveAliasesForResources( + // language=JSON + ` +{ + "data": { + "id": "123", + "type": "foo", + "name": "Alpha" + } +}`) + SaveAliasesForResources( + // language=JSON + ` +{ + "data": [{ + "id": "123", + "type": "foo", + "name":"Beta" + }] +}`) + + aliases := GetAliasesForJsonApiTypeAndAlternates("foo", []string{}) + + // Verification + + require.Len(t, aliases, 4, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) + + require.Contains(t, aliases, "id=123") + require.Equal(t, "123", aliases["id=123"].Id) + + require.Contains(t, aliases, "name=Beta") + require.Equal(t, "123", aliases["name=Beta"].Id) + require.Contains(t, aliases, "last_read=entity") + require.Equal(t, "123", aliases["last_read=entity"].Id) + require.Contains(t, aliases, "last_read=array[0]") + require.Equal(t, "123", aliases["last_read=array[0]"].Id) +} + func TestDeleteAliasByIdDeletesAnAlias(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") + } + SaveAliasesForResources( + // language=JSON + ` +{ + "data": { + "id": "123", + "type": "foo", + "name": "Steve", + "sku": "456", + "slug": "foo-123" + } +}`) + SaveAliasesForResources( + // language=JSON + ` +{ + "data": { + "id": "456", + "type": "foo", + "name": "Steve", + "sku": "456", + "slug": "foo-456" + } +}`) + + // Execute SUT + + DeleteAliasesById("123", "foo") + + aliases := GetAliasesForJsonApiTypeAndAlternates("foo", []string{}) + + // Verification + + require.Len(t, aliases, 5, "There should be %d typeToAliasNameToIdMap in map not %d", 5, len(aliases)) + + require.Contains(t, aliases, "id=456") + require.Equal(t, "456", aliases["id=456"].Id) + + require.Contains(t, aliases, "last_read=entity") + require.Equal(t, "456", aliases["last_read=entity"].Id) + + require.Contains(t, aliases, "name=Steve") + require.Equal(t, "456", aliases["name=Steve"].Id) + + require.Contains(t, aliases, "sku=456") + require.Equal(t, "456", aliases["sku=456"].Id) + + require.Contains(t, aliases, "slug=foo-456") + require.Equal(t, "456", aliases["slug=foo-456"].Id) +} + +func TestDeleteAliasByIdDeletesAnAliasOnly(t *testing.T) { + + // Fixture Setup + err := ClearAllAliases() + if err != nil { + t.Fatalf("Could not clear typeToAliasNameToIdMap") } SaveAliasesForResources( // language=JSON @@ -213,7 +318,7 @@ func TestDeleteAliasByIdDeletesAnAlias(t *testing.T) { // Verification - require.Len(t, aliases, 2, "There should be %d aliases in map not %d", 2, len(aliases)) + require.Len(t, aliases, 2, "There should be %d typeToAliasNameToIdMap in map not %d", 2, len(aliases)) require.Contains(t, aliases, "id=456") require.Equal(t, "456", aliases["id=456"].Id) @@ -227,7 +332,7 @@ func TestAllAliasesAreReturnedInAllAliasesForArrayResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -250,7 +355,7 @@ func TestAllAliasesAreReturnedInAllAliasesForArrayResponse(t *testing.T) { // Verification - require.Len(t, aliases, 4, "There should be %d aliases in map not %d", 4, len(aliases)) + require.Len(t, aliases, 4, "There should be %d typeToAliasNameToIdMap in map not %d", 4, len(aliases)) require.Contains(t, aliases, "id=123") require.Equal(t, "123", aliases["id=123"].Id) @@ -270,7 +375,7 @@ func TestSavedAliasIsReturnedForAnEmailInLegacyObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -289,7 +394,7 @@ func TestSavedAliasIsReturnedForAnEmailInLegacyObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "email=test@test.com") require.Equal(t, "123", aliases["email=test@test.com"].Id) @@ -306,7 +411,7 @@ func TestSavedAliasIsReturnedForAnSkuInLegacyObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -325,7 +430,7 @@ func TestSavedAliasIsReturnedForAnSkuInLegacyObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "sku=test") require.Equal(t, "123", aliases["sku=test"].Id) @@ -351,7 +456,7 @@ func TestSavedAliasIsReturnedForAnCodeInLegacyObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -370,7 +475,7 @@ func TestSavedAliasIsReturnedForAnCodeInLegacyObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "code=hello") require.Equal(t, "123", aliases["code=hello"].Id) @@ -396,7 +501,7 @@ func TestSavedAliasIsReturnedForASlugInLegacyObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -415,7 +520,7 @@ func TestSavedAliasIsReturnedForASlugInLegacyObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "slug=test") require.Equal(t, "123", aliases["slug=test"].Id) @@ -441,7 +546,7 @@ func TestSavedAliasIsReturnedForANameInLegacyObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -460,7 +565,7 @@ func TestSavedAliasIsReturnedForANameInLegacyObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "name=Test_Testerson") require.Equal(t, "123", aliases["name=Test_Testerson"].Id) @@ -478,7 +583,7 @@ func TestSavedAliasIsReturnedForAnEmailInComplaintObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -499,7 +604,7 @@ func TestSavedAliasIsReturnedForAnEmailInComplaintObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "email=test@test.com") require.Equal(t, "123", aliases["email=test@test.com"].Id) @@ -517,7 +622,7 @@ func TestSavedAliasIsReturnedForAnSkuInComplaintObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -538,7 +643,7 @@ func TestSavedAliasIsReturnedForAnSkuInComplaintObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "sku=test") require.Equal(t, "123", aliases["sku=test"].Id) @@ -564,7 +669,7 @@ func TestSavedAliasIsReturnedForASlugInComplaintObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -585,7 +690,7 @@ func TestSavedAliasIsReturnedForASlugInComplaintObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "slug=test") require.Equal(t, "123", aliases["slug=test"].Id) @@ -611,7 +716,7 @@ func TestSavedAliasIsReturnedForANameInComplaintObjectResponse(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -632,7 +737,7 @@ func TestSavedAliasIsReturnedForANameInComplaintObjectResponse(t *testing.T) { // Verification - require.Len(t, aliases, 3, "There should be %d aliases in map not %d", 3, len(aliases)) + require.Len(t, aliases, 3, "There should be %d typeToAliasNameToIdMap in map not %d", 3, len(aliases)) require.Contains(t, aliases, "name=Test_Testerson") require.Equal(t, "123", aliases["name=Test_Testerson"].Id) @@ -648,7 +753,7 @@ func TestSavedAliasIsReturnedForANameInComplaintObjectResponse(t *testing.T) { func TestSavedAliasIsReturnedForARelationshipObjectInArrayResponse(t *testing.T) { err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -686,7 +791,7 @@ func TestSavedAliasIsReturnedForARelationshipObjectInArrayResponse(t *testing.T) aliases := GetAliasesForJsonApiTypeAndAlternates("bar", []string{}) - require.Len(t, aliases, 8, "There should be %d aliases in map not %d", 8, len(aliases)) + require.Len(t, aliases, 8, "There should be %d typeToAliasNameToIdMap in map not %d", 8, len(aliases)) require.Contains(t, aliases, "id=abc") require.Equal(t, "abc", aliases["id=abc"].Id) @@ -716,7 +821,7 @@ func TestSavedAliasIsReturnedForARelationshipObjectInArrayResponse(t *testing.T) func TestSavedAliasIsReturnedForARelationshipObjectInSingleResponse(t *testing.T) { err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -742,7 +847,7 @@ func TestSavedAliasIsReturnedForARelationshipObjectInSingleResponse(t *testing.T aliases := GetAliasesForJsonApiTypeAndAlternates("bar", []string{}) - require.Len(t, aliases, 4, "There should be %d aliases in map not %d", 4, len(aliases)) + require.Len(t, aliases, 4, "There should be %d typeToAliasNameToIdMap in map not %d", 4, len(aliases)) require.Contains(t, aliases, "id=456") require.Equal(t, "456", aliases["id=456"].Id) @@ -762,7 +867,7 @@ func TestResolveAliasValuesReturnsAliasForMatchingValue(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -790,7 +895,7 @@ func TestResolveAliasValuesReturnsAliasSkuForMatchingValue(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -820,7 +925,7 @@ func TestResolveAliasValuesReturnsAliasCodeForMatchingValue(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -850,7 +955,7 @@ func TestResolveAliasValuesReturnsAliasSlugForMatchingValue(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -880,7 +985,7 @@ func TestResolveAliasValuesReturnsAliasForMatchingValueAsAlternateType(t *testin // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -908,7 +1013,7 @@ func TestResolveAliasValuesReturnsAliasForTypeAndNotAlternateTypeWhenCollisionOc // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -944,7 +1049,7 @@ func TestResolveAliasValuesReturnsRequestForUnMatchingValue(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -974,7 +1079,7 @@ func TestResolveAliasValuesReturnsRequestForUnMatchingValueAndType(t *testing.T) // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } // Execute SUT @@ -1003,7 +1108,7 @@ func TestClearAllAliasesClearsAllAliases(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } SaveAliasesForResources( @@ -1029,7 +1134,7 @@ func TestClearAllAliasesClearsAllAliases(t *testing.T) { // Execute SUT err = ClearAllAliases() if err != nil { - t.Errorf("Couldn't clear aliases %v", err) + t.Errorf("Couldn't clear typeToAliasNameToIdMap %v", err) return } @@ -1054,7 +1159,7 @@ func TestClearAllAliasesForJsonTypeOnlyClearsJsonType(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } SaveAliasesForResources( @@ -1081,7 +1186,7 @@ func TestClearAllAliasesForJsonTypeOnlyClearsJsonType(t *testing.T) { err = ClearAllAliasesForJsonApiType("foo") if err != nil { - t.Errorf("Couldn't clear aliases %v", err) + t.Errorf("Couldn't clear typeToAliasNameToIdMap %v", err) return } @@ -1105,7 +1210,7 @@ func TestThatCorruptAliasFileDoesntCrashProgramWhenReadingAliases(t *testing.T) // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } SaveAliasesForResources( @@ -1137,7 +1242,7 @@ func TestThatCorruptAliasFileDoesntCrashProgramWhenReadingAliases(t *testing.T) aliases := GetAliasesForJsonApiTypeAndAlternates("foo", []string{}) // Verification - require.Len(t, aliases, 0, "There should be %d aliases in map not %d", 0, len(aliases)) + require.Len(t, aliases, 0, "There should be %d typeToAliasNameToIdMap in map not %d", 0, len(aliases)) } @@ -1145,7 +1250,7 @@ func TestThatCorruptAliasFileDoesntCrashProgramWhenSavingAliases(t *testing.T) { // Fixture Setup err := ClearAllAliases() if err != nil { - t.Fatalf("Could not clear aliases") + t.Fatalf("Could not clear typeToAliasNameToIdMap") } SaveAliasesForResources( @@ -1186,7 +1291,7 @@ func TestThatCorruptAliasFileDoesntCrashProgramWhenSavingAliases(t *testing.T) { aliases := GetAliasesForJsonApiTypeAndAlternates("foo", []string{}) // Verification - require.Len(t, aliases, 2, "There should be %d aliases in map not %d", 2, len(aliases)) + require.Len(t, aliases, 2, "There should be %d typeToAliasNameToIdMap in map not %d", 2, len(aliases)) require.Contains(t, aliases, "id=456") require.Equal(t, "456", aliases["id=456"].Id) diff --git a/external/crud/outstanding_request_counter.go b/external/crud/outstanding_request_counter.go new file mode 100644 index 0000000..d3e6e41 --- /dev/null +++ b/external/crud/outstanding_request_counter.go @@ -0,0 +1,5 @@ +package crud + +import "sync" + +var OutstandingRequestCounter = sync.WaitGroup{} diff --git a/external/httpclient/httpclient.go b/external/httpclient/httpclient.go index 5475647..a9b9121 100644 --- a/external/httpclient/httpclient.go +++ b/external/httpclient/httpclient.go @@ -8,6 +8,7 @@ import ( "github.com/elasticpath/epcc-cli/external/authentication" "github.com/elasticpath/epcc-cli/external/json" "github.com/elasticpath/epcc-cli/external/profiles" + "github.com/elasticpath/epcc-cli/external/shutdown" "github.com/elasticpath/epcc-cli/external/version" log "github.com/sirupsen/logrus" "golang.org/x/time/rate" @@ -29,6 +30,8 @@ const EnvNameHttpPrefix = "EPCC_CLI_HTTP_HEADER_" var httpHeaders = map[string]string{} +var DontLog2xxs = false + var stats = struct { totalRateLimitedTimeInMs int64 totalHttpRequestProcessingTime int64 @@ -56,6 +59,29 @@ func init() { } } } + + go func() { + lastTotalRequests := uint64(0) + + for { + time.Sleep(15 * time.Second) + + statsLock.Lock() + + deltaRequests := stats.totalRequests - lastTotalRequests + lastTotalRequests = stats.totalRequests + statsLock.Unlock() + + if shutdown.ShutdownFlag.Load() { + break + } + + if deltaRequests > 0 { + log.Infof("Total requests %d, requests in past 15 seconds %d, latest %d requests per second.", lastTotalRequests, deltaRequests, deltaRequests/15.0) + } + + } + }() } var Limit *rate.Limiter = nil @@ -106,6 +132,11 @@ var noApiEndpointUrlWarningMessageLogged = false // DoRequest makes a html request to the EPCC API and handles the response. func doRequestInternal(ctx context.Context, method string, contentType string, path string, query string, payload io.Reader) (response *http.Response, error error) { + + if shutdown.ShutdownFlag.Load() { + return nil, fmt.Errorf("Shutting down") + } + reqURL, err := url.Parse(config.Envs.EPCC_API_BASE_URL) if err != nil { return nil, err @@ -271,7 +302,9 @@ func doRequestInternal(ctx context.Context, method string, contentType string, p logf("(%0.4d) %s %s%s ==> %s %s%s", requestNumber, req.Method, getUrl(reqURL), requestHeaders, resp.Proto, resp.Status, responseHeaders) } } else { - log.Infof("(%0.4d) %s %s ==> %s %s", requestNumber, method, getUrl(reqURL), resp.Proto, resp.Status) + if resp.StatusCode >= 300 || !DontLog2xxs { + log.Infof("(%0.4d) %s %s ==> %s %s", requestNumber, method, getUrl(reqURL), resp.Proto, resp.Status) + } } dumpRes, err := httputil.DumpResponse(resp, true) diff --git a/external/id/idable_attributes.go b/external/id/idable_attributes.go index fb0ce79..57e0d12 100644 --- a/external/id/idable_attributes.go +++ b/external/id/idable_attributes.go @@ -2,7 +2,7 @@ package id type IdableAttributes struct { Id string `yaml:"id"` - Slug string `yaml:"slug"` - Sku string `yaml:"sku"` - Code string `yaml:"code"` + Slug string `yaml:"slug,omitempty"` + Sku string `yaml:"sku,omitempty"` + Code string `yaml:"code,omitempty"` } diff --git a/external/shutdown/shutdown_flag.go b/external/shutdown/shutdown_flag.go new file mode 100644 index 0000000..f7aa5e1 --- /dev/null +++ b/external/shutdown/shutdown_flag.go @@ -0,0 +1,5 @@ +package shutdown + +import "sync/atomic" + +var ShutdownFlag = atomic.Bool{} diff --git a/wiremock/Dockerfile b/wiremock/Dockerfile new file mode 100644 index 0000000..6521380 --- /dev/null +++ b/wiremock/Dockerfile @@ -0,0 +1,6 @@ +FROM rodolpheche/wiremock:2.31.0 +RUN apt-get update && apt-get install -y reflex +ADD https://repo1.maven.org/maven2/com/opentable/wiremock-body-transformer/1.1.6/wiremock-body-transformer-1.1.6.jar /var/wiremock/extensions/ + +ADD reflex-entrypoint.sh / +ENTRYPOINT /reflex-entrypoint.sh diff --git a/wiremock/files/generic-entity-get-body.json b/wiremock/files/generic-entity-get-body.json new file mode 100644 index 0000000..efdd985 --- /dev/null +++ b/wiremock/files/generic-entity-get-body.json @@ -0,0 +1,6 @@ +{ + "data": { + "id": "$(id)", + "type": "$(type)" + } +} diff --git a/wiremock/files/generic-entity-get-relationships-body.json b/wiremock/files/generic-entity-get-relationships-body.json new file mode 100644 index 0000000..15a6609 --- /dev/null +++ b/wiremock/files/generic-entity-get-relationships-body.json @@ -0,0 +1,6 @@ +{ + "data": { + "id": "{{randomValue type='UUID'}}", + "type": "$(type)" + } +} diff --git a/wiremock/files/generic-entity-post-body.json b/wiremock/files/generic-entity-post-body.json new file mode 100644 index 0000000..15a6609 --- /dev/null +++ b/wiremock/files/generic-entity-post-body.json @@ -0,0 +1,6 @@ +{ + "data": { + "id": "{{randomValue type='UUID'}}", + "type": "$(type)" + } +} diff --git a/wiremock/files/generic-list-get-with-one-element-body.json b/wiremock/files/generic-list-get-with-one-element-body.json new file mode 100644 index 0000000..68c557e --- /dev/null +++ b/wiremock/files/generic-list-get-with-one-element-body.json @@ -0,0 +1,17 @@ +{ + "data": [{ + "id": "{{randomValue type='UUID'}}", + "type": "$(type)" + }], + "meta": { + "page": { + "limit": 10, + "offset": 0, + "current": 1, + "total": 1 + }, + "results": { + "total": 1 + } + } +} diff --git a/wiremock/files/v2-customers-post-body.json b/wiremock/files/v2-customers-post-body.json new file mode 100644 index 0000000..371ed21 --- /dev/null +++ b/wiremock/files/v2-customers-post-body.json @@ -0,0 +1,10 @@ +{ + "data": { + "id": "{{randomValue type='UUID'}}", + "type": "customer", + "name": "{{jsonPath request.body '$.data.name'}}", + "email": "{{jsonPath request.body '$.data.email'}}", + "password": true + } +} + diff --git a/wiremock/mappings/generic/generic-entity-delete.json b/wiremock/mappings/generic/generic-entity-delete.json new file mode 100644 index 0000000..bd81506 --- /dev/null +++ b/wiremock/mappings/generic/generic-entity-delete.json @@ -0,0 +1,11 @@ +{ + "priority": 100, + "request": { + "method": "DELETE", + "urlPattern": "/v2.*[a-zA-Z-_]+/[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}/?$" + }, + "response": { + "status":204, + "statusMessage": "No Content" + } +} diff --git a/wiremock/mappings/generic/generic-entity-get-relationships.json b/wiremock/mappings/generic/generic-entity-get-relationships.json new file mode 100644 index 0000000..8757b2e --- /dev/null +++ b/wiremock/mappings/generic/generic-entity-get-relationships.json @@ -0,0 +1,18 @@ +{ + "priority": 100, + "request": { + "method": "GET", + "urlPattern": "/v2(/[a-zA-Z-_]*)*/[a-zA-Z-_]+/[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}/relationships/[a-zA-Z-_]+/?$" + }, + "response": { + "status":200, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "generic-entity-get-relationships-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/relationships/(?[^/]+)?$" + } + } +} diff --git a/wiremock/mappings/generic/generic-entity-get.json b/wiremock/mappings/generic/generic-entity-get.json new file mode 100644 index 0000000..4102272 --- /dev/null +++ b/wiremock/mappings/generic/generic-entity-get.json @@ -0,0 +1,18 @@ +{ + "priority": 100, + "request": { + "method": "GET", + "urlPattern": "/v2(/[a-zA-Z-_]*)*/[a-zA-Z-_]+/[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}/?$" + }, + "response": { + "status":200, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "generic-entity-get-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/(?[^/]+)/(?[^/]+)/?$" + } + } +} diff --git a/wiremock/mappings/generic/generic-entity-post.json b/wiremock/mappings/generic/generic-entity-post.json new file mode 100644 index 0000000..4da6bbf --- /dev/null +++ b/wiremock/mappings/generic/generic-entity-post.json @@ -0,0 +1,18 @@ +{ + "priority": "1000", + "request": { + "method": "POST", + "urlPattern": "/v2(/[a-zA-Z-_]*)*/[a-zA-Z-_]+/?$" + }, + "response": { + "status": 201, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "generic-entity-post-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/(?[^/]+)/?$" + } + } +} diff --git a/wiremock/mappings/generic/generic-entity-put-relationships.json b/wiremock/mappings/generic/generic-entity-put-relationships.json new file mode 100644 index 0000000..21ad7a5 --- /dev/null +++ b/wiremock/mappings/generic/generic-entity-put-relationships.json @@ -0,0 +1,18 @@ +{ + "priority": 100, + "request": { + "method": "PUT", + "urlPattern": "/v2(/[a-zA-Z-_]*)*/[a-zA-Z-_]+/[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}/relationships/[a-zA-Z-_]+/?$" + }, + "response": { + "status":200, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "generic-entity-get-relationships-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/relationships/(?[^/]+)?$" + } + } +} diff --git a/wiremock/mappings/generic/generic-entity-put.json b/wiremock/mappings/generic/generic-entity-put.json new file mode 100644 index 0000000..22cee7a --- /dev/null +++ b/wiremock/mappings/generic/generic-entity-put.json @@ -0,0 +1,18 @@ +{ + "priority": 100, + "request": { + "method": "PUT", + "urlPattern": "/v2(/[a-zA-Z-_]*)*/[a-zA-Z-_]+/[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}/?$" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "generic-entity-get-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/(?[^/]+)/(?[^/]+)/?$" + } + } +} diff --git a/wiremock/mappings/generic/generic-list-get.json b/wiremock/mappings/generic/generic-list-get.json new file mode 100644 index 0000000..c8a068d --- /dev/null +++ b/wiremock/mappings/generic/generic-list-get.json @@ -0,0 +1,18 @@ +{ + "priority": 100, + "request": { + "method": "GET", + "urlPathPattern": "/v2(/[a-zA-Z-_]*)*/[a-zA-Z-_]+/?$" + }, + "response": { + "status":200, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "generic-list-get-with-one-element-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/(?[^?/]+)/?(\\?.*)?$" + } + } +} diff --git a/wiremock/mappings/oauth/access_token/post.json b/wiremock/mappings/oauth/access_token/post.json new file mode 100644 index 0000000..69acd7a --- /dev/null +++ b/wiremock/mappings/oauth/access_token/post.json @@ -0,0 +1,20 @@ +{ + "priority": "1", + "request": { + "method": "POST", + "urlPattern": "/oauth/access_token$" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "expires": 1999999999, + "identifier": "client_credentials", + "expires_in": 100000000, + "access_token": "0011223344556677889900112233445566778899", + "token_type": "Bearer" + } + } +} diff --git a/wiremock/mappings/v2/customers/post.json b/wiremock/mappings/v2/customers/post.json new file mode 100644 index 0000000..cf3f383 --- /dev/null +++ b/wiremock/mappings/v2/customers/post.json @@ -0,0 +1,23 @@ +{ + "priority": "1", + "request": { + "method": "POST", + "urlPattern": "/v2/customers/?$" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "bodyFileName": "v2-customers-post-body.json", + "transformers": ["response-template", "body-transformer"], + "transformerParameters": { + "urlRegex": ".*/(?[^/]+)/?$" + }, + "delayDistribution": { + "type": "lognormal", + "median": 20, + "sigma": 0.4 + } + } +} diff --git a/wiremock/mappings/v2/settings/get.json b/wiremock/mappings/v2/settings/get.json new file mode 100644 index 0000000..1e93de0 --- /dev/null +++ b/wiremock/mappings/v2/settings/get.json @@ -0,0 +1,23 @@ +{ + "priority": 1, + "request": { + "method": "GET", + "urlPattern": "/v2/settings/?" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "data": { + "type": "settings", + "id": "00000000-0000-4000-b000-000000000000", + "page_length": 120, + "list_child_products": true, + "additional_languages": [], + "calculation_method": "simple" + } + } + } +} diff --git a/wiremock/reflex-entrypoint.sh b/wiremock/reflex-entrypoint.sh new file mode 100755 index 0000000..fc9f0e3 --- /dev/null +++ b/wiremock/reflex-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd /home/wiremock + +reflex -r '\.json' -s -- sh -c "bash /docker-entrypoint.sh --extensions com.opentable.extension.BodyTransformer --local-response-templating --no-request-journal"