diff --git a/deployment/data-streams/changeset/jd_register_nodes.go b/deployment/data-streams/changeset/jd_register_nodes.go index 28002371924..42be94446cc 100644 --- a/deployment/data-streams/changeset/jd_register_nodes.go +++ b/deployment/data-streams/changeset/jd_register_nodes.go @@ -3,37 +3,90 @@ package changeset import ( "errors" "fmt" - "strconv" nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" "github.com/smartcontractkit/chainlink/deployment" ) +type NodeType int + +const ( + NodeTypeOracle NodeType = iota + NodeTypeBootstrap +) + +func (nt NodeType) String() string { + switch nt { + case NodeTypeOracle: + return "oracle" + case NodeTypeBootstrap: + return "bootstrap" + default: + return "unknown" + } +} + type RegisterNodesInput struct { EnvLabel string ProductName string - DONs DONConfigMap + // Will be deleted after migration to DONConfigMap + DONs DONConfigMap `json:"dons,omitempty"` + DONsList []DONConfig `json:"dons_list,omitempty"` } type DONConfigMap map[string]DONConfig type DONConfig struct { + ID int `json:"id"` + ChainSelector string `json:"chainSelector"` Name string `json:"name"` ChannelConfigStore string `json:"channelConfigStore"` Verifier string `json:"verifier"` Configurator string `json:"configurator"` Nodes []NodeCfg `json:"nodes"` + BootstrapNodes []NodeCfg `json:"bootstrapNodes"` } type NodeCfg struct { Name string `json:"name"` CSAKey string `json:"csa_key"` - IsBootstrap bool `json:"isBootstrap"` + IsBootstrap bool `json:"isBootstrap,omitempty"` +} + +func validateNodeSlice(nodes []NodeCfg, nodeType string, donIndex int) error { + for _, node := range nodes { + if node.Name == "" { + return fmt.Errorf("DON[%d] has %s node with empty Name", donIndex, nodeType) + } + if node.CSAKey == "" { + return fmt.Errorf("DON[%d] %s node %s has empty CSAKey", donIndex, nodeType, node.Name) + } + } + return nil +} + +func registerNodesForDON(e deployment.Environment, nodes []NodeCfg, baseLabels []*ptypes.Label, nodeType NodeType) { + ntStr := nodeType.String() + for _, node := range nodes { + labels := append([]*ptypes.Label(nil), baseLabels...) + labels = append(labels, &ptypes.Label{ + Key: "nodeType", + Value: &ntStr, + }) + nodeID, err := e.Offchain.RegisterNode(e.GetContext(), &nodev1.RegisterNodeRequest{ + Name: node.Name, + PublicKey: node.CSAKey, + Labels: labels, + }) + if err != nil { + e.Logger.Errorw("failed to register node", "node", node.Name, "error", err) + } else { + e.Logger.Infow("registered node", "name", node.Name, "id", nodeID) + } + } } -// RegisterNodesWithJD registers each node from the config with the Job Distributor. -// It logs errors but continues to register remaining nodes even if some fail (we may revisit this in the future). func RegisterNodesWithJD(e deployment.Environment, cfg RegisterNodesInput) (deployment.ChangesetOutput, error) { baseLabels := []*ptypes.Label{ { @@ -46,27 +99,9 @@ func RegisterNodesWithJD(e deployment.Environment, cfg RegisterNodesInput) (depl }, } - for _, don := range cfg.DONs { - for _, node := range don.Nodes { - labels := append([]*ptypes.Label(nil), baseLabels...) - isBootstrapStr := strconv.FormatBool(node.IsBootstrap) - - labels = append(labels, &ptypes.Label{ - Key: "isBootstrap", - Value: &isBootstrapStr, - }) - - nodeID, err := e.Offchain.RegisterNode(e.GetContext(), &nodev1.RegisterNodeRequest{ - Name: node.Name, - PublicKey: node.CSAKey, - Labels: labels, - }) - if err != nil { - e.Logger.Errorw("failed to register node", "node", node.Name, "error", err) - } else { - e.Logger.Infow("registered node", "name", node.Name, "id", nodeID) - } - } + for _, don := range cfg.DONsList { + registerNodesForDON(e, don.Nodes, baseLabels, NodeTypeOracle) + registerNodesForDON(e, don.BootstrapNodes, baseLabels, NodeTypeBootstrap) } return deployment.ChangesetOutput{}, nil @@ -80,17 +115,18 @@ func (cfg RegisterNodesInput) Validate() error { return errors.New("ProductName must not be empty") } - for donName, don := range cfg.DONs { + for i, don := range cfg.DONsList { if don.Name == "" { - return fmt.Errorf("DON[%s] has empty Name", donName) + return fmt.Errorf("DON[%d] has empty Name", i) + } + if err := validateNodeSlice(don.Nodes, "node", i); err != nil { + return err + } + if len(don.BootstrapNodes) == 0 { + return fmt.Errorf("DON[%d] has no bootstrap nodes", i) } - for _, node := range don.Nodes { - if node.Name == "" { - return fmt.Errorf("DON[%s] has node with empty Name", donName) - } - if node.CSAKey == "" { - return fmt.Errorf("DON[%s] node %s has empty CSAKey", donName, node.Name) - } + if err := validateNodeSlice(don.BootstrapNodes, "bootstrap", i); err != nil { + return err } } return nil diff --git a/deployment/data-streams/changeset/jd_register_nodes_test.go b/deployment/data-streams/changeset/jd_register_nodes_test.go index f95e07b19cb..c0b539a1f15 100644 --- a/deployment/data-streams/changeset/jd_register_nodes_test.go +++ b/deployment/data-streams/changeset/jd_register_nodes_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" "github.com/smartcontractkit/chainlink/deployment" @@ -21,7 +20,6 @@ func TestRegisterNodesWithJD(t *testing.T) { nodeP2pKey := e.NodeIDs[0] jobClient, ok := e.Offchain.(*memory.JobClient) - require.True(t, ok, "expected Offchain to be of type *memory.JobClient") require.Lenf(t, jobClient.Nodes, 1, "expected exactly 1 node") require.Emptyf(t, jobClient.RegisteredNodes, "no registered nodes expected") @@ -34,10 +32,10 @@ func TestRegisterNodesWithJD(t *testing.T) { RegisterNodesInput{ EnvLabel: "test-env", ProductName: "test-product", - DONs: DONConfigMap{ - "don1": { + DONsList: []DONConfig{ + { Name: "don1", - Nodes: []NodeCfg{ + BootstrapNodes: []NodeCfg{ {Name: "node1", CSAKey: csaKey}, }, }, @@ -55,11 +53,14 @@ func TestRegisterNodesInput_Validate(t *testing.T) { cfg := RegisterNodesInput{ EnvLabel: "test-env", ProductName: "test-product", - DONs: DONConfigMap{ - "don1": { + DONsList: []DONConfig{ + { Name: "MyDON", Nodes: []NodeCfg{ - {Name: "node1", CSAKey: "0xabc", IsBootstrap: false}, + {Name: "node1", CSAKey: "0xabc"}, + }, + BootstrapNodes: []NodeCfg{ + {Name: "bootstrap1", CSAKey: "0xdef"}, }, }, }, @@ -70,13 +71,17 @@ func TestRegisterNodesInput_Validate(t *testing.T) { t.Run("missing product name", func(t *testing.T) { cfg := RegisterNodesInput{ - EnvLabel: "test-env", - DONs: DONConfigMap{ - "don2": { + EnvLabel: "test-env", + ProductName: "", + DONsList: []DONConfig{ + { Name: "AnotherDON", Nodes: []NodeCfg{ {Name: "node1", CSAKey: "0xdef"}, }, + BootstrapNodes: []NodeCfg{ + {Name: "node2", CSAKey: "0xabc"}, + }, }, }, } @@ -88,11 +93,14 @@ func TestRegisterNodesInput_Validate(t *testing.T) { cfg := RegisterNodesInput{ EnvLabel: "test-env", ProductName: "test-product", - DONs: DONConfigMap{ - "don3": { + DONsList: []DONConfig{ + { Name: "EmptyCSA", Nodes: []NodeCfg{ - {Name: "node1", CSAKey: "", IsBootstrap: true}, + {Name: "node1", CSAKey: ""}, + }, + BootstrapNodes: []NodeCfg{ + {Name: "bootstrap1", CSAKey: ""}, }, }, }, @@ -100,4 +108,21 @@ func TestRegisterNodesInput_Validate(t *testing.T) { err := cfg.Validate() require.Error(t, err, "expected an error when CSAKey is empty") }) + + t.Run("missing BootstrapNode", func(t *testing.T) { + cfg := RegisterNodesInput{ + EnvLabel: "test-env", + ProductName: "test-product", + DONsList: []DONConfig{ + { + Name: "EmptyCSA", + Nodes: []NodeCfg{ + {Name: "node1", CSAKey: "0xaaa"}, + }, + }, + }, + } + err := cfg.Validate() + require.Error(t, err, "expected an error when BooststrapNodes is empty") + }) } diff --git a/deployment/data-streams/clients/entity_tool.go b/deployment/data-streams/clients/entity_tool.go new file mode 100644 index 00000000000..98e313a4e36 --- /dev/null +++ b/deployment/data-streams/clients/entity_tool.go @@ -0,0 +1,136 @@ +package clients + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" +) + +type GetOverridesRequest struct { + asset string + quote string + product *string +} + +type GetAssetEAsRequest struct { + asset string + quote string + product *string +} + +type ( + GetOverridesResponse map[string]string + GetAssetEAsResponse []string +) + +type EntityToolClient interface { + GetOverrides(ctx context.Context, in *GetOverridesRequest) (*GetOverridesResponse, error) + GetAssetEAs(ctx context.Context, in *GetAssetEAsRequest) (*GetAssetEAsResponse, error) +} + +type EntityToolClientImpl struct { + baseURL string + client *http.Client +} + +func NewEntityToolClient(baseURL string, client *http.Client) EntityToolClient { + if client == nil { + client = http.DefaultClient + } + return &EntityToolClientImpl{ + baseURL: baseURL, + client: client, + } +} + +type feedBuildObjectResponse struct { + ExternalAdapterRequestParams struct { + Overrides map[string]string `json:"overrides"` + } `json:"externalAdapterRequestParams"` + APIs []string `json:"apis"` +} + +func (c *EntityToolClientImpl) getFeedBuildObjectResponse(ctx context.Context, asset, quote, product string) (*feedBuildObjectResponse, error) { + endpoint := c.baseURL + "/api/feed-build-object" + values := url.Values{} + values.Add("base", asset) + values.Add("quote", quote) + values.Add("product", product) + values.Add("configType", "DATA_STREAMS") + reqURL := endpoint + "?" + values.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, errors.New("failed to create request: " + err.Error()) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, errors.New("request failed: " + err.Error()) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errors.New("unexpected status code: " + http.StatusText(resp.StatusCode)) + } + var result feedBuildObjectResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, errors.New("failed to decode response: " + err.Error()) + } + return &result, nil +} + +func (c *EntityToolClientImpl) GetOverrides(ctx context.Context, in *GetOverridesRequest) (*GetOverridesResponse, error) { + product := "crypto" + if in.product != nil { + product = *in.product + } + res, err := c.getFeedBuildObjectResponse(ctx, in.asset, in.quote, product) + if err != nil { + return nil, err + } + if res.ExternalAdapterRequestParams.Overrides == nil { + return nil, errors.New("no APIs found for the given asset and quote") + } + overrides := GetOverridesResponse(res.ExternalAdapterRequestParams.Overrides) + return &overrides, nil +} + +func (c *EntityToolClientImpl) GetAssetEAs(ctx context.Context, in *GetAssetEAsRequest) (*GetAssetEAsResponse, error) { + product := "crypto" + if in.product != nil { + product = *in.product + } + res, err := c.getFeedBuildObjectResponse(ctx, in.asset, in.quote, product) + if err != nil { + return nil, err + } + if len(res.APIs) == 0 { + return nil, errors.New("no APIs found for the given asset and quote") + } + eas := GetAssetEAsResponse(res.APIs) + return &eas, nil +} + +func NewGetOverridesRequest(asset, quote string, product ...string) *GetOverridesRequest { + var prod *string + if len(product) > 0 { + prod = &product[0] + } + return &GetOverridesRequest{ + asset: asset, + quote: quote, + product: prod, + } +} + +func NewGetAssetEAsRequest(asset, quote string, product ...string) *GetAssetEAsRequest { + var prod *string + if len(product) > 0 { + prod = &product[0] + } + return &GetAssetEAsRequest{ + asset: asset, + quote: quote, + product: prod, + } +} diff --git a/deployment/data-streams/clients/entity_tool_test.go b/deployment/data-streams/clients/entity_tool_test.go new file mode 100644 index 00000000000..31f07d6e00b --- /dev/null +++ b/deployment/data-streams/clients/entity_tool_test.go @@ -0,0 +1,211 @@ +package clients + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func TestEntityToolClient_GetOverrides(t *testing.T) { + type testCase struct { + name string + asset string + quote string + product *string + responseStatus int + responseBody string + expectedOverrides GetOverridesResponse + expectError bool + } + + testCases := []testCase{ + { + name: "success default product", + asset: "BTC", + quote: "USD", + product: nil, + responseStatus: http.StatusOK, + responseBody: `{ + "externalAdapterRequestParams": { + "overrides": { + "api1": "value1", + "api2": "value2" + } + }, + "apis": [] + }`, + expectedOverrides: GetOverridesResponse{ + "api1": "value1", + "api2": "value2", + }, + expectError: false, + }, + { + name: "non-200 response", + asset: "BTC", + quote: "USD", + product: nil, + responseStatus: http.StatusInternalServerError, + responseBody: "internal error", + expectError: true, + }, + { + name: "no overrides present", + asset: "BTC", + quote: "USD", + product: nil, + responseStatus: http.StatusOK, + responseBody: `{ + "externalAdapterRequestParams": { + "overrides": null + }, + "apis": [] + }`, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if q.Get("base") != tc.asset || q.Get("quote") != tc.quote { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.responseStatus) + fmt.Fprint(w, tc.responseBody) + }) + server := httptest.NewServer(handler) + defer server.Close() + + client := NewEntityToolClient(server.URL, server.Client()) + + var req *GetOverridesRequest + if tc.product != nil { + req = NewGetOverridesRequest(tc.asset, tc.quote, *tc.product) + } else { + req = NewGetOverridesRequest(tc.asset, tc.quote) + } + + ctx := context.Background() + resp, err := client.GetOverrides(ctx, req) + if tc.expectError { + if err == nil { + t.Fatalf("expected error but got none") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(*resp) != len(tc.expectedOverrides) { + t.Fatalf("expected %d override entries, got %d", len(tc.expectedOverrides), len(*resp)) + } + for key, expectedVal := range tc.expectedOverrides { + if val, ok := (*resp)[key]; !ok || val != expectedVal { + t.Errorf("expected %s to be %s, got %v", key, expectedVal, val) + } + } + }) + } +} + +func TestEntityToolClient_GetAssetEAs(t *testing.T) { + type testCase struct { + name string + asset string + quote string + product *string + responseStatus int + responseBody string + expectedAPIs []string + expectError bool + } + + testCases := []testCase{ + { + name: "success default product", + asset: "BTC", + quote: "USD", + product: nil, + responseStatus: http.StatusOK, + responseBody: `{ + "externalAdapterRequestParams": { "overrides": {} }, + "apis": ["tiingo", "ncfx"] + }`, + expectedAPIs: []string{"tiingo", "ncfx"}, + expectError: false, + }, + { + name: "non-200 response", + asset: "BTC", + quote: "USD", + product: nil, + responseStatus: http.StatusInternalServerError, + responseBody: "internal error", + expectError: true, + }, + { + name: "empty APIs list", + asset: "BTC", + quote: "USD", + product: nil, + responseStatus: http.StatusOK, + responseBody: `{ + "externalAdapterRequestParams": { "overrides": {"dummy": "dummy"} }, + "apis": [] + }`, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if q.Get("base") != tc.asset || q.Get("quote") != tc.quote { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.responseStatus) + fmt.Fprint(w, tc.responseBody) + }) + server := httptest.NewServer(handler) + defer server.Close() + + client := NewEntityToolClient(server.URL, server.Client()) + + var req *GetAssetEAsRequest + if tc.product != nil { + req = NewGetAssetEAsRequest(tc.asset, tc.quote, *tc.product) + } else { + req = NewGetAssetEAsRequest(tc.asset, tc.quote) + } + + ctx := context.Background() + resp, err := client.GetAssetEAs(ctx, req) + if tc.expectError { + if err == nil { + t.Fatalf("expected error but got none") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(*resp) != len(tc.expectedAPIs) { + t.Fatalf("expected %d APIs, got %d", len(tc.expectedAPIs), len(*resp)) + } + for i, expected := range tc.expectedAPIs { + if (*resp)[i] != expected { + t.Errorf("expected API at index %d to be %s, got %s", i, expected, (*resp)[i]) + } + } + }) + } +} diff --git a/deployment/data-streams/clients/external_adapter.go b/deployment/data-streams/clients/external_adapter.go new file mode 100644 index 00000000000..0eb3650856c --- /dev/null +++ b/deployment/data-streams/clients/external_adapter.go @@ -0,0 +1,97 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" +) + +type ExternalAdapterRequest struct { + AdapterURL string `json:"adapterUrl"` + Asset string `json:"asset"` + Quote string `json:"quote"` + Endpoint string `json:"endpoint"` +} + +type ExternalAdapterResponse struct { + Pass bool `json:"pass"` + StatusCode int `json:"statusCode"` + Data map[string]interface{} `json:"data,omitempty"` + ErrorMessage string `json:"errorMessage"` +} + +type ExternalAdapterClient interface { + Query(ctx context.Context, req ExternalAdapterRequest) (ExternalAdapterResponse, error) +} + +type externalAdapterClientImpl struct { + client *http.Client +} + +// If client is nil, http.DefaultClient is used. +func NewExternalAdapterClient(client *http.Client) ExternalAdapterClient { + if client == nil { + client = http.DefaultClient + } + return &externalAdapterClientImpl{ + client: client, + } +} + +func (c *externalAdapterClientImpl) Query(ctx context.Context, reqData ExternalAdapterRequest) (ExternalAdapterResponse, error) { + payload := map[string]interface{}{ + "data": map[string]interface{}{ + "from": reqData.Asset, + "to": reqData.Quote, + "endpoint": reqData.Endpoint, + }, + } + body, err := json.Marshal(payload) + if err != nil { + return ExternalAdapterResponse{}, fmt.Errorf("failed to marshal request payload: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, reqData.AdapterURL, bytes.NewBuffer(body)) + if err != nil { + return ExternalAdapterResponse{}, fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := c.client.Do(httpReq) + if err != nil { + return ExternalAdapterResponse{ + Pass: false, + StatusCode: http.StatusInternalServerError, + ErrorMessage: err.Error(), + }, nil + } + defer resp.Body.Close() + + var result ExternalAdapterResponse + if resp.StatusCode == http.StatusOK { + if err := json.NewDecoder(resp.Body).Decode(&result.Data); err != nil { + return ExternalAdapterResponse{ + Pass: false, + StatusCode: resp.StatusCode, + ErrorMessage: fmt.Sprintf("failed to decode JSON: %v", err), + }, nil + } + result.Pass = true + result.StatusCode = resp.StatusCode + result.ErrorMessage = "" + } else { + var errData map[string]interface{} + _ = json.NewDecoder(resp.Body).Decode(&errData) + result = ExternalAdapterResponse{ + Pass: false, + StatusCode: resp.StatusCode, + ErrorMessage: resp.Status, + Data: errData, + } + } + + return result, nil +} diff --git a/deployment/data-streams/clients/external_adapter_test.go b/deployment/data-streams/clients/external_adapter_test.go new file mode 100644 index 00000000000..0a9fe541399 --- /dev/null +++ b/deployment/data-streams/clients/external_adapter_test.go @@ -0,0 +1,125 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +type queryTestCase struct { + name string + asset string + quote string + endpoint string + responseStatus int + responseBody string + expectedPass bool + expectedStatusCode int + expectedData map[string]interface{} + expectedErrorSubstr string +} + +func TestExternalAdapterClient_Query(t *testing.T) { + testCases := []queryTestCase{ + { + name: "success", + asset: "BTC", + quote: "USD", + endpoint: "price", + responseStatus: http.StatusOK, + responseBody: `{"result": "ok", "value": 123}`, + expectedPass: true, + expectedStatusCode: http.StatusOK, + expectedData: map[string]interface{}{ + "result": "ok", + "value": float64(123), + }, + }, + { + name: "failure - non-200 response", + asset: "BTC", + quote: "USD", + endpoint: "price", + responseStatus: http.StatusInternalServerError, + responseBody: `internal error`, + expectedPass: false, + expectedStatusCode: http.StatusInternalServerError, + expectedErrorSubstr: "500", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var reqData map[string]map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&reqData); err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + data := reqData["data"] + if data["from"] != tc.asset || data["to"] != tc.quote || data["endpoint"] != tc.endpoint { + http.Error(w, "invalid payload", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.responseStatus) + if _, err := w.Write([]byte(tc.responseBody)); err != nil { + return + } + })) + defer server.Close() + + client := NewExternalAdapterClient(server.Client()) + req := ExternalAdapterRequest{ + AdapterURL: server.URL, + Asset: tc.asset, + Quote: tc.quote, + Endpoint: tc.endpoint, + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + resp, err := client.Query(ctx, req) + if err != nil { + t.Fatalf("unexpected error from Query: %v", err) + } + + if resp.Pass != tc.expectedPass { + t.Errorf("expected Pass to be %v, got %v", tc.expectedPass, resp.Pass) + } + if resp.StatusCode != tc.expectedStatusCode { + t.Errorf("expected status code %d, got %d", tc.expectedStatusCode, resp.StatusCode) + } + + if !tc.expectedPass { + if resp.ErrorMessage == "" { + t.Error("expected an error message, but got empty") + } + if !contains(resp.ErrorMessage, tc.expectedErrorSubstr) { + t.Errorf("expected error message to contain %q, got %q", tc.expectedErrorSubstr, resp.ErrorMessage) + } + return + } + + gotData, err := json.Marshal(resp.Data) + if err != nil { + t.Fatalf("failed to marshal response data: %v", err) + } + expectedData, err := json.Marshal(tc.expectedData) + if err != nil { + t.Fatalf("failed to marshal expected data: %v", err) + } + if !bytes.Equal(gotData, expectedData) { + t.Errorf("expected data %s, got %s", expectedData, gotData) + } + }) + } +} + +func contains(str, substr string) bool { + return len(str) >= len(substr) && (substr == "" || (len(substr) > 0 && (str == substr || (len(str) > len(substr) && (str[len(str)-len(substr):] == substr || str[:len(substr)] == substr))))) +}