diff --git a/internal/app/api/api.go b/internal/app/api/api.go index ce4588b8..8dfaaa39 100644 --- a/internal/app/api/api.go +++ b/internal/app/api/api.go @@ -30,6 +30,8 @@ var ( defaultRetryMax = 4 ) +const activityHistoryLogURL = "/accounts/history/v1/activity_log/list" + // Client is used to with iAuditor API's. type Client struct { logger *zap.SugaredLogger @@ -159,76 +161,6 @@ func OptAddTLSCert(certPath string) Opt { } } -// FeedMetadata is a representation of the metadata returned when fetching a feed -type FeedMetadata struct { - NextPage string `json:"next_page"` - RemainingRecords int64 `json:"remaining_records"` -} - -// GetFeedParams is a list of all parameters we can set when fetching a feed -type GetFeedParams struct { - ModifiedAfter time.Time `url:"modified_after,omitempty"` - TemplateIDs []string `url:"template,omitempty"` - Archived string `url:"archived,omitempty"` - Completed string `url:"completed,omitempty"` - IncludeInactive bool `url:"include_inactive,omitempty"` - Limit int `url:"limit,omitempty"` - WebReportLink string `url:"web_report_link,omitempty"` - - // Applicable only for sites - IncludeDeleted bool `url:"include_deleted,omitempty"` - ShowOnlyLeafNodes *bool `url:"show_only_leaf_nodes,omitempty"` -} - -// GetFeedRequest has all the data needed to make a request to get a feed -type GetFeedRequest struct { - URL string - InitialURL string - Params GetFeedParams -} - -// GetFeedResponse is a representation of the data returned when fetching a feed -type GetFeedResponse struct { - Metadata FeedMetadata `json:"metadata"` - - Data json.RawMessage `json:"data"` -} - -// GetMediaRequest has all the data needed to make a request to get a media -type GetMediaRequest struct { - URL string - AuditID string -} - -// GetMediaResponse is a representation of the data returned when fetching media -type GetMediaResponse struct { - ContentType string - Body []byte - MediaID string -} - -// ListInspectionsParams is a list of all parameters we can set when fetching inspections -type ListInspectionsParams struct { - ModifiedAfter time.Time `url:"modified_after,omitempty"` - TemplateIDs []string `url:"template,omitempty"` - Archived string `url:"archived,omitempty"` - Completed string `url:"completed,omitempty"` - Limit int `url:"limit,omitempty"` -} - -// Inspection represents some of the properties present in an inspection -type Inspection struct { - ID string `json:"audit_id"` - ModifiedAt time.Time `json:"modified_at"` -} - -// ListInspectionsResponse represents the response of listing inspections -type ListInspectionsResponse struct { - Count int `json:"count"` - Total int `json:"total"` - Inspections []Inspection `json:"audits"` -} - func (a *Client) do(doer HTTPDoer) (*http.Response, error) { url := doer.URL() @@ -265,6 +197,10 @@ func (a *Client) do(doer HTTPDoer) (*http.Response, error) { ) return resp, nil + case status == http.StatusForbidden: + a.logger.Debugw("no access to this resource", "url", url, "status", status) + return resp, nil + default: a.logger.Errorw("http request error status", "url", url, @@ -355,7 +291,8 @@ func (a *Client) GetFeed(ctx context.Context, request *GetFeedRequest) (*GetFeed initialURL = request.URL } - sl := a.sling.New().Get(initialURL). + sl := a.sling.New(). + Get(initialURL). Set(string(Authorization), "Bearer "+a.accessToken). Set(string(IntegrationID), "iauditor-exporter"). Set(string(IntegrationVersion), version.GetVersion()). @@ -404,6 +341,56 @@ func (a *Client) DrainFeed(ctx context.Context, request *GetFeedRequest, feedFn return nil } +// ListOrganisationActivityLog returns response from AccountsActivityLog or error +func (a *Client) ListOrganisationActivityLog(ctx context.Context, request *GetAccountsActivityLogRequestParams) (*GetAccountsActivityLogResponse, error) { + sl := a.sling.New(). + Post(activityHistoryLogURL). + Set(string(Authorization), "Bearer "+a.accessToken). + Set(string(IntegrationID), "iauditor-exporter"). + Set(string(IntegrationVersion), version.GetVersion()). + Set(string(XRequestID), util.RequestIDFromContext(ctx)). + BodyJSON(request) + + req, _ := sl.Request() + req = req.WithContext(ctx) + + var res GetAccountsActivityLogResponse + var errMsg json.RawMessage + _, err := a.do(&slingHTTPDoer{ + sl: sl, + req: req, + successV: &res, + failureV: &errMsg, + }) + if err != nil { + return nil, errors.Wrap(err, "Failed request to API") + } + + return &res, nil +} + +// DrainAccountActivityHistoryLog cycle throgh GetAccountsActivityLogResponse and adapts the filter whule there is a next page +func (a *Client) DrainAccountActivityHistoryLog(ctx context.Context, req *GetAccountsActivityLogRequestParams, feedFn func(*GetAccountsActivityLogResponse) error) error { + for { + res, err := a.ListOrganisationActivityLog(ctx, req) + if err != nil { + return err + } + + err = feedFn(res) + if err != nil { + return err + } + + if res.NextPageToken != "" { + req.PageToken = res.NextPageToken + } else { + break + } + } + return nil +} + // ListInspections retrieves the list of inspections from iAuditor func (a *Client) ListInspections(ctx context.Context, params *ListInspectionsParams) (*ListInspectionsResponse, error) { var ( @@ -499,15 +486,6 @@ func (a *Client) DrainInspections( return nil } -type initiateInspectionReportExportRequest struct { - Format string `json:"format"` - PreferenceID string `json:"preference_id,omitempty"` -} - -type initiateInspectionReportExportResponse struct { - MessageID string `json:"messageId"` -} - // InitiateInspectionReportExport export the report of the given auditID. func (a *Client) InitiateInspectionReportExport(ctx context.Context, auditID string, format string, preferenceID string) (string, error) { var ( @@ -544,12 +522,6 @@ func (a *Client) InitiateInspectionReportExport(ctx context.Context, auditID str return result.MessageID, nil } -// InspectionReportExportCompletionResponse represents the response of report export completion status -type InspectionReportExportCompletionResponse struct { - Status string `json:"status"` - URL string `json:"url,omitempty"` -} - // CheckInspectionReportExportCompletion checks if the report export is complete. func (a *Client) CheckInspectionReportExportCompletion(ctx context.Context, auditID string, messageID string) (*InspectionReportExportCompletionResponse, error) { var ( @@ -605,14 +577,6 @@ func (a *Client) DownloadInspectionReportFile(ctx context.Context, url string) ( return res.Body, nil } -// WhoAmIResponse represents the the response of WhoAmI -type WhoAmIResponse struct { - UserID string `json:"user_id"` - OrganisationID string `json:"organisation_id"` - Firstname string `json:"firstname"` - Lastname string `json:"lastname"` -} - // WhoAmI returns the details for the user who is making the request func (a *Client) WhoAmI(ctx context.Context) (*WhoAmIResponse, error) { var ( diff --git a/internal/app/api/api_test.go b/internal/app/api/api_test.go index 77f94df8..c23dbecc 100644 --- a/internal/app/api/api_test.go +++ b/internal/app/api/api_test.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "github.com/stretchr/testify/require" + "net/http" "net/url" + "path" "strings" "testing" "time" @@ -17,6 +19,117 @@ import ( "gopkg.in/h2non/gock.v1" ) +func TestClient_DrainDeletedInspections(t *testing.T) { + defer gock.Off() + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":4,"page_token":"","filters":{"timeframe":{"from":"2022-06-30T10:43:17Z"},"event_types":["inspection.deleted"],"limit":4}}`). + Reply(http.StatusOK). + File(path.Join("fixtures", "inspections_deleted_page_1.json")) + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":4,"page_token":"eyJldmVudF90eXBlcyI6WyJpbnNwZWN0aW9uLmFyY2hpdmVkIl0sImxpbWl0Ijo0LCJvZmZzZXQiOjR9","filters":{"timeframe":{"from":"2022-06-30T10:43:17Z"},"event_types":["inspection.deleted"],"limit":4}}`). + Reply(http.StatusOK). + File(path.Join("fixtures", "inspections_deleted_page_2.json")) + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":4,"page_token":"eyJldmVudF90eXBlcyI6WyJpbnNwZWN0aW9uLmFyY2hpdmVkIl0sImxpbWl0Ijo0LCJvZmZzZXQiOjh9","filters":{"timeframe":{"from":"2022-06-30T10:43:17Z"},"event_types":["inspection.deleted"],"limit":4}}`). + Reply(http.StatusOK). + File(path.Join("fixtures", "inspections_deleted_page_3.json")) + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":4,"page_token":"eyJldmVudF90eXBlcyI6WyJpbnNwZWN0aW9uLmFyY2hpdmVkIl0sImxpbWl0Ijo0LCJvZmZzZXQiOjEyfQ==","filters":{"timeframe":{"from":"2022-06-30T10:43:17Z"},"event_types":["inspection.deleted"],"limit":4}}`). + Reply(http.StatusOK). + File(path.Join("fixtures", "inspections_deleted_page_4.json")) + + apiClient := api.GetTestClient() + gock.InterceptClient(apiClient.HTTPClient()) + + fakeTime, err := time.Parse(time.RFC3339, "2022-06-30T10:43:17Z") + require.Nil(t, err) + req := api.NewGetAccountsActivityLogRequest(4, fakeTime) + + calls := 0 + var deletedIds = make([]string, 0, 15) + fn := func(res *api.GetAccountsActivityLogResponse) error { + calls++ + for _, a := range res.Activities { + deletedIds = append(deletedIds, a.Metadata["inspection_id"]) + } + return nil + } + err = apiClient.DrainAccountActivityHistoryLog(context.TODO(), req, fn) + require.Nil(t, err) + assert.EqualValues(t, 4, calls) + require.EqualValues(t, 15, len(deletedIds)) + assert.EqualValues(t, "3b8ac4f4-e904-453e-b5a0-b5cceedb0ee1", deletedIds[0]) + assert.EqualValues(t, "4b3bc1d5-3011-4f81-94d4-125d2bce7ca8", deletedIds[1]) + assert.EqualValues(t, "6bd628a6-5188-425f-89ef-81f9dfcdf5cd", deletedIds[2]) + assert.EqualValues(t, "d722fc86-defa-4de2-b8d7-c0a3e0ec6ce4", deletedIds[3]) + assert.EqualValues(t, "ed8b3911-4141-41c4-946c-167bb6f61109", deletedIds[4]) + assert.EqualValues(t, "fd95cb4b-e1e7-488b-ba58-93fecd2379dc", deletedIds[5]) + assert.EqualValues(t, "1878c1e2-8a42-4f63-9e07-2e605f76762b", deletedIds[6]) + assert.EqualValues(t, "9e28ab2c-ce8c-44a7-81d3-76d0ac47dc91", deletedIds[7]) + assert.EqualValues(t, "48d61915-98c8-4d05-b786-4948dad199be", deletedIds[8]) + assert.EqualValues(t, "331727d2-4976-45da-857a-6d080dc645a9", deletedIds[9]) + assert.EqualValues(t, "1f2c9c1b-6f35-4bae-9b38-4094b40e13c1", deletedIds[10]) + assert.EqualValues(t, "35583d49-6421-40a8-a6f5-591c718c6025", deletedIds[11]) + assert.EqualValues(t, "eb49e9f8-4a3c-4b8f-a180-7ba0d171e93d", deletedIds[12]) + assert.EqualValues(t, "47ac0dce-16f9-4d73-b517-8372368af162", deletedIds[13]) + assert.EqualValues(t, "6d2f8bd5-a965-4046-b2b4-ccdf8341c9f0", deletedIds[14]) +} + +func TestClient_DrainDeletedInspections_WhenApiReturnsError(t *testing.T) { + defer gock.Off() + + gock.New("http://localhost:9999"). + Persist(). + Post("/accounts/history/v1/activity_log/list"). + Reply(http.StatusInternalServerError). + JSON(`{"error": "something bad happened"}`) + + apiClient := api.GetTestClient() + gock.InterceptClient(apiClient.HTTPClient()) + + fakeTime, err := time.Parse(time.RFC3339, "2022-06-30T10:43:17Z") + require.Nil(t, err) + req := api.NewGetAccountsActivityLogRequest(14, fakeTime) + fn := func(res *api.GetAccountsActivityLogResponse) error { + return nil + } + err = apiClient.DrainAccountActivityHistoryLog(context.TODO(), req, fn) + require.NotNil(t, err) + assert.EqualValues(t, "Failed request to API: http://localhost:9999/accounts/history/v1/activity_log/list giving up after 2 attempt(s)", err.Error()) +} + +func TestClient_DrainDeletedInspections_WhenFeedFnReturnsError(t *testing.T) { + defer gock.Off() + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":4,"page_token":"","filters":{"timeframe":{"from":"2022-06-30T10:43:17Z"},"event_types":["inspection.deleted"],"limit":4}}`). + Reply(http.StatusOK). + File(path.Join("fixtures", "inspections_deleted_page_1.json")) + + apiClient := api.GetTestClient() + gock.InterceptClient(apiClient.HTTPClient()) + + fakeTime, err := time.Parse(time.RFC3339, "2022-06-30T10:43:17Z") + require.Nil(t, err) + req := api.NewGetAccountsActivityLogRequest(4, fakeTime) + + fn := func(res *api.GetAccountsActivityLogResponse) error { + return fmt.Errorf("ERROR_GetAccountsActivityLogResponse") + } + err = apiClient.DrainAccountActivityHistoryLog(context.TODO(), req, fn) + require.NotNil(t, err) + assert.EqualValues(t, "ERROR_GetAccountsActivityLogResponse", err.Error()) +} + func TestAPIClientDrainFeed_should_return_for_as_long_next_page_set(t *testing.T) { defer gock.Off() diff --git a/internal/app/api/fixtures/inspections_deleted_page_1.json b/internal/app/api/fixtures/inspections_deleted_page_1.json new file mode 100644 index 00000000..09d3b322 --- /dev/null +++ b/internal/app/api/fixtures/inspections_deleted_page_1.json @@ -0,0 +1,69 @@ +{ + "activities": [ + { + "id": "ef1639d3-9820-4821-b0f8-5097bf0dc950", + "event_at": "2022-07-05T01:02:32.837Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "3b8ac4f4-e904-453e-b5a0-b5cceedb0ee1", + "inspection_name": "5 Jul 2022 / #10 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "e4691ff3-1840-4466-8c1b-04a2dcef4589", + "event_at": "2022-07-05T01:02:27.940Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "4b3bc1d5-3011-4f81-94d4-125d2bce7ca8", + "inspection_name": "5 Jul 2022 / #14 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "431c5481-b8c9-4567-9905-2a899dc9dffc", + "event_at": "2022-07-05T01:02:26.580Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "6bd628a6-5188-425f-89ef-81f9dfcdf5cd", + "inspection_name": "5 Jul 2022 / #15 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "97705279-0123-4693-9a2b-62ede848b647", + "event_at": "2022-07-05T01:02:25.278Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "d722fc86-defa-4de2-b8d7-c0a3e0ec6ce4", + "inspection_name": "5 Jul 2022 / #13 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "eyJldmVudF90eXBlcyI6WyJpbnNwZWN0aW9uLmFyY2hpdmVkIl0sImxpbWl0Ijo0LCJvZmZzZXQiOjR9" +} diff --git a/internal/app/api/fixtures/inspections_deleted_page_2.json b/internal/app/api/fixtures/inspections_deleted_page_2.json new file mode 100644 index 00000000..1685d8a7 --- /dev/null +++ b/internal/app/api/fixtures/inspections_deleted_page_2.json @@ -0,0 +1,69 @@ +{ + "activities": [ + { + "id": "a60a70ca-da24-4ba1-9a95-f4a594726988", + "event_at": "2022-07-05T01:02:23.997Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "ed8b3911-4141-41c4-946c-167bb6f61109", + "inspection_name": "5 Jul 2022 / #12 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "8aa75b1d-2c65-404e-9a99-63f759c7db80", + "event_at": "2022-07-05T01:02:22.715Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "fd95cb4b-e1e7-488b-ba58-93fecd2379dc", + "inspection_name": "5 Jul 2022 / #9 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "cb4d39d5-a8cb-4b0f-9544-4fef9d725782", + "event_at": "2022-07-05T01:02:20.544Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "1878c1e2-8a42-4f63-9e07-2e605f76762b", + "inspection_name": "5 Jul 2022 / #11 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "ba126877-945f-4089-9c8c-acf497e76916", + "event_at": "2022-07-05T01:02:19.085Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "9e28ab2c-ce8c-44a7-81d3-76d0ac47dc91", + "inspection_name": "5 Jul 2022 / #9 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "eyJldmVudF90eXBlcyI6WyJpbnNwZWN0aW9uLmFyY2hpdmVkIl0sImxpbWl0Ijo0LCJvZmZzZXQiOjh9" +} diff --git a/internal/app/api/fixtures/inspections_deleted_page_3.json b/internal/app/api/fixtures/inspections_deleted_page_3.json new file mode 100644 index 00000000..6271f88b --- /dev/null +++ b/internal/app/api/fixtures/inspections_deleted_page_3.json @@ -0,0 +1,69 @@ +{ + "activities": [ + { + "id": "b3ff8746-7121-4602-bb0f-7b8c625fb9d2", + "event_at": "2022-07-05T01:02:17.263Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "48d61915-98c8-4d05-b786-4948dad199be", + "inspection_name": "5 Jul 2022 / #5 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "3089609c-8351-4225-ac6b-71236b62e7d3", + "event_at": "2022-07-05T01:02:15.941Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "331727d2-4976-45da-857a-6d080dc645a9", + "inspection_name": "5 Jul 2022 / #7 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "09723ba3-bb85-4f41-9057-10cbad397bbf", + "event_at": "2022-07-05T01:02:14.487Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "1f2c9c1b-6f35-4bae-9b38-4094b40e13c1", + "inspection_name": "5 Jul 2022 / #8 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "35ffb060-2705-44a7-b070-9690ee036c85", + "event_at": "2022-07-05T01:02:12.643Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "35583d49-6421-40a8-a6f5-591c718c6025", + "inspection_name": "5 Jul 2022 / #6 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "eyJldmVudF90eXBlcyI6WyJpbnNwZWN0aW9uLmFyY2hpdmVkIl0sImxpbWl0Ijo0LCJvZmZzZXQiOjEyfQ==" +} diff --git a/internal/app/api/fixtures/inspections_deleted_page_4.json b/internal/app/api/fixtures/inspections_deleted_page_4.json new file mode 100644 index 00000000..a4c306b5 --- /dev/null +++ b/internal/app/api/fixtures/inspections_deleted_page_4.json @@ -0,0 +1,53 @@ +{ + "activities": [ + { + "id": "a7f7990e-a3d8-4985-b3c1-fb137b192a2c", + "event_at": "2022-07-05T01:02:10.887Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "eb49e9f8-4a3c-4b8f-a180-7ba0d171e93d", + "inspection_name": "5 Jul 2022 / #4 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "24fbaead-6d4b-4cd8-992c-8f71d67a90f4", + "event_at": "2022-07-04T03:48:05.044Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "47ac0dce-16f9-4d73-b517-8372368af162", + "inspection_name": "4 Jul 2022 / #3 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "ca54b734-75ee-44db-8ac5-7d629e5f234d", + "event_at": "2022-07-04T03:47:44.810Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "6d2f8bd5-a965-4046-b2b4-ccdf8341c9f0", + "inspection_name": "4 Jul 2022 / #2 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "" +} diff --git a/internal/app/api/models.go b/internal/app/api/models.go new file mode 100644 index 00000000..607a960d --- /dev/null +++ b/internal/app/api/models.go @@ -0,0 +1,144 @@ +package api + +import ( + "encoding/json" + "time" +) + +// NewGetAccountsActivityLogRequest build a request for AccountsActivityLog +// for now it serves the purposes only for inspection.deleted. If we need later, we can change this builder +func NewGetAccountsActivityLogRequest(pageSize int, from time.Time) *GetAccountsActivityLogRequestParams { + return &GetAccountsActivityLogRequestParams{ + PageSize: pageSize, + Filters: accountsActivityLogFilter{ + Timeframe: timeFrame{ + From: from, + }, + Limit: pageSize, + EventTypes: []string{"inspection.deleted"}, + }, + } +} + +// FeedMetadata is a representation of the metadata returned when fetching a feed +type FeedMetadata struct { + NextPage string `json:"next_page"` + RemainingRecords int64 `json:"remaining_records"` +} + +// GetFeedParams is a list of all parameters we can set when fetching a feed +type GetFeedParams struct { + ModifiedAfter time.Time `url:"modified_after,omitempty"` + TemplateIDs []string `url:"template,omitempty"` + Archived string `url:"archived,omitempty"` + Completed string `url:"completed,omitempty"` + IncludeInactive bool `url:"include_inactive,omitempty"` + Limit int `url:"limit,omitempty"` + WebReportLink string `url:"web_report_link,omitempty"` + + // Applicable only for sites + IncludeDeleted bool `url:"include_deleted,omitempty"` + ShowOnlyLeafNodes *bool `url:"show_only_leaf_nodes,omitempty"` +} + +// GetFeedRequest has all the data needed to make a request to get a feed +type GetFeedRequest struct { + URL string + InitialURL string + Params GetFeedParams +} + +// GetFeedResponse is a representation of the data returned when fetching a feed +type GetFeedResponse struct { + Metadata FeedMetadata `json:"metadata"` + + Data json.RawMessage `json:"data"` +} + +// GetAccountsActivityLogRequestParams contains fields required to make a post request to activity log history api +type GetAccountsActivityLogRequestParams struct { + OrgID string `json:"org_id"` + PageSize int `json:"page_size"` + PageToken string `json:"page_token"` + Filters accountsActivityLogFilter `json:"filters"` +} + +// accountsActivityLogFilter filter for AccountsActivityLog +type accountsActivityLogFilter struct { + Timeframe timeFrame `json:"timeframe,omitempty"` + EventTypes []string `json:"event_types"` + Limit int `json:"limit"` +} + +type timeFrame struct { + From time.Time `json:"from"` +} + +// GetAccountsActivityLogResponse is the response from activity log history api +type GetAccountsActivityLogResponse struct { + Activities []activityResponse + NextPageToken string `json:"next_page_token"` +} + +type activityResponse struct { + Type string `json:"type"` + Metadata map[string]string `json:"metadata"` +} + +// GetMediaRequest has all the data needed to make a request to get a media +type GetMediaRequest struct { + URL string + AuditID string +} + +// GetMediaResponse is a representation of the data returned when fetching media +type GetMediaResponse struct { + ContentType string + Body []byte + MediaID string +} + +// ListInspectionsParams is a list of all parameters we can set when fetching inspections +type ListInspectionsParams struct { + ModifiedAfter time.Time `url:"modified_after,omitempty"` + TemplateIDs []string `url:"template,omitempty"` + Archived string `url:"archived,omitempty"` + Completed string `url:"completed,omitempty"` + Limit int `url:"limit,omitempty"` +} + +// Inspection represents some properties present in an inspection +type Inspection struct { + ID string `json:"audit_id"` + ModifiedAt time.Time `json:"modified_at"` +} + +// ListInspectionsResponse represents the response of listing inspections +type ListInspectionsResponse struct { + Count int `json:"count"` + Total int `json:"total"` + Inspections []Inspection `json:"audits"` +} + +// WhoAmIResponse represents the response of WhoAmI +type WhoAmIResponse struct { + UserID string `json:"user_id"` + OrganisationID string `json:"organisation_id"` + Firstname string `json:"firstname"` + Lastname string `json:"lastname"` +} + +// InspectionReportExportCompletionResponse represents the response of report export completion status +type InspectionReportExportCompletionResponse struct { + Status string `json:"status"` + URL string `json:"url,omitempty"` +} + +type initiateInspectionReportExportRequest struct { + Format string `json:"format"` + PreferenceID string `json:"preference_id,omitempty"` +} + +type initiateInspectionReportExportResponse struct { + MessageID string `json:"messageId"` +} diff --git a/internal/app/feed/export_feeds_intg_test.go b/internal/app/feed/export_feeds_intg_test.go index ef2c2246..95be36ca 100644 --- a/internal/app/feed/export_feeds_intg_test.go +++ b/internal/app/feed/export_feeds_intg_test.go @@ -4,6 +4,8 @@ package feed_test import ( + "net/http" + "path" "path/filepath" "testing" @@ -107,7 +109,6 @@ func TestIntegrationDbExportFeeds_should_export_all_feeds_to_file(t *testing.T) filesEqualish(t, "mocks/set_1/outputs/schedule_occurrences.csv", filepath.Join(exporter.ExportPath, "schedule_occurrences.csv")) filesEqualish(t, "mocks/set_1/outputs/issues.csv", filepath.Join(exporter.ExportPath, "issues.csv")) - } // Expectation of this test is that group_users and schedule_assignees are truncated and refreshed @@ -124,6 +125,24 @@ func TestIntegrationDbExportFeeds_should_perform_incremental_update_on_second_ru apiClient := api.GetTestClient() initMockFeedsSet1(apiClient.HTTPClient()) + gock.New("http://localhost:9999"). + Get("/accounts/user/v1/user:WhoAmI"). + Times(2). + Reply(200). + BodyString(` + { + "user_id": "user_123", + "organisation_id": "role_123", + "firstname": "Test", + "lastname": "Test" + } + `) + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + Reply(http.StatusOK). + File(path.Join("mocks", "set_2", "inspections_deleted_single_page.json")) + err = feed.ExportFeeds(viperConfig, apiClient, exporter) assert.Nil(t, err) @@ -163,6 +182,25 @@ func TestIntegrationDbExportFeeds_should_handle_lots_of_rows_ok(t *testing.T) { apiClient := api.GetTestClient() initMockFeedsSet3(apiClient.HTTPClient()) + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"0001-01-01T00:00:00Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + BodyString(`{"activites": []}`) + + gock.New("http://localhost:9999"). + Get("/accounts/user/v1/user:WhoAmI"). + Times(2). + Reply(200). + BodyString(` + { + "user_id": "user_123", + "organisation_id": "role_123", + "firstname": "Test", + "lastname": "Test" + } + `) + err = feed.ExportFeeds(viperConfig, apiClient, exporter) assert.Nil(t, err) @@ -199,6 +237,17 @@ func TestIntegrationDbExportFeeds_should_update_action_assignees_on_second_run(t apiClient := api.GetTestClient() initMockFeedsSet1(apiClient.HTTPClient()) + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"2014-03-17T11:35:40+11:00"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + File(path.Join("mocks", "set_1", "inspections_deleted_single_page.json")) + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"2014-03-17T00:35:40Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + File(path.Join("mocks", "set_1", "inspections_deleted_single_page.json")) err = feed.ExportFeeds(viperConfig, apiClient, exporter) assert.Nil(t, err) diff --git a/internal/app/feed/export_feeds_test.go b/internal/app/feed/export_feeds_test.go index 5ad43b19..bd1f9027 100644 --- a/internal/app/feed/export_feeds_test.go +++ b/internal/app/feed/export_feeds_test.go @@ -1,6 +1,8 @@ package feed_test import ( + "net/http" + "path" "path/filepath" "testing" @@ -104,6 +106,18 @@ func TestExportFeeds_should_perform_incremental_update_on_second_run(t *testing. } `) + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"0001-01-01T00:00:00Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + File(path.Join("mocks", "set_2", "inspections_deleted_single_page.json")) + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"2014-03-17T00:35:40Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + File(path.Join("mocks", "set_2", "inspections_deleted_single_page.json")) + exporter, err := getTemporaryCSVExporter() assert.Nil(t, err) @@ -180,6 +194,12 @@ func TestExportFeeds_should_handle_lots_of_rows_ok(t *testing.T) { apiClient := api.GetTestClient() initMockFeedsSet3(apiClient.HTTPClient()) + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"0001-01-01T00:00:00Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + File(path.Join("mocks", "set_3", "inspections_deleted_single_page.json")) + gock.New("http://localhost:9999"). Get("/accounts/user/v1/user:WhoAmI"). Reply(200). diff --git a/internal/app/feed/exporter_report_test.go b/internal/app/feed/exporter_report_test.go index 5c4b5f27..e9c63779 100644 --- a/internal/app/feed/exporter_report_test.go +++ b/internal/app/feed/exporter_report_test.go @@ -3,6 +3,7 @@ package feed_test import ( "bytes" "fmt" + "net/http" "os" "path/filepath" "testing" @@ -72,12 +73,12 @@ func TestExportReports_should_export_all_reports(t *testing.T) { assert.Nil(t, err) fileExists(t, filepath.Join(exporter.ExportPath, "My-Audit.pdf")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_2.pdf")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_3.pdf")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4e28ab2cce8c44a781d376d0ac47dc92.pdf")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4d95cb4be1e7488bba5893fecd2379d2.pdf")) fileExists(t, filepath.Join(exporter.ExportPath, "My-Audit.docx")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_2.docx")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_3.docx")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4e28ab2cce8c44a781d376d0ac47dc92.docx")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4d95cb4be1e7488bba5893fecd2379d2.docx")) } func TestExportReports_should_export_all_reports_with_ID_filename(t *testing.T) { @@ -125,13 +126,13 @@ func TestExportReports_should_export_all_reports_with_ID_filename(t *testing.T) err = feed.ExportInspectionReports(viperConfig, apiClient, exporter) assert.Nil(t, err) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_1.pdf")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_2.pdf")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_3.pdf")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_47ac0dce16f94d73b5178372368af162.pdf")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4e28ab2cce8c44a781d376d0ac47dc92.pdf")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4d95cb4be1e7488bba5893fecd2379d2.pdf")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_1.docx")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_2.docx")) - fileExists(t, filepath.Join(exporter.ExportPath, "audit_3.docx")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_47ac0dce16f94d73b5178372368af162.docx")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4e28ab2cce8c44a781d376d0ac47dc92.docx")) + fileExists(t, filepath.Join(exporter.ExportPath, "audit_4d95cb4be1e7488bba5893fecd2379d2.docx")) } func TestExportReports_should_not_run_if_all_exported(t *testing.T) { @@ -161,6 +162,12 @@ func TestExportReports_should_not_run_if_all_exported(t *testing.T) { } `) + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"2014-03-17T00:35:40Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + BodyString(`{"activites": []}`) + gock.New(mockAPIBaseURL). Post(initiateReportURL). Times(3). @@ -247,6 +254,12 @@ func TestExportReports_should_take_care_of_invalid_file_names(t *testing.T) { Reply(200). Body(bytes.NewBuffer([]byte(`file content`))) + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"0001-01-01T00:00:00Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + BodyString(`{"activites": []}`) + err = feed.ExportInspectionReports(viperConfig, apiClient, exporter) assert.Nil(t, err) diff --git a/internal/app/feed/exporter_sql.go b/internal/app/feed/exporter_sql.go index 64f3fe1b..6455444c 100644 --- a/internal/app/feed/exporter_sql.go +++ b/internal/app/feed/exporter_sql.go @@ -119,6 +119,19 @@ func (e *SQLExporter) WriteRows(feed Feed, rows interface{}) error { return nil } +// UpdateRows batch updates. Returns number of rows updated or error. Works with single PKey, not with composed PKeys +func (e *SQLExporter) UpdateRows(feed Feed, primaryKeys []string, element map[string]interface{}) (int64, error) { + result := e.DB. + Model(feed.Model()). + Where(primaryKeys). + Updates(element) + if result.Error != nil { + return 0, errors.Wrap(result.Error, "Unable to update rows") + } + + return result.RowsAffected, nil +} + type modifiedAtRow struct { // ExportedAt is here so gorm has an additional field to sort on in SQL Server ExportedAt time.Time @@ -179,12 +192,6 @@ func (e *SQLExporter) WriteMedia(auditID, mediaID, contentType string, body []by // NewSQLExporter creates a new instance of the SQLExporter func NewSQLExporter(dialect, connectionString string, autoMigrate bool, exportMediaPath string) (*SQLExporter, error) { - logger := util.GetLogger() - gormLogger := &util.GormLogger{ - SugaredLogger: logger, - SlowThreshold: 30 * time.Second, - } - var dialector gorm.Dialector switch dialect { case "mysql": @@ -199,16 +206,25 @@ func NewSQLExporter(dialect, connectionString string, autoMigrate bool, exportMe return nil, fmt.Errorf("invalid database dialect %s", dialect) } - db, err := gorm.Open(dialector, &gorm.Config{ - Logger: gormLogger, - }) + l := util.GetLogger() + gormLogger := &util.GormLogger{ + SugaredLogger: l, + SlowThreshold: 30 * time.Second, + } + + gormConfig := gorm.Config{ + Logger: gormLogger, //use logger.Default.LogMode(logger.Info) for checking the statements (gorm.io/logger) + } + + db, err := gorm.Open(dialector, &gormConfig) + if err != nil { return nil, errors.Wrap(err, "Unable to connect to DB") } return &SQLExporter{ DB: db, - Logger: logger, + Logger: l, AutoMigrate: autoMigrate, ExportMediaPath: exportMediaPath, duration: 0, diff --git a/internal/app/feed/feed.go b/internal/app/feed/feed.go index 7c175a10..1aa5b117 100644 --- a/internal/app/feed/feed.go +++ b/internal/app/feed/feed.go @@ -33,6 +33,8 @@ type Exporter interface { CreateSchema(feed Feed, rows interface{}) error WriteRows(feed Feed, rows interface{}) error + UpdateRows(feed Feed, primaryKeys []string, element map[string]interface{}) (int64, error) + FinaliseExport(feed Feed, rows interface{}) error LastModifiedAt(feed Feed, modifiedAfter time.Time, orgID string) (time.Time, error) WriteMedia(auditID string, mediaID string, contentType string, body []byte) error diff --git a/internal/app/feed/feed_inspection.go b/internal/app/feed/feed_inspection.go index 6bcf9ed7..ca0cef69 100644 --- a/internal/app/feed/feed_inspection.go +++ b/internal/app/feed/feed_inspection.go @@ -3,6 +3,8 @@ package feed import ( "context" "encoding/json" + "fmt" + "strings" "time" "github.com/SafetyCulture/iauditor-exporter/internal/app/api" @@ -42,6 +44,7 @@ type Inspection struct { Latitude *float64 `json:"latitude" csv:"latitude"` Longitude *float64 `json:"longitude" csv:"longitude"` WebReportLink string `json:"web_report_link" csv:"web_report_link"` + Deleted bool `json:"deleted" csv:"deleted"` } // InspectionFeed is a representation of the inspections feed @@ -109,6 +112,7 @@ func (f *InspectionFeed) Columns() []string { "latitude", "longitude", "web_report_link", + "deleted", } } @@ -117,7 +121,7 @@ func (f *InspectionFeed) Order() string { return "modified_at ASC, audit_id" } -func (f *InspectionFeed) writeRows(exporter Exporter, rows []*Inspection) error { +func (f *InspectionFeed) writeRows(exporter Exporter, rows []Inspection) error { skipIDs := map[string]bool{} for _, id := range f.SkipIDs { skipIDs[id] = true @@ -133,7 +137,7 @@ func (f *InspectionFeed) writeRows(exporter Exporter, rows []*Inspection) error // Some audits in production have the same item ID multiple times // We can't insert them simultaneously. This means we are dropping data, which sucks. - rowsToInsert := []*Inspection{} + var rowsToInsert []Inspection for _, row := range rows[i:j] { skip := skipIDs[row.ID] if !skip { @@ -171,7 +175,22 @@ func (f *InspectionFeed) Export(ctx context.Context, apiClient *api.Client, expo logger.Infof("%s: exporting for org_id: %s since: %s - %s", feedName, orgID, f.ModifiedAfter.Format(time.RFC1123), f.WebReportLink) - err = apiClient.DrainFeed(ctx, &api.GetFeedRequest{ + // Process Inspections + err = f.processNewInspections(ctx, apiClient, exporter) + util.Check(err, "Failed to export feed") + + // Process Deleted Inspections + err = f.processDeletedInspections(ctx, apiClient, exporter) + if err != nil { + logger.Errorf("failed to process deleted inspections. %v", err.Error()) + } + + return exporter.FinaliseExport(f, &[]*Inspection{}) +} + +func (f *InspectionFeed) processNewInspections(ctx context.Context, apiClient *api.Client, exporter Exporter) error { + lg := util.GetLogger() + req := api.GetFeedRequest{ InitialURL: "/feed/inspections", Params: api.GetFeedParams{ ModifiedAfter: f.ModifiedAfter, @@ -181,25 +200,55 @@ func (f *InspectionFeed) Export(ctx context.Context, apiClient *api.Client, expo Limit: f.Limit, WebReportLink: f.WebReportLink, }, - }, func(resp *api.GetFeedResponse) error { - rows := []*Inspection{} + } + feedFn := func(resp *api.GetFeedResponse) error { + var rows []Inspection err := json.Unmarshal(resp.Data, &rows) util.Check(err, "Failed to unmarshal inspections data to struct") if len(rows) != 0 { err = f.writeRows(exporter, rows) - util.Check(err, "Failed to write data to exporter") + if err != nil { + return err + } } - logger.Info(GetLogString(f.Name(), &LogStringConfig{ + lg.Info(GetLogString(f.Name(), &LogStringConfig{ RemainingRecords: resp.Metadata.RemainingRecords, HTTPDuration: apiClient.Duration, ExporterDuration: exporter.GetDuration(), })) return nil - }) - util.Check(err, "Failed to export feed") + } - return exporter.FinaliseExport(f, &[]*Inspection{}) + return apiClient.DrainFeed(ctx, &req, feedFn) +} + +func (f *InspectionFeed) processDeletedInspections(ctx context.Context, apiClient *api.Client, exporter Exporter) error { + lg := util.GetLogger() + dreq := api.NewGetAccountsActivityLogRequest(f.Limit, f.ModifiedAfter) + delFn := func(resp *api.GetAccountsActivityLogResponse) error { + var pkeys = make([]string, 0, len(resp.Activities)) + for _, a := range resp.Activities { + uid := getPrefixID(a.Metadata["inspection_id"]) + if uid != "" { + pkeys = append(pkeys, uid) + } + } + if len(pkeys) > 0 { + rowsUpdated, err := exporter.UpdateRows(f, pkeys, map[string]interface{}{"deleted": true}) + if err != nil { + return err + } + lg.Infof("there were %d rows marked as deleted", rowsUpdated) + } + + return nil + } + return apiClient.DrainAccountActivityHistoryLog(ctx, dreq, delFn) +} + +func getPrefixID(id string) string { + return fmt.Sprintf("audit_%s", strings.ReplaceAll(id, "-", "")) } diff --git a/internal/app/feed/feed_inspection_test.go b/internal/app/feed/feed_inspection_test.go index e0a1b833..e991a0d2 100644 --- a/internal/app/feed/feed_inspection_test.go +++ b/internal/app/feed/feed_inspection_test.go @@ -2,6 +2,9 @@ package feed_test import ( "context" + "gopkg.in/h2non/gock.v1" + "net/http" + "path" "testing" "time" @@ -17,7 +20,10 @@ func TestInspectionFeedExport_should_export_rows_to_sql_db(t *testing.T) { apiClient := api.GetTestClient() initMockFeedsSet1(apiClient.HTTPClient()) - + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + Reply(http.StatusOK). + File(path.Join("mocks", "set_1", "inspections_deleted_single_page.json")) inspectionsFeed := feed.InspectionFeed{ SkipIDs: []string{}, ModifiedAfter: time.Now(), @@ -35,5 +41,12 @@ func TestInspectionFeedExport_should_export_rows_to_sql_db(t *testing.T) { assert.Nil(t, resp.Error) assert.Equal(t, 3, len(rows)) - assert.Equal(t, "audit_1", rows[0].ID) + assert.Equal(t, "audit_47ac0dce16f94d73b5178372368af162", rows[0].ID) + assert.True(t, rows[0].Deleted) + + assert.Equal(t, "audit_4e28ab2cce8c44a781d376d0ac47dc92", rows[1].ID) + assert.False(t, rows[1].Deleted) + + assert.Equal(t, "audit_4d95cb4be1e7488bba5893fecd2379d2", rows[2].ID) + assert.False(t, rows[2].Deleted) } diff --git a/internal/app/feed/mock_feeds_test.go b/internal/app/feed/mock_feeds_test.go index db5ae352..0eba9dce 100644 --- a/internal/app/feed/mock_feeds_test.go +++ b/internal/app/feed/mock_feeds_test.go @@ -1,9 +1,9 @@ package feed_test import ( - "net/http" - "gopkg.in/h2non/gock.v1" + "net/http" + "path" ) func initMockFeedsSet1(httpClient *http.Client) { @@ -93,6 +93,12 @@ func initMockFeedsSet1(httpClient *http.Client) { Get("/feed/issues"). Reply(200). File("mocks/set_1/feed_issues_2.json") + + gock.New("http://localhost:9999"). + Post("/accounts/history/v1/activity_log/list"). + BodyString(`{"org_id":"","page_size":0,"page_token":"","filters":{"timeframe":{"from":"0001-01-01T00:00:00Z"},"event_types":["inspection.deleted"],"limit":0}}`). + Reply(http.StatusOK). + File(path.Join("mocks", "set_1", "inspections_deleted_single_page.json")) } func initMockFeedsSet2(httpClient *http.Client) { diff --git a/internal/app/feed/mocks/set_1/feed_inspections_1.json b/internal/app/feed/mocks/set_1/feed_inspections_1.json index 57a12bc0..4f01aa07 100644 --- a/internal/app/feed/mocks/set_1/feed_inspections_1.json +++ b/internal/app/feed/mocks/set_1/feed_inspections_1.json @@ -5,7 +5,7 @@ }, "data": [ { - "id": "audit_1", + "id": "audit_47ac0dce16f94d73b5178372368af162", "name": "My Audit", "archived": true, "owner_name": "A User", @@ -32,11 +32,11 @@ "client_site": null, "latitude": null, "longitude": null, - "web_report_link": "https://app.safetyculture.io/report/audit/audit_1", + "web_report_link": "https://app.safetyculture.io/report/audit/audit_47ac0dce16f94d73b5178372368af162", "organisation_id": "role_123" }, { - "id": "audit_2", + "id": "audit_4e28ab2cce8c44a781d376d0ac47dc92", "name": "", "archived": true, "owner_name": "A User", @@ -63,7 +63,7 @@ "client_site": null, "latitude": -33.8858784, "longitude": 151.2116864, - "web_report_link": "https://app.safetyculture.io/report/audit/audit_2", + "web_report_link": "https://app.safetyculture.io/report/audit/audit_4e28ab2cce8c44a781d376d0ac47dc92", "organisation_id": "role_123" } ] diff --git a/internal/app/feed/mocks/set_1/feed_inspections_2.json b/internal/app/feed/mocks/set_1/feed_inspections_2.json index 0ac76f14..69f7a3fa 100644 --- a/internal/app/feed/mocks/set_1/feed_inspections_2.json +++ b/internal/app/feed/mocks/set_1/feed_inspections_2.json @@ -5,7 +5,7 @@ }, "data": [ { - "id": "audit_2", + "id": "audit_4e28ab2cce8c44a781d376d0ac47dc92", "name": "", "archived": true, "owner_name": "A User", @@ -32,11 +32,11 @@ "client_site": null, "latitude": -33.8858784, "longitude": 151.2116864, - "web_report_link": "https://app.safetyculture.io/report/audit/audit_2", + "web_report_link": "https://app.safetyculture.io/report/audit/audit_4e28ab2cce8c44a781d376d0ac47dc92", "organisation_id": "role_123" }, { - "id": "audit_3", + "id": "audit_4d95cb4be1e7488bba5893fecd2379d2", "name": "", "archived": true, "owner_name": "Anonymous", @@ -63,7 +63,7 @@ "client_site": null, "latitude": null, "longitude": null, - "web_report_link": "https://app.safetyculture.io/report/audit/audit_3", + "web_report_link": "https://app.safetyculture.io/report/audit/audit_4d95cb4be1e7488bba5893fecd2379d2", "organisation_id": "role_123" } ] diff --git a/internal/app/feed/mocks/set_1/inspections_deleted_single_page.json b/internal/app/feed/mocks/set_1/inspections_deleted_single_page.json new file mode 100644 index 00000000..5c614f95 --- /dev/null +++ b/internal/app/feed/mocks/set_1/inspections_deleted_single_page.json @@ -0,0 +1,21 @@ +{ + "activities": [ + { + "id": "a7f7990e-a3d8-4985-b3c1-fb137b192a2c", + "event_at": "2022-07-05T01:02:10.887Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "47ac0dce-16f9-4d73-b517-8372368af162", + "inspection_name": "5 Jul 2022 / #4 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "" +} diff --git a/internal/app/feed/mocks/set_1/outputs/inspections.csv b/internal/app/feed/mocks/set_1/outputs/inspections.csv index f14f6eb9..3fc922e7 100644 --- a/internal/app/feed/mocks/set_1/outputs/inspections.csv +++ b/internal/app/feed/mocks/set_1/outputs/inspections.csv @@ -1,4 +1,4 @@ -audit_id,name,archived,owner_name,owner_id,author_name,author_id,score,max_score,score_percentage,duration,template_id,organisation_id,template_name,template_author,site_id,date_started,date_completed,date_modified,created_at,modified_at,exported_at,document_no,prepared_by,location,conducted_on,personnel,client_site,latitude,longitude,web_report_link -audit_1,My Audit,true,A User,user_1,A User,user_1,0,107,0,61,template_1,role_123,General Workplace Inspection,Anonymous,,2014-01-28T23:14:23Z,,2014-01-28T23:15:24Z,2014-01-28T23:14:23Z,2014-01-28T23:14:23Z,2020-11-06T08:21:16.469988+11:00,,,,2014-01-28T23:14:23Z,,,,,https://app.safetyculture.io/report/audit/audit_1 -audit_2,,true,A User,user_1,Another User,user_2,59,141,0,183,template_2,role_123,Group 9 - Townsville Tourism Walking Tour,A User,,2014-03-06T04:47:26Z,,2014-03-07T02:43:15Z,2014-03-06T04:47:26Z,2014-03-06T04:47:26Z,2020-11-06T08:21:16.492024+11:00,,,,2014-03-06T04:47:26Z,,,-33.8858784,151.2116864,https://app.safetyculture.io/report/audit/audit_2 -audit_3,,true,Anonymous,user_3,Anonymous,user_3,0,1,0,2,template_3,role_123,,A User,,2014-03-17T00:35:40Z,,2014-03-17T00:35:40Z,2014-03-17T00:35:40Z,2014-03-17T00:35:40Z,2020-11-06T08:21:16.492024+11:00,000001,,,2014-03-17T00:35:40Z,,,,,https://app.safetyculture.io/report/audit/audit_3 +audit_id,name,archived,owner_name,owner_id,author_name,author_id,score,max_score,score_percentage,duration,template_id,organisation_id,template_name,template_author,site_id,date_started,date_completed,date_modified,created_at,modified_at,exported_at,document_no,prepared_by,location,conducted_on,personnel,client_site,latitude,longitude,web_report_link,deleted +audit_47ac0dce16f94d73b5178372368af162,My Audit,true,A User,user_1,A User,user_1,0,107,0,61,template_1,role_123,General Workplace Inspection,Anonymous,,2014-01-28T23:14:23Z,,2014-01-28T23:15:24Z,2014-01-28T23:14:23Z,2014-01-28T23:14:23Z,2020-11-06T08:21:16.469988+11:00,,,,2014-01-28T23:14:23Z,,,,,https://app.safetyculture.io/report/audit/audit_47ac0dce16f94d73b5178372368af162,true +audit_4e28ab2cce8c44a781d376d0ac47dc92,,true,A User,user_1,Another User,user_2,59,141,0,183,template_2,role_123,Group 9 - Townsville Tourism Walking Tour,A User,,2014-03-06T04:47:26Z,,2014-03-07T02:43:15Z,2014-03-06T04:47:26Z,2014-03-06T04:47:26Z,2020-11-06T08:21:16.492024+11:00,,,,2014-03-06T04:47:26Z,,,-33.8858784,151.2116864,https://app.safetyculture.io/report/audit/audit_4e28ab2cce8c44a781d376d0ac47dc92,false +audit_4d95cb4be1e7488bba5893fecd2379d2,,true,Anonymous,user_3,Anonymous,user_3,0,1,0,2,template_3,role_123,,A User,,2014-03-17T00:35:40Z,,2014-03-17T00:35:40Z,2014-03-17T00:35:40Z,2014-03-17T00:35:40Z,2020-11-06T08:21:16.492024+11:00,000001,,,2014-03-17T00:35:40Z,,,,,https://app.safetyculture.io/report/audit/audit_4d95cb4be1e7488bba5893fecd2379d2,false diff --git a/internal/app/feed/mocks/set_1/schemas/formatted/inspections.txt b/internal/app/feed/mocks/set_1/schemas/formatted/inspections.txt index 01592cc3..df105b10 100644 --- a/internal/app/feed/mocks/set_1/schemas/formatted/inspections.txt +++ b/internal/app/feed/mocks/set_1/schemas/formatted/inspections.txt @@ -32,4 +32,5 @@ | latitude | REAL | | | longitude | REAL | | | web_report_link | TEXT | | +| deleted | numeric | | +------------------+----------+-------------+ diff --git a/internal/app/feed/mocks/set_1/schemas/inspections.csv b/internal/app/feed/mocks/set_1/schemas/inspections.csv index 927adeaf..fff2da6e 100644 --- a/internal/app/feed/mocks/set_1/schemas/inspections.csv +++ b/internal/app/feed/mocks/set_1/schemas/inspections.csv @@ -1 +1 @@ -audit_id,name,archived,owner_name,owner_id,author_name,author_id,score,max_score,score_percentage,duration,template_id,organisation_id,template_name,template_author,site_id,date_started,date_completed,date_modified,created_at,modified_at,exported_at,document_no,prepared_by,location,conducted_on,personnel,client_site,latitude,longitude,web_report_link +audit_id,name,archived,owner_name,owner_id,author_name,author_id,score,max_score,score_percentage,duration,template_id,organisation_id,template_name,template_author,site_id,date_started,date_completed,date_modified,created_at,modified_at,exported_at,document_no,prepared_by,location,conducted_on,personnel,client_site,latitude,longitude,web_report_link,deleted diff --git a/internal/app/feed/mocks/set_2/inspections_deleted_single_page.json b/internal/app/feed/mocks/set_2/inspections_deleted_single_page.json new file mode 100644 index 00000000..f98c0a94 --- /dev/null +++ b/internal/app/feed/mocks/set_2/inspections_deleted_single_page.json @@ -0,0 +1,37 @@ +{ + "activities": [ + { + "id": "24fbaead-6d4b-4cd8-992c-8f71d67a90f4", + "event_at": "2022-07-04T03:48:05.044Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "47ac0dce-16f9-4d73-b517-8372368af162", + "inspection_name": "4 Jul 2022 / #3 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "ca54b734-75ee-44db-8ac5-7d629e5f234d", + "event_at": "2022-07-04T03:47:44.810Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "4d2f8bd5-a965-4046-b2b4-ccdf8341c9f2", + "inspection_name": "4 Jul 2022 / #2 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "" +} diff --git a/internal/app/feed/mocks/set_2/outputs/inspections.csv b/internal/app/feed/mocks/set_2/outputs/inspections.csv index d4f3ffe8..be7ad549 100644 --- a/internal/app/feed/mocks/set_2/outputs/inspections.csv +++ b/internal/app/feed/mocks/set_2/outputs/inspections.csv @@ -1,4 +1,4 @@ -audit_id,name,archived,owner_name,owner_id,author_name,author_id,score,max_score,score_percentage,duration,template_id,organisation_id,template_name,template_author,site_id,date_started,date_completed,date_modified,created_at,modified_at,exported_at,document_no,prepared_by,location,conducted_on,personnel,client_site,latitude,longitude,web_report_link -audit_1,My Audit,true,A User,user_1,A User,user_1,0,107,0,61,template_1,role_123,General Workplace Inspection,Anonymous,,--date--,,--date--,--date--,--date--,--date--,,,,--date--,,,,,https://app.safetyculture.io/report/audit/audit_1 -audit_2,,true,A User,user_1,Another User,user_2,59,141,0,183,template_2,role_123,Group 9 - Townsville Tourism Walking Tour,A User,,--date--,,--date--,--date--,--date--,--date--,,,,--date--,,,-33.8858784,151.2116864,https://app.safetyculture.io/report/audit/audit_2 -audit_3,,true,Anonymous,user_3,Anonymous,user_3,0,1,0,2,template_3,role_123,,A User,,--date--,,--date--,--date--,--date--,--date--,000001,,,--date--,,,,,https://app.safetyculture.io/report/audit/audit_3 +audit_id,name,archived,owner_name,owner_id,author_name,author_id,score,max_score,score_percentage,duration,template_id,organisation_id,template_name,template_author,site_id,date_started,date_completed,date_modified,created_at,modified_at,exported_at,document_no,prepared_by,location,conducted_on,personnel,client_site,latitude,longitude,web_report_link,deleted +audit_47ac0dce16f94d73b5178372368af162,My Audit,true,A User,user_1,A User,user_1,0,107,0,61,template_1,role_123,General Workplace Inspection,Anonymous,,--date--,,--date--,--date--,--date--,--date--,,,,--date--,,,,,https://app.safetyculture.io/report/audit/audit_47ac0dce16f94d73b5178372368af162,true +audit_4e28ab2cce8c44a781d376d0ac47dc92,,true,A User,user_1,Another User,user_2,59,141,0,183,template_2,role_123,Group 9 - Townsville Tourism Walking Tour,A User,,--date--,,--date--,--date--,--date--,--date--,,,,--date--,,,-33.8858784,151.2116864,https://app.safetyculture.io/report/audit/audit_4e28ab2cce8c44a781d376d0ac47dc92,false +audit_4d95cb4be1e7488bba5893fecd2379d2,,true,Anonymous,user_3,Anonymous,user_3,0,1,0,2,template_3,role_123,,A User,,--date--,,--date--,--date--,--date--,--date--,000001,,,--date--,,,,,https://app.safetyculture.io/report/audit/audit_4d95cb4be1e7488bba5893fecd2379d2,false diff --git a/internal/app/feed/mocks/set_3/inspections_deleted_single_page.json b/internal/app/feed/mocks/set_3/inspections_deleted_single_page.json new file mode 100644 index 00000000..5c614f95 --- /dev/null +++ b/internal/app/feed/mocks/set_3/inspections_deleted_single_page.json @@ -0,0 +1,21 @@ +{ + "activities": [ + { + "id": "a7f7990e-a3d8-4985-b3c1-fb137b192a2c", + "event_at": "2022-07-05T01:02:10.887Z", + "type": "inspection.deleted", + "user_id": "7d4f9200-bf23-4642-941b-248623bb586e", + "org_id": "30bafca8-9f56-4cbe-9e20-b2f4e586f238", + "client_class": "", + "agent": "", + "trace_id": "", + "metadata": { + "inspection_id": "47ac0dce-16f9-4d73-b517-8372368af162", + "inspection_name": "5 Jul 2022 / #4 M S" + }, + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ], + "next_page_token": "" +}