-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[INTG-2143] consume issues feed (#216)
* INTG-2143 add unit tests * INTG-2143 enable issues export * INTG-2143 wip tests 1 * INTG-2143 integration tests are passing * INTG-2143 go doc * INTG-2143 go doc
- Loading branch information
1 parent
7fc0751
commit d99e696
Showing
13 changed files
with
1,156 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| package feed | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "github.com/SafetyCulture/iauditor-exporter/internal/app/api" | ||
| "github.com/SafetyCulture/iauditor-exporter/internal/app/util" | ||
| "time" | ||
| ) | ||
|
|
||
| const ( | ||
| feedName = "issues" | ||
| feedURL = "/feed/issues" | ||
| ) | ||
|
|
||
| // Issue represents a row from the issues feed | ||
| type Issue struct { | ||
| ID string `json:"id" csv:"id" gorm:"primarykey;column:id;size:36"` | ||
| Title string `json:"title" csv:"title"` | ||
| Description string `json:"description" csv:"description"` | ||
| CreatorID string `json:"creator_id" csv:"creator_id"` | ||
| CreatorUserName string `json:"creator_user_name"` | ||
| CreatedAt time.Time `json:"created_at" csv:"created_at"` | ||
| DueAt *time.Time `json:"due_at,omitempty" csv:"due_at"` | ||
| Priority string `json:"priority" csv:"priority"` | ||
| Status string `json:"status" csv:"status"` | ||
| TemplateID string `json:"template_id" csv:"template_id"` | ||
| InspectionID string `json:"inspection_id" csv:"inspection_id"` | ||
| InspectionName string `json:"inspection_name" csv:"inspection_name"` | ||
| SiteID string `json:"site_id" csv:"site_id"` | ||
| SiteName string `json:"site_name" csv:"site_name"` | ||
| LocationName string `json:"location_name" csv:"location_name"` | ||
| CategoryID string `json:"category_id" csv:"category_id"` | ||
| CategoryLabel string `json:"category_label" csv:"category_label"` | ||
| } | ||
|
|
||
| // IssueFeed is a representation of the issues feed | ||
| type IssueFeed struct { | ||
| Limit int | ||
| Incremental bool | ||
| } | ||
|
|
||
| // Name returns the name of the feed | ||
| func (f *IssueFeed) Name() string { | ||
| return feedName | ||
| } | ||
|
|
||
| // Model returns the model of the feed row | ||
| func (f *IssueFeed) Model() interface{} { | ||
| return Issue{} | ||
| } | ||
|
|
||
| // RowsModel returns the model of the feed rows | ||
| func (f *IssueFeed) RowsModel() interface{} { | ||
| return &[]*Issue{} | ||
| } | ||
|
|
||
| // PrimaryKey return the primary key | ||
| func (f *IssueFeed) PrimaryKey() []string { | ||
| return []string{"id"} | ||
| } | ||
|
|
||
| // Columns returns the columns of the row | ||
| func (f *IssueFeed) Columns() []string { | ||
| return []string{ | ||
| "id", "title", "description", "creator_id", "creator_user_name", | ||
| "created_at", "due_at", "priority", "status", "template_id", | ||
| "inspection_id", "inspection_name", "site_id", "site_name", | ||
| "location_name", "category_id", "category_label", | ||
| } | ||
| } | ||
|
|
||
| // Order returns the ordering when retrieving an export | ||
| func (f *IssueFeed) Order() string { | ||
| return "id" | ||
| } | ||
|
|
||
| // CreateSchema creates the schema of the feed for the supplied exporter | ||
| func (f *IssueFeed) CreateSchema(exporter Exporter) error { | ||
| return exporter.CreateSchema(f, &[]*Issue{}) | ||
| } | ||
|
|
||
| // Export exports the feed to the supplied exporter | ||
| func (f *IssueFeed) Export(ctx context.Context, apiClient *api.Client, exporter Exporter, orgID string) error { | ||
| logger := util.GetLogger() | ||
|
|
||
| _ = exporter.InitFeed(f, &InitFeedOptions{ | ||
| // Delete data if incremental refresh is disabled so there is no duplicates | ||
| Truncate: !f.Incremental, | ||
| }) | ||
|
|
||
| var request = &api.GetFeedRequest{ | ||
| InitialURL: feedURL, | ||
| Params: api.GetFeedParams{ | ||
| Limit: f.Limit, | ||
| }, | ||
| } | ||
|
|
||
| var feedFn = func(resp *api.GetFeedResponse) error { | ||
| var rows []*Issue | ||
| err := json.Unmarshal(resp.Data, &rows) | ||
| util.Check(err, "Failed to unmarshal actions data to struct") | ||
|
|
||
| if len(rows) != 0 { | ||
| // Calculate the size of the batch we can insert into the DB at once. | ||
| // Column count + buffer to account for primary keys | ||
| batchSize := exporter.ParameterLimit() / (len(f.Columns()) + 4) | ||
|
|
||
| for i := 0; i < len(rows); i += batchSize { | ||
| j := i + batchSize | ||
| if j > len(rows) { | ||
| j = len(rows) | ||
| } | ||
|
|
||
| err = exporter.WriteRows(f, rows[i:j]) | ||
| util.Check(err, "Failed to write data to exporter") | ||
| } | ||
| } | ||
|
|
||
| logger.Infof("%s: %d remaining", f.Name(), resp.Metadata.RemainingRecords) | ||
| return nil | ||
| } | ||
|
|
||
| err := apiClient.DrainFeed(ctx, request, feedFn) | ||
| util.Check(err, "Failed to export feed") | ||
| return exporter.FinaliseExport(f, &[]*Issue{}) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| package feed_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "github.com/SafetyCulture/iauditor-exporter/internal/app/api" | ||
| "github.com/SafetyCulture/iauditor-exporter/internal/app/feed" | ||
| "github.com/stretchr/testify/assert" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| func TestIssueFeed_Export_ShouldExportRows(t *testing.T) { | ||
| exporter, err := getInmemorySQLExporter("") | ||
| assert.Nil(t, err) | ||
|
|
||
| apiClient := api.GetTestClient() | ||
| defer resetMocks(apiClient.HTTPClient()) | ||
| initMockIssuesFeed(apiClient.HTTPClient()) | ||
|
|
||
| actionsFeed := feed.IssueFeed{ | ||
| Limit: 100, | ||
| } | ||
|
|
||
| err = actionsFeed.Export(context.Background(), apiClient, exporter, "") | ||
| assert.Nil(t, err) | ||
|
|
||
| var rows []feed.Issue | ||
| resp := exporter.DB.Table("issues").Scan(&rows) | ||
|
|
||
| assert.Nil(t, resp.Error) | ||
| assert.Equal(t, 39, len(rows)) | ||
| testAllValues(t, &rows[0]) | ||
| testAllNulls(t, &rows[1]) | ||
|
|
||
| } | ||
|
|
||
| func testAllNulls(t *testing.T, issue *feed.Issue) { | ||
| assert.Equal(t, "52a88aeb-5ec6-4876-8c6c-85a642e4bddc", issue.ID) | ||
| assert.Equal(t, "", issue.Title) | ||
| assert.Equal(t, "", issue.Description) | ||
| assert.Equal(t, "user_0590e8a0dfbc64798a2426c2fa76a7415", issue.CreatorID) | ||
| assert.Equal(t, "", issue.CreatorUserName) | ||
|
|
||
| // uses .Now() if missing | ||
| assert.NotEqual(t, time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), issue.CreatedAt) | ||
|
|
||
| // not sure how correct is this approach | ||
| //assert.Equal(t, time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), issue.DueAt) | ||
| assert.Nil(t, issue.DueAt) | ||
|
|
||
| assert.Equal(t, "", issue.Priority) | ||
| assert.Equal(t, "", issue.Status) | ||
| assert.Equal(t, "", issue.TemplateID) | ||
| assert.Equal(t, "", issue.InspectionID) | ||
| assert.Equal(t, "", issue.InspectionName) | ||
| assert.Equal(t, "", issue.SiteID) | ||
| assert.Equal(t, "", issue.SiteName) | ||
| assert.Equal(t, "", issue.LocationName) | ||
| assert.Equal(t, "", issue.CategoryID) | ||
| assert.Equal(t, "", issue.CategoryLabel) | ||
| } | ||
|
|
||
| func testAllValues(t *testing.T, issue *feed.Issue) { | ||
| assert.Equal(t, "56bc5efa-2420-483d-bad1-27b35922c403", issue.ID) | ||
| assert.Equal(t, "Injury - 14 Apr 2020, 10:36 am", issue.Title) | ||
| assert.Equal(t, "some description", issue.Description) | ||
| assert.Equal(t, "user_51d3dbc686eb4790980f6414513d1c05", issue.CreatorID) | ||
| assert.Equal(t, "🦄", issue.CreatorUserName) | ||
| assert.Equal(t, time.Date(2020, time.April, 14, 0, 36, 53, 304000000, time.UTC), issue.CreatedAt) | ||
| expected := time.Date(2020, time.April, 14, 0, 36, 53, 304000000, time.UTC) | ||
| assert.Equal(t, &expected, issue.DueAt) | ||
| assert.Equal(t, "NONE", issue.Priority) | ||
| assert.Equal(t, "OPEN", issue.Status) | ||
| assert.Equal(t, "55bc5efa-2420-483d-bad1-27b35922c455", issue.TemplateID) | ||
| assert.Equal(t, "66bc5efa-2420-483d-bad1-27b35922c466", issue.InspectionID) | ||
| assert.Equal(t, "some name", issue.InspectionName) | ||
| assert.Equal(t, "77bc5efa-2420-483d-bad1-27b35922c477", issue.SiteID) | ||
| assert.Equal(t, "site name", issue.SiteName) | ||
| assert.Equal(t, "88bc5efa-2420-483d-bad1-27b35922c488", issue.LocationName) | ||
| assert.Equal(t, "592ec130-90e0-4c0e-a1c0-1f37f12f5fb5", issue.CategoryID) | ||
| assert.Equal(t, "Tow Trucks", issue.CategoryLabel) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.