diff --git a/README.md b/README.md index 8609d2ca..3d8e65b0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ This way we can preserve the CSV columns in the export files. ### Testing -Locally you can run `go test ./...`, this will run all of the Unit tests and Integration tests that can be run without an external DB. +Locally you can run `go test ./...`, this will run all the Unit tests and Integration tests that can be run without an external DB. SQL Database integration tests can be run by starting the SQL DBs `docker-compose up -d` and then running `make integration-tests`. diff --git a/pkg/api/configuration_manager.go b/pkg/api/configuration_manager.go index aace050e..309f195a 100644 --- a/pkg/api/configuration_manager.go +++ b/pkg/api/configuration_manager.go @@ -39,6 +39,12 @@ type ExporterConfiguration struct { Asset struct { Limit int `yaml:"limit"` } `yaml:"asset"` + Course struct { + Progress struct { + Limit int `yaml:"limit"` + CompletionStatus string `yaml:"completion_status"` + } `yaml:"progress"` + } `yaml:"course"` Incremental bool `yaml:"incremental"` Inspection struct { Archived string `yaml:"archived"` @@ -161,6 +167,15 @@ func (c *ConfigurationManager) ApplySafetyGuards() { c.Configuration.Export.Issue.Limit = defaultCfg.Export.Issue.Limit } + // caps course progress batch limit to 1000 + if c.Configuration.Export.Course.Progress.Limit > 1000 || c.Configuration.Export.Course.Progress.Limit == 0 { + c.Configuration.Export.Course.Progress.Limit = defaultCfg.Export.Course.Progress.Limit + } + + if c.Configuration.Export.Course.Progress.CompletionStatus == "" { + c.Configuration.Export.Course.Progress.CompletionStatus = defaultCfg.Export.Course.Progress.CompletionStatus + } + if c.Configuration.Export.Inspection.Limit == 0 { c.Configuration.Export.Inspection.Limit = defaultCfg.Export.Inspection.Limit } @@ -257,6 +272,8 @@ func BuildConfigurationWithDefaults() *ExporterConfiguration { cfg.Export.TemplateIds = []string{} cfg.Export.Action.Limit = 100 cfg.Export.Asset.Limit = 100 + cfg.Export.Course.Progress.Limit = 1000 + cfg.Export.Course.Progress.CompletionStatus = "COMPLETION_STATUS_COMPLETED" cfg.Export.Incremental = true cfg.Export.Inspection.Archived = "false" cfg.Export.Inspection.Completed = "true" @@ -341,6 +358,7 @@ func (ec *ExporterConfiguration) ToExporterConfig() *feed.ExporterFeedCfg { ExportSiteIncludeFullHierarchy: ec.Export.Site.IncludeFullHierarchy, ExportIssueLimit: ec.Export.Issue.Limit, ExportAssetLimit: ec.Export.Asset.Limit, + ExportCourseProgressLimit: ec.Export.Course.Progress.Limit, MaxConcurrentGoRoutines: ec.API.MaxConcurrency, } } diff --git a/pkg/api/configuration_manager_test.go b/pkg/api/configuration_manager_test.go index ded687fd..c9188123 100644 --- a/pkg/api/configuration_manager_test.go +++ b/pkg/api/configuration_manager_test.go @@ -116,6 +116,8 @@ func TestNewConfigurationManagerFromFile_when_filename_exists_without_time(t *te assert.Equal(t, "true", cfg.Export.Inspection.Completed) assert.False(t, cfg.Export.Inspection.IncludedInactiveItems) assert.Equal(t, 100, cfg.Export.Inspection.Limit) + assert.Equal(t, 1000, cfg.Export.Course.Progress.Limit) + assert.Equal(t, "COMPLETION_STATUS_COMPLETED", cfg.Export.Course.Progress.CompletionStatus) assert.Equal(t, []string{"ID1", "ID2"}, cfg.Export.Inspection.SkipIds) assert.Equal(t, "private", cfg.Export.Inspection.WebReportLink) assert.False(t, cfg.Export.Media) @@ -165,6 +167,8 @@ func TestConfigurationManager_SaveConfiguration(t *testing.T) { assert.EqualValues(t, "https://app.sheqsy.com", newCm.Configuration.API.SheqsyURL) assert.EqualValues(t, 1000000, newCm.Configuration.Csv.MaxRowsPerFile) assert.EqualValues(t, 100, newCm.Configuration.Export.Action.Limit) + assert.EqualValues(t, 1000, newCm.Configuration.Export.Course.Progress.Limit) + assert.EqualValues(t, "COMPLETION_STATUS_COMPLETED", newCm.Configuration.Export.Course.Progress.CompletionStatus) assert.True(t, newCm.Configuration.Export.Incremental) assert.EqualValues(t, "false", newCm.Configuration.Export.Inspection.Archived) assert.EqualValues(t, "true", newCm.Configuration.Export.Inspection.Completed) @@ -185,6 +189,7 @@ func TestMapViperConfigToConfigurationOptions_ShouldRespectLimit(t *testing.T) { require.NotNil(t, cm) assert.EqualValues(t, 50, cm.Configuration.Export.Action.Limit) assert.EqualValues(t, 50, cm.Configuration.Export.Issue.Limit) + assert.EqualValues(t, 50, cm.Configuration.Export.Course.Progress.Limit) } func TestMapViperConfigToConfigurationOptions_ShouldEnforceLimit(t *testing.T) { @@ -193,6 +198,14 @@ func TestMapViperConfigToConfigurationOptions_ShouldEnforceLimit(t *testing.T) { require.NotNil(t, cm) assert.EqualValues(t, 100, cm.Configuration.Export.Action.Limit) assert.EqualValues(t, 100, cm.Configuration.Export.Issue.Limit) + assert.EqualValues(t, 1000, cm.Configuration.Export.Course.Progress.Limit) +} + +func TestMapViperConfigToConfigurationOptions_CustomValues(t *testing.T) { + cm, err := api.NewConfigurationManagerFromFile("", "fixtures/test_custom_values.yaml") + require.Nil(t, err) + require.NotNil(t, cm) + assert.EqualValues(t, "COMPLETION_STATUS_ALL", cm.Configuration.Export.Course.Progress.CompletionStatus) } func TestNewConfigurationManagerFromFile_WhenZeroLengthFile(t *testing.T) { @@ -210,6 +223,8 @@ func TestNewConfigurationManagerFromFile_WhenZeroLengthFile(t *testing.T) { assert.EqualValues(t, 100, cm.Configuration.Export.Issue.Limit) assert.EqualValues(t, 100, cm.Configuration.Export.Inspection.Limit) assert.EqualValues(t, 100, cm.Configuration.Export.Asset.Limit) + assert.EqualValues(t, 1000, cm.Configuration.Export.Course.Progress.Limit) + assert.EqualValues(t, "COMPLETION_STATUS_COMPLETED", cm.Configuration.Export.Course.Progress.CompletionStatus) assert.EqualValues(t, "true", cm.Configuration.Export.Inspection.Completed) assert.EqualValues(t, "false", cm.Configuration.Export.Inspection.Archived) assert.EqualValues(t, "UTC", cm.Configuration.Export.TimeZone) diff --git a/pkg/api/export_feeds_test.go b/pkg/api/export_feeds_test.go index 5902531f..e42c79e5 100644 --- a/pkg/api/export_feeds_test.go +++ b/pkg/api/export_feeds_test.go @@ -41,6 +41,8 @@ func TestExporterFeedClient_ExportFeeds_should_create_all_schemas_to_file(t *tes filesEqualish(t, "mocks/set_1/schemas/schedules.csv", filepath.Join(exporter.ExportPath, "schedules.csv")) filesEqualish(t, "mocks/set_1/schemas/schedule_assignees.csv", filepath.Join(exporter.ExportPath, "schedule_assignees.csv")) filesEqualish(t, "mocks/set_1/schemas/schedule_occurrences.csv", filepath.Join(exporter.ExportPath, "schedule_occurrences.csv")) + + filesEqualish(t, "mocks/set_1/schemas/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv")) } func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testing.T) { @@ -119,6 +121,8 @@ func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testi filesEqualish(t, "mocks/set_1/outputs/sheqsy_shifts.csv", filepath.Join(exporter.ExportPath, "sheqsy_shifts.csv")) filesEqualish(t, "mocks/set_1/outputs/sheqsy_activities.csv", filepath.Join(exporter.ExportPath, "sheqsy_activities.csv")) filesEqualish(t, "mocks/set_1/outputs/sheqsy_departments.csv", filepath.Join(exporter.ExportPath, "sheqsy_departments.csv")) + + filesEqualish(t, "mocks/set_1/outputs/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv")) } func TestExporterFeedClient_ExportFeeds_should_err_when_not_auth(t *testing.T) { diff --git a/pkg/api/fixtures/test_custom_values.yaml b/pkg/api/fixtures/test_custom_values.yaml new file mode 100644 index 00000000..9823e37b --- /dev/null +++ b/pkg/api/fixtures/test_custom_values.yaml @@ -0,0 +1,48 @@ +--- +access_token: "fake_token" +api: + proxy_url: https://fake_proxy.com + sheqsy_url: https://app.sheqsy.com + tls_cert: "" + tls_skip_verify: false + url: https://api.safetyculture.io +csv: + max_rows_per_file: 1000000 +db: + connection_string: "fake_connection_string" + dialect: mysql +export: + action: + limit: 50 + issue: + limit: 50 + course: + progress: + limit: 50 + completion_status: "COMPLETION_STATUS_ALL" + incremental: true + inspection: + archived: "false" + completed: "true" + included_inactive_items: false + limit: 100 + skip_ids: ["ID1", "ID2"] + web_report_link: private + media: false + media_path: ./export/media/ + modified_after: "" + path: ./export/ + site: + include_deleted: false + include_full_hierarchy: false + tables: [TA1, TA2, TA3] + template_ids: [] +report: + filename_convention: INSPECTION_TITLE + format: + - PDF + preference_id: "" + retry_timeout: 15 +sheqsy_company_id: fake_company_id +sheqsy_password: 123456 +sheqsy_username: fake_username diff --git a/pkg/api/fixtures/test_limit_101.yaml b/pkg/api/fixtures/test_limit_101.yaml index 4297ac98..66f1fd6e 100644 --- a/pkg/api/fixtures/test_limit_101.yaml +++ b/pkg/api/fixtures/test_limit_101.yaml @@ -16,6 +16,9 @@ export: limit: 101 issue: limit: 101 + course: + progress: + limit: 1001 incremental: true inspection: archived: "false" diff --git a/pkg/api/fixtures/test_limit_50.yaml b/pkg/api/fixtures/test_limit_50.yaml index e8a2ac8b..53c12d75 100644 --- a/pkg/api/fixtures/test_limit_50.yaml +++ b/pkg/api/fixtures/test_limit_50.yaml @@ -16,6 +16,9 @@ export: limit: 50 issue: limit: 50 + course: + progress: + limit: 50 incremental: true inspection: archived: "false" diff --git a/pkg/api/mock_feeds_test.go b/pkg/api/mock_feeds_test.go index 3efbbd02..1ae1cc6a 100644 --- a/pkg/api/mock_feeds_test.go +++ b/pkg/api/mock_feeds_test.go @@ -154,6 +154,11 @@ func initMockFeedsSet1(httpClient *http.Client) { Get("/feed/assets"). Reply(200). File("mocks/set_1/feed_assets_1.json") + + gock.New("http://localhost:9999"). + Get("/feed/training-course-progress"). + Reply(200). + File("mocks/set_1/feed_training_course_progress_1.json") } func initMockFeedsSet2(httpClient *http.Client) { @@ -259,6 +264,11 @@ func initMockFeedsSet2(httpClient *http.Client) { Get("/feed/assets"). Reply(200). File("mocks/set_1/feed_assets_1.json") + + gock.New("http://localhost:9999"). + Get("/feed/training-course-progress"). + Reply(200). + File("mocks/set_1/feed_training_course_progress_1.json") } func initMockFeedsSet3(httpClient *http.Client) { @@ -358,6 +368,11 @@ func initMockFeedsSet3(httpClient *http.Client) { Get("/feed/assets"). Reply(200). File("mocks/set_1/feed_assets_1.json") + + gock.New("http://localhost:9999"). + Get("/feed/training-course-progress"). + Reply(200). + File("mocks/set_1/feed_training_course_progress_1.json") } func initMockIssuesFeed(httpClient *http.Client) { diff --git a/pkg/api/mocks/set_1/feed_training_course_progress_1.json b/pkg/api/mocks/set_1/feed_training_course_progress_1.json new file mode 100644 index 00000000..aa196355 --- /dev/null +++ b/pkg/api/mocks/set_1/feed_training_course_progress_1.json @@ -0,0 +1,93 @@ +{ + "metadata": { + "next_page": null, + "next_page_token": null + }, + "data": [ + { + "opened_at": "2022-10-07T02:08:46.289Z", + "completed_at": "", + "total_lessons": 6, + "completed_lessons": 0, + "course_id": "733e547a8a7dfb438d27acc8", + "course_external_id": "", + "course_title": "Leading Under Pressure", + "user_email": "a@safetyculture.io", + "user_first_name": "Tony", + "user_last_name": "Blair", + "user_id": "user_4c5b19c1b75449a4ac3ea72d158a5a3d", + "user_external_id": "", + "progress_percent": 0, + "score": 0, + "due_at": "" + }, + { + "opened_at": "2022-10-07T04:39:05.861Z", + "completed_at": "", + "total_lessons": 6, + "completed_lessons": 0, + "course_id": "733e547a8a7dfb438d27acc8", + "course_external_id": "", + "course_title": "Leading Under Pressure", + "user_email": "b@safetyculture.io", + "user_first_name": "John", + "user_last_name": "Murphy", + "user_id": "user_d64b89887cf44b8d83681103c319f922", + "user_external_id": "", + "progress_percent": 0, + "score": 0, + "due_at": "" + }, + { + "opened_at": "2022-10-10T21:06:22.727Z", + "completed_at": "", + "total_lessons": 6, + "completed_lessons": 0, + "course_id": "733e547a8a7dfb438d27acc8", + "course_external_id": "", + "course_title": "Leading Under Pressure", + "user_email": "c@safetyculture.io", + "user_first_name": "Ryan", + "user_last_name": "Black", + "user_id": "user_839468bf0d574dd18c5cacb31fcfa6c1", + "user_external_id": "", + "progress_percent": 0, + "score": 0, + "due_at": "" + }, + { + "opened_at": "2022-10-11T04:21:30.861Z", + "completed_at": "", + "total_lessons": 6, + "completed_lessons": 0, + "course_id": "733e547a8a7dfb438d27acc8", + "course_external_id": "", + "course_title": "Leading Under Pressure", + "user_email": "d@safetyculture.io", + "user_first_name": "Joe", + "user_last_name": "White", + "user_id": "user_3f2e6fc4924e3f049929a0684d670afc", + "user_external_id": "", + "progress_percent": 0, + "score": 0, + "due_at": "" + }, + { + "opened_at": "2022-10-11T05:07:52.748Z", + "completed_at": "", + "total_lessons": 6, + "completed_lessons": 0, + "course_id": "733e547a8a7dfb438d27acc8", + "course_external_id": "", + "course_title": "Leading Under Pressure", + "user_email": "e@safetyculture.io", + "user_first_name": "Janet", + "user_last_name": "Polqa", + "user_id": "user_a424d7f87378442dbc22f4df684165c3", + "user_external_id": "", + "progress_percent": 0, + "score": 0, + "due_at": "" + } + ] +} diff --git a/pkg/api/mocks/set_1/outputs/training_course_progresses.csv b/pkg/api/mocks/set_1/outputs/training_course_progresses.csv new file mode 100644 index 00000000..e84127f1 --- /dev/null +++ b/pkg/api/mocks/set_1/outputs/training_course_progresses.csv @@ -0,0 +1,6 @@ +opened_at,completed_at,total_lessons,completed_lessons,course_id,course_external_id,course_title,user_email,user_first_name,user_last_name,user_id,user_external_id,progress_percent,score,due_at +--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,a@safetyculture.io,Tony,Blair,user_4c5b19c1b75449a4ac3ea72d158a5a3d,,0,0, +--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,b@safetyculture.io,John,Murphy,user_d64b89887cf44b8d83681103c319f922,,0,0, +--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,c@safetyculture.io,Ryan,Black,user_839468bf0d574dd18c5cacb31fcfa6c1,,0,0, +--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,d@safetyculture.io,Joe,White,user_3f2e6fc4924e3f049929a0684d670afc,,0,0, +--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,e@safetyculture.io,Janet,Polqa,user_a424d7f87378442dbc22f4df684165c3,,0,0, diff --git a/pkg/api/mocks/set_1/schemas/training_course_progresses.csv b/pkg/api/mocks/set_1/schemas/training_course_progresses.csv new file mode 100644 index 00000000..dfa9bb36 --- /dev/null +++ b/pkg/api/mocks/set_1/schemas/training_course_progresses.csv @@ -0,0 +1 @@ +opened_at,completed_at,total_lessons,completed_lessons,course_id,course_external_id,course_title,user_email,user_first_name,user_last_name,user_id,user_external_id,progress_percent,score,due_at diff --git a/pkg/internal/feed/drain_feed.go b/pkg/internal/feed/drain_feed.go index f139f398..b7bb151d 100644 --- a/pkg/internal/feed/drain_feed.go +++ b/pkg/internal/feed/drain_feed.go @@ -74,4 +74,8 @@ type GetFeedParams struct { // Applicable only for sites IncludeDeleted bool `url:"include_deleted,omitempty"` ShowOnlyLeafNodes *bool `url:"show_only_leaf_nodes,omitempty"` + + // Applicable only for course progress + Offset int `url:"offset,omitempty"` + CompletionStatus string `url:"completion_status,omitempty"` } diff --git a/pkg/internal/feed/exporter_schema.go b/pkg/internal/feed/exporter_schema.go index d1bae7fd..ee358af9 100644 --- a/pkg/internal/feed/exporter_schema.go +++ b/pkg/internal/feed/exporter_schema.go @@ -38,7 +38,6 @@ func (e *SchemaExporter) WriteSchema(feed Feed) error { e.Logger.With("feed", feed.Name()).Info("writing schema") schema := &[]*schema{} - resp := e.DB.Raw(fmt.Sprintf("PRAGMA table_info('%s') ", feed.Name())).Scan(schema) if resp.Error != nil { return resp.Error diff --git a/pkg/internal/feed/exporter_schema_test.go b/pkg/internal/feed/exporter_schema_test.go index 4fd3d17b..2fcebff8 100644 --- a/pkg/internal/feed/exporter_schema_test.go +++ b/pkg/internal/feed/exporter_schema_test.go @@ -24,9 +24,9 @@ func TestSchemaWriter_should_write_schema(t *testing.T) { err = exporter.WriteSchema(f) assert.Nil(t, err, fmt.Sprintf("something is wrong when writing schema %s, %v", f.Name(), err)) - actual, err := os.ReadFile(fmt.Sprintf("fixtures/schemas/formatted/%s.txt", f.Name())) + expected, err := os.ReadFile(fmt.Sprintf("fixtures/schemas/formatted/%s.txt", f.Name())) assert.Nil(t, err, fmt.Sprintf("something is wrong when reading file %s.txt, %v", f.Name(), err)) - assert.Equal(t, strings.TrimSpace(buf.String()), strings.TrimSpace(string(actual))) + assert.Equal(t, strings.TrimSpace(string(expected)), strings.TrimSpace(buf.String())) buf.Reset() } @@ -35,12 +35,10 @@ func TestSchemaWriter_should_write_schema(t *testing.T) { exporterApp := feed.NewExporterApp(nil, nil, cfg.ToExporterConfig()) for _, f := range exporterApp.GetFeeds() { - fmt.Printf("TESTING FEED: %s\n", f.Name()) testSchema(f) } for _, f := range exporterApp.GetSheqsyFeeds() { - fmt.Printf("TESTING FEED: %s\n", f.Name()) testSchema(f) } } diff --git a/pkg/internal/feed/feed.go b/pkg/internal/feed/feed.go index 0812f03c..168bda7f 100644 --- a/pkg/internal/feed/feed.go +++ b/pkg/internal/feed/feed.go @@ -21,6 +21,9 @@ type Feed interface { CreateSchema(exporter Exporter) error Export(ctx context.Context, apiClient *httpapi.Client, exporter Exporter, orgID string) error + + // HasRemainingInformation - true if the feed data source provides remaining number of items + HasRemainingInformation() bool } // InitFeedOptions contains the options used when initialising a feed diff --git a/pkg/internal/feed/feed_action.go b/pkg/internal/feed/feed_action.go index 4204e58d..58c8e92d 100644 --- a/pkg/internal/feed/feed_action.go +++ b/pkg/internal/feed/feed_action.go @@ -53,6 +53,11 @@ func (f *ActionFeed) Name() string { return "actions" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *ActionFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *ActionFeed) Model() interface{} { return Action{} diff --git a/pkg/internal/feed/feed_action_assignees.go b/pkg/internal/feed/feed_action_assignees.go index 51428d19..3e7f050b 100644 --- a/pkg/internal/feed/feed_action_assignees.go +++ b/pkg/internal/feed/feed_action_assignees.go @@ -37,6 +37,11 @@ func (f *ActionAssigneeFeed) Name() string { return "action_assignees" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *ActionAssigneeFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *ActionAssigneeFeed) Model() interface{} { return ActionAssignee{} diff --git a/pkg/internal/feed/feed_action_timeline_item.go b/pkg/internal/feed/feed_action_timeline_item.go index 8b1155ee..d782faad 100644 --- a/pkg/internal/feed/feed_action_timeline_item.go +++ b/pkg/internal/feed/feed_action_timeline_item.go @@ -38,6 +38,11 @@ func (f *ActionTimelineItemFeed) Name() string { return "action_timeline_items" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *ActionTimelineItemFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *ActionTimelineItemFeed) Model() interface{} { return ActionTimelineItem{} diff --git a/pkg/internal/feed/feed_asset.go b/pkg/internal/feed/feed_asset.go index f0e3b621..38f2bdc2 100644 --- a/pkg/internal/feed/feed_asset.go +++ b/pkg/internal/feed/feed_asset.go @@ -36,6 +36,11 @@ func (f *AssetFeed) Name() string { return "assets" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *AssetFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *AssetFeed) Model() interface{} { return Asset{} diff --git a/pkg/internal/feed/feed_exporter.go b/pkg/internal/feed/feed_exporter.go index c5e474b4..b511f385 100644 --- a/pkg/internal/feed/feed_exporter.go +++ b/pkg/internal/feed/feed_exporter.go @@ -60,6 +60,7 @@ type ExporterFeedCfg struct { ExportSiteIncludeFullHierarchy bool ExportIssueLimit int ExportAssetLimit int + ExportCourseProgressLimit int MaxConcurrentGoRoutines int } @@ -152,7 +153,7 @@ func (e *ExporterFeedClient) ExportFeeds(exporter Exporter, ctx context.Context) return default: log.Infof(" ... queueing %s\n", f.Name()) - status.StartFeedExport(f.Name(), true) + status.StartFeedExport(f.Name(), f.HasRemainingInformation()) exportErr := f.Export(c, e.apiClient, exporter, resp.OrganisationID) var curatedErr error if exportErr != nil { @@ -300,6 +301,11 @@ func (e *ExporterFeedClient) GetFeeds() []Feed { Incremental: false, // Assets API doesn't support modified after filters Limit: e.configuration.ExportAssetLimit, }, + &TrainingCourseProgressFeed{ + Incremental: false, // CourseProgress doesn't support modified after filters, + Limit: e.configuration.ExportCourseProgressLimit, + CompletionStatus: "COMPLETION_STATUS_COMPLETED", + }, } } @@ -355,7 +361,7 @@ func (e *ExporterFeedClient) ExportInspectionReports(exporter *ReportExporter, c log.Infof("Exporting inspection reports by user: %s %s", resp.Firstname, resp.Lastname) feed := e.getInspectionFeed() - status.StartFeedExport(feed.Name(), true) + status.StartFeedExport(feed.Name(), feed.HasRemainingInformation()) if err := feed.Export(ctx, e.apiClient, exporter, resp.OrganisationID); err != nil { status.FinishFeedExport(feed.Name(), err) status.MarkExportCompleted() diff --git a/pkg/internal/feed/feed_group.go b/pkg/internal/feed/feed_group.go index 2cffad77..95272866 100644 --- a/pkg/internal/feed/feed_group.go +++ b/pkg/internal/feed/feed_group.go @@ -29,6 +29,11 @@ func (f *GroupFeed) Name() string { return "groups" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *GroupFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *GroupFeed) Model() interface{} { return Group{} diff --git a/pkg/internal/feed/feed_group_user.go b/pkg/internal/feed/feed_group_user.go index c3dc2ef8..751795d2 100644 --- a/pkg/internal/feed/feed_group_user.go +++ b/pkg/internal/feed/feed_group_user.go @@ -29,6 +29,11 @@ func (f *GroupUserFeed) Name() string { return "group_users" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *GroupUserFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *GroupUserFeed) Model() interface{} { return GroupUser{} diff --git a/pkg/internal/feed/feed_inspection.go b/pkg/internal/feed/feed_inspection.go index 7254d0d8..91d8f849 100644 --- a/pkg/internal/feed/feed_inspection.go +++ b/pkg/internal/feed/feed_inspection.go @@ -69,6 +69,11 @@ func (f *InspectionFeed) Name() string { return "inspections" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *InspectionFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *InspectionFeed) Model() interface{} { return Inspection{} diff --git a/pkg/internal/feed/feed_inspection_item.go b/pkg/internal/feed/feed_inspection_item.go index eafdc6a7..15d077e2 100644 --- a/pkg/internal/feed/feed_inspection_item.go +++ b/pkg/internal/feed/feed_inspection_item.go @@ -74,6 +74,11 @@ func (f *InspectionItemFeed) Name() string { return "inspection_items" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *InspectionItemFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *InspectionItemFeed) Model() interface{} { return InspectionItem{} diff --git a/pkg/internal/feed/feed_issue.go b/pkg/internal/feed/feed_issue.go index 3cd833e4..0a0d6e81 100644 --- a/pkg/internal/feed/feed_issue.go +++ b/pkg/internal/feed/feed_issue.go @@ -49,6 +49,11 @@ func (f *IssueFeed) Name() string { return "issues" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *IssueFeed) HasRemainingInformation() bool { + return false +} + // Model returns the model of the feed row func (f *IssueFeed) Model() interface{} { return Issue{} @@ -104,7 +109,8 @@ func (f *IssueFeed) Export(ctx context.Context, apiClient *httpapi.Client, expor return events.NewEventErrorWithMessage(err, events.ErrorSeverityError, events.ErrorSubSystemDataIntegrity, false, "map data") } - if len(rows) != 0 { + numRows := len(rows) + if numRows != 0 { // Calculate the size of the batch we can insert into the DB at once. // Column count + buffer to account for primary keys batchSize := exporter.ParameterLimit() / (len(f.Columns()) + 4) @@ -120,7 +126,8 @@ func (f *IssueFeed) Export(ctx context.Context, apiClient *httpapi.Client, expor } } - status.UpdateStatus(f.Name(), resp.Metadata.RemainingRecords, exporter.GetDuration().Milliseconds()) + // note: this feed api doesn't return remaining items + status.IncrementStatus(f.Name(), int64(numRows), apiClient.Duration.Milliseconds()) l.With( "estimated_remaining", resp.Metadata.RemainingRecords, diff --git a/pkg/internal/feed/feed_issue_timeline_item.go b/pkg/internal/feed/feed_issue_timeline_item.go index 6c005fa4..5ee4a3f3 100644 --- a/pkg/internal/feed/feed_issue_timeline_item.go +++ b/pkg/internal/feed/feed_issue_timeline_item.go @@ -37,6 +37,11 @@ func (f *IssueTimelineItemFeed) Name() string { return "issue_timeline_items" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *IssueTimelineItemFeed) HasRemainingInformation() bool { + return false +} + // Model returns the model of the feed row func (f *IssueTimelineItemFeed) Model() interface{} { return IssueTimelineItem{} @@ -96,7 +101,8 @@ func (f *IssueTimelineItemFeed) Export(ctx context.Context, apiClient *httpapi.C return fmt.Errorf("map data: %w", err) } - if len(rows) != 0 { + numRows := len(rows) + if numRows != 0 { // Calculate the size of the batch we can insert into the DB at once. // Column count + buffer to account for primary keys batchSize := exporter.ParameterLimit() / (len(f.Columns()) + 4) @@ -112,7 +118,8 @@ func (f *IssueTimelineItemFeed) Export(ctx context.Context, apiClient *httpapi.C } } - status.UpdateStatus(f.Name(), resp.Metadata.RemainingRecords, exporter.GetDuration().Milliseconds()) + // note: this feed api doesn't return remaining items + status.IncrementStatus(f.Name(), int64(numRows), apiClient.Duration.Milliseconds()) l.With( "estimated_remaining", resp.Metadata.RemainingRecords, diff --git a/pkg/internal/feed/feed_schedule.go b/pkg/internal/feed/feed_schedule.go index 5d3ff7ff..6a89b597 100644 --- a/pkg/internal/feed/feed_schedule.go +++ b/pkg/internal/feed/feed_schedule.go @@ -45,6 +45,11 @@ func (f *ScheduleFeed) Name() string { return "schedules" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *ScheduleFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *ScheduleFeed) Model() interface{} { return Schedule{} diff --git a/pkg/internal/feed/feed_schedule_assignee.go b/pkg/internal/feed/feed_schedule_assignee.go index a552568c..812aad99 100644 --- a/pkg/internal/feed/feed_schedule_assignee.go +++ b/pkg/internal/feed/feed_schedule_assignee.go @@ -34,6 +34,11 @@ func (f *ScheduleAssigneeFeed) Name() string { return "schedule_assignees" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *ScheduleAssigneeFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *ScheduleAssigneeFeed) Model() interface{} { return ScheduleAssignee{} diff --git a/pkg/internal/feed/feed_schedule_occurrence.go b/pkg/internal/feed/feed_schedule_occurrence.go index 52acbc6a..63b451bf 100644 --- a/pkg/internal/feed/feed_schedule_occurrence.go +++ b/pkg/internal/feed/feed_schedule_occurrence.go @@ -42,6 +42,11 @@ func (f *ScheduleOccurrenceFeed) Name() string { return "schedule_occurrences" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *ScheduleOccurrenceFeed) HasRemainingInformation() bool { + return true +} + // RowsModel returns the model of feed rows func (f *ScheduleOccurrenceFeed) RowsModel() interface{} { return &[]*ScheduleOccurrence{} diff --git a/pkg/internal/feed/feed_sheqsy_activity.go b/pkg/internal/feed/feed_sheqsy_activity.go index ad7e6167..38575bf3 100644 --- a/pkg/internal/feed/feed_sheqsy_activity.go +++ b/pkg/internal/feed/feed_sheqsy_activity.go @@ -52,6 +52,11 @@ func (f *SheqsyActivityFeed) Name() string { return "sheqsy_activities" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SheqsyActivityFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SheqsyActivityFeed) Model() interface{} { return SheqsyActivity{} diff --git a/pkg/internal/feed/feed_sheqsy_department.go b/pkg/internal/feed/feed_sheqsy_department.go index f4513dc5..b6fe328b 100644 --- a/pkg/internal/feed/feed_sheqsy_department.go +++ b/pkg/internal/feed/feed_sheqsy_department.go @@ -32,6 +32,11 @@ func (f *SheqsyDepartmentFeed) Name() string { return "sheqsy_departments" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SheqsyDepartmentFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SheqsyDepartmentFeed) Model() interface{} { return SheqsyDepartment{} diff --git a/pkg/internal/feed/feed_sheqsy_department_employee.go b/pkg/internal/feed/feed_sheqsy_department_employee.go index aa82fed6..18eeba47 100644 --- a/pkg/internal/feed/feed_sheqsy_department_employee.go +++ b/pkg/internal/feed/feed_sheqsy_department_employee.go @@ -30,6 +30,11 @@ func (f *SheqsyDepartmentEmployeeFeed) Name() string { return "sheqsy_department_employees" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SheqsyDepartmentEmployeeFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SheqsyDepartmentEmployeeFeed) Model() interface{} { return SheqsyDepartmentEmployee{} diff --git a/pkg/internal/feed/feed_sheqsy_employee.go b/pkg/internal/feed/feed_sheqsy_employee.go index c7ad8194..747eadee 100644 --- a/pkg/internal/feed/feed_sheqsy_employee.go +++ b/pkg/internal/feed/feed_sheqsy_employee.go @@ -41,6 +41,11 @@ func (f *SheqsyEmployeeFeed) Name() string { return "sheqsy_employees" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SheqsyEmployeeFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SheqsyEmployeeFeed) Model() interface{} { return SheqsyEmployee{} diff --git a/pkg/internal/feed/feed_sheqsy_shift.go b/pkg/internal/feed/feed_sheqsy_shift.go index cbaf3b42..6bcbb08e 100644 --- a/pkg/internal/feed/feed_sheqsy_shift.go +++ b/pkg/internal/feed/feed_sheqsy_shift.go @@ -40,6 +40,11 @@ func (f *SheqsyShiftFeed) Name() string { return "sheqsy_shifts" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SheqsyShiftFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SheqsyShiftFeed) Model() interface{} { return SheqsyShift{} diff --git a/pkg/internal/feed/feed_site.go b/pkg/internal/feed/feed_site.go index 42cca72f..5888f576 100644 --- a/pkg/internal/feed/feed_site.go +++ b/pkg/internal/feed/feed_site.go @@ -37,6 +37,11 @@ func (f *SiteFeed) Name() string { return "sites" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SiteFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SiteFeed) Model() interface{} { return Site{} diff --git a/pkg/internal/feed/feed_site_member.go b/pkg/internal/feed/feed_site_member.go index 861c00cb..17df9e57 100644 --- a/pkg/internal/feed/feed_site_member.go +++ b/pkg/internal/feed/feed_site_member.go @@ -29,6 +29,11 @@ func (f *SiteMemberFeed) Name() string { return "site_members" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *SiteMemberFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *SiteMemberFeed) Model() interface{} { return SiteMember{} diff --git a/pkg/internal/feed/feed_template.go b/pkg/internal/feed/feed_template.go index 51f26b77..0e83a410 100644 --- a/pkg/internal/feed/feed_template.go +++ b/pkg/internal/feed/feed_template.go @@ -40,6 +40,11 @@ func (f *TemplateFeed) Name() string { return "templates" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *TemplateFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *TemplateFeed) Model() interface{} { return Template{} diff --git a/pkg/internal/feed/feed_template_permission.go b/pkg/internal/feed/feed_template_permission.go index b01c85ef..d50e5efc 100644 --- a/pkg/internal/feed/feed_template_permission.go +++ b/pkg/internal/feed/feed_template_permission.go @@ -34,6 +34,11 @@ func (f *TemplatePermissionFeed) Name() string { return "template_permissions" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *TemplatePermissionFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *TemplatePermissionFeed) Model() interface{} { return TemplatePermission{} diff --git a/pkg/internal/feed/feed_training_course_progress.go b/pkg/internal/feed/feed_training_course_progress.go new file mode 100644 index 00000000..bb461636 --- /dev/null +++ b/pkg/internal/feed/feed_training_course_progress.go @@ -0,0 +1,151 @@ +package feed + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/SafetyCulture/safetyculture-exporter/pkg/httpapi" + "github.com/SafetyCulture/safetyculture-exporter/pkg/internal/events" + "github.com/SafetyCulture/safetyculture-exporter/pkg/internal/util" + "github.com/SafetyCulture/safetyculture-exporter/pkg/logger" +) + +// TrainingCourseProgress represents a row for the feed +type TrainingCourseProgress struct { + OpenedAt string `json:"opened_at" csv:"opened_at"` + CompletedAt string `json:"completed_at" csv:"completed_at"` + TotalLessons int32 `json:"total_lessons" csv:"total_lessons"` + CompletedLessons int32 `json:"completed_lessons" csv:"completed_lessons"` + CourseID string `json:"course_id" csv:"course_id" gorm:"primarykey;column:course_id;size:64"` + CourseExternalID string `json:"course_external_id" csv:"course_external_id" gorm:"size:256"` + CourseTitle string `json:"course_title" csv:"course_title"` + UserEmail string `json:"user_email" csv:"user_email" gorm:"size:256"` + UserFirstName string `json:"user_first_name" csv:"user_first_name"` + UserLastName string `json:"user_last_name" csv:"user_last_name"` + UserID string `json:"user_id" csv:"user_id" gorm:"primarykey;column:user_id;size:37"` + UserExternalID string `json:"user_external_id" csv:"user_external_id"` + ProgressPercent float32 `json:"progress_percent" csv:"progress_percent"` + Score int32 `json:"score" csv:"score"` + DueAt string `json:"due_at" csv:"due_at"` +} + +// TrainingCourseProgressFeed is a representation of the feed +type TrainingCourseProgressFeed struct { + Incremental bool + Limit int + CompletionStatus string +} + +// Name is the name of the feed +func (f *TrainingCourseProgressFeed) Name() string { + return "training_course_progresses" +} + +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *TrainingCourseProgressFeed) HasRemainingInformation() bool { + return false +} + +// Model returns the model of the feed row +func (f *TrainingCourseProgressFeed) Model() interface{} { + return TrainingCourseProgress{} +} + +// RowsModel returns the model of feed rows +func (f *TrainingCourseProgressFeed) RowsModel() interface{} { + return &[]*TrainingCourseProgress{} +} + +// PrimaryKey returns the primary key(s) +func (f *TrainingCourseProgressFeed) PrimaryKey() []string { + return []string{"course_id", "user_id"} +} + +func (f *TrainingCourseProgressFeed) Columns() []string { + return []string{ + "opened_at", + "completed_at", + "total_lessons", + "completed_lessons", + "course_id", + "course_external_id", + "course_title", + "user_email", + "user_first_name", + "user_last_name", + "user_id", + "user_external_id", + "progress_percent", + "score", + "due_at", + } +} + +// Order returns the ordering when retrieving an export +func (f *TrainingCourseProgressFeed) Order() string { + return "opened_at" +} + +func (f *TrainingCourseProgressFeed) CreateSchema(exporter Exporter) error { + return exporter.CreateSchema(f, &[]*TrainingCourseProgress{}) +} + +func (f *TrainingCourseProgressFeed) Export(ctx context.Context, apiClient *httpapi.Client, exporter Exporter, orgID string) error { + l := logger.GetLogger().With("feed", f.Name(), "org_id", orgID) + status := GetExporterStatus() + + if err := exporter.InitFeed(f, &InitFeedOptions{ + // Delete data if incremental refresh is disabled so there is no duplicates + Truncate: !f.Incremental, + }); err != nil { + return events.WrapEventError(err, "init feed") + } + + drainFn := func(resp *GetFeedResponse) error { + var rows []*TrainingCourseProgress + + if err := json.Unmarshal(resp.Data, &rows); err != nil { + return events.NewEventErrorWithMessage(err, events.ErrorSeverityError, events.ErrorSubSystemDataIntegrity, false, "map data") + } + + numRows := len(rows) + if numRows != 0 { + // Calculate the size of the batch we can insert into the DB at once. Column count + buffer to account for primary keys + batchSize := exporter.ParameterLimit() / (len(f.Columns()) + 4) + err := util.SplitSliceInBatch(batchSize, rows, func(batch []*TrainingCourseProgress) error { + if err := exporter.WriteRows(f, batch); err != nil { + return events.WrapEventError(err, "write rows") + } + return nil + }) + + if err != nil { + return err + } + } + + // note: this feed api doesn't return remaining items + status.IncrementStatus(f.Name(), int64(numRows), apiClient.Duration.Milliseconds()) + + l.With( + "downloaded", numRows, + "duration_ms", apiClient.Duration.Milliseconds(), + "export_duration_ms", exporter.GetDuration().Milliseconds(), + ).Info("export batch complete") + return nil + } + + req := &GetFeedRequest{ + InitialURL: "/training/v1/feed/training-course-progress", + Params: GetFeedParams{ + Limit: f.Limit, + CompletionStatus: f.CompletionStatus, + }, + } + + if err := DrainFeed(ctx, apiClient, req, drainFn); err != nil { + return events.WrapEventError(err, fmt.Sprintf("feed %q", f.Name())) + } + return exporter.FinaliseExport(f, &[]*TrainingCourseProgress{}) +} diff --git a/pkg/internal/feed/feed_user.go b/pkg/internal/feed/feed_user.go index 7ee64773..1e9631b3 100644 --- a/pkg/internal/feed/feed_user.go +++ b/pkg/internal/feed/feed_user.go @@ -33,6 +33,11 @@ func (f *UserFeed) Name() string { return "users" } +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *UserFeed) HasRemainingInformation() bool { + return true +} + // Model returns the model of the feed row func (f *UserFeed) Model() interface{} { return User{} diff --git a/pkg/internal/feed/fixtures/schemas/formatted/training_course_progresses.txt b/pkg/internal/feed/fixtures/schemas/formatted/training_course_progresses.txt new file mode 100644 index 00000000..cfaf398d --- /dev/null +++ b/pkg/internal/feed/fixtures/schemas/formatted/training_course_progresses.txt @@ -0,0 +1,19 @@ ++--------------------+---------+-------------+ +| NAME | TYPE | PRIMARY KEY | ++--------------------+---------+-------------+ +| opened_at | TEXT | | +| completed_at | TEXT | | +| total_lessons | INTEGER | | +| completed_lessons | INTEGER | | +| course_id | TEXT | true | +| course_external_id | TEXT | | +| course_title | TEXT | | +| user_email | TEXT | | +| user_first_name | TEXT | | +| user_last_name | TEXT | | +| user_id | TEXT | true | +| user_external_id | TEXT | | +| progress_percent | REAL | | +| score | INTEGER | | +| due_at | TEXT | | ++--------------------+---------+-------------+