INTG-3044 course progress feeds (#439)

* INTG-3044 adding course progress * INTG-3044 adding course progress * INTG-3044 adding course progress * INTG-3044 test export 1 passes * INTG-3044 test export 2 passes * INTG-3044 fix path * INTG-3044 lint * INTG-3044 add filters * INTG-3044 add filters * INTG-3044 feed courses will send the correct status * INTG-3044 feed issues will send the correct status * INTG-3044 feed issues will send the correct status
SafetyCulture · Oct 9, 2023 · 3b7a19a · 3b7a19a
1 parent 55be17a
commit 3b7a19a
Show file tree

Hide file tree

Showing 41 changed files with 517 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ This way we can preserve the CSV columns in the export files.
 
 ### Testing
 
-Locally you can run `go test ./...`, this will run all of the Unit tests and Integration tests that can be run without an external DB.
+Locally you can run `go test ./...`, this will run all the Unit tests and Integration tests that can be run without an external DB.
 
 SQL Database integration tests can be run by starting the SQL DBs `docker-compose up -d` and then running `make integration-tests`.
 

diff --git a/pkg/api/configuration_manager.go b/pkg/api/configuration_manager.go
@@ -39,6 +39,12 @@ type ExporterConfiguration struct {
 		Asset struct {
 			Limit int `yaml:"limit"`
 		} `yaml:"asset"`
+		Course struct {
+			Progress struct {
+				Limit            int    `yaml:"limit"`
+				CompletionStatus string `yaml:"completion_status"`
+			} `yaml:"progress"`
+		} `yaml:"course"`
 		Incremental bool `yaml:"incremental"`
 		Inspection  struct {
 			Archived              string   `yaml:"archived"`
@@ -161,6 +167,15 @@ func (c *ConfigurationManager) ApplySafetyGuards() {
 		c.Configuration.Export.Issue.Limit = defaultCfg.Export.Issue.Limit
 	}
 
+	// caps course progress batch limit to 1000
+	if c.Configuration.Export.Course.Progress.Limit > 1000 || c.Configuration.Export.Course.Progress.Limit == 0 {
+		c.Configuration.Export.Course.Progress.Limit = defaultCfg.Export.Course.Progress.Limit
+	}
+
+	if c.Configuration.Export.Course.Progress.CompletionStatus == "" {
+		c.Configuration.Export.Course.Progress.CompletionStatus = defaultCfg.Export.Course.Progress.CompletionStatus
+	}
+
 	if c.Configuration.Export.Inspection.Limit == 0 {
 		c.Configuration.Export.Inspection.Limit = defaultCfg.Export.Inspection.Limit
 	}
@@ -257,6 +272,8 @@ func BuildConfigurationWithDefaults() *ExporterConfiguration {
 	cfg.Export.TemplateIds = []string{}
 	cfg.Export.Action.Limit = 100
 	cfg.Export.Asset.Limit = 100
+	cfg.Export.Course.Progress.Limit = 1000
+	cfg.Export.Course.Progress.CompletionStatus = "COMPLETION_STATUS_COMPLETED"
 	cfg.Export.Incremental = true
 	cfg.Export.Inspection.Archived = "false"
 	cfg.Export.Inspection.Completed = "true"
@@ -341,6 +358,7 @@ func (ec *ExporterConfiguration) ToExporterConfig() *feed.ExporterFeedCfg {
 		ExportSiteIncludeFullHierarchy:        ec.Export.Site.IncludeFullHierarchy,
 		ExportIssueLimit:                      ec.Export.Issue.Limit,
 		ExportAssetLimit:                      ec.Export.Asset.Limit,
+		ExportCourseProgressLimit:             ec.Export.Course.Progress.Limit,
 		MaxConcurrentGoRoutines:               ec.API.MaxConcurrency,
 	}
 }

diff --git a/pkg/api/configuration_manager_test.go b/pkg/api/configuration_manager_test.go
@@ -116,6 +116,8 @@ func TestNewConfigurationManagerFromFile_when_filename_exists_without_time(t *te
 	assert.Equal(t, "true", cfg.Export.Inspection.Completed)
 	assert.False(t, cfg.Export.Inspection.IncludedInactiveItems)
 	assert.Equal(t, 100, cfg.Export.Inspection.Limit)
+	assert.Equal(t, 1000, cfg.Export.Course.Progress.Limit)
+	assert.Equal(t, "COMPLETION_STATUS_COMPLETED", cfg.Export.Course.Progress.CompletionStatus)
 	assert.Equal(t, []string{"ID1", "ID2"}, cfg.Export.Inspection.SkipIds)
 	assert.Equal(t, "private", cfg.Export.Inspection.WebReportLink)
 	assert.False(t, cfg.Export.Media)
@@ -165,6 +167,8 @@ func TestConfigurationManager_SaveConfiguration(t *testing.T) {
 	assert.EqualValues(t, "https://app.sheqsy.com", newCm.Configuration.API.SheqsyURL)
 	assert.EqualValues(t, 1000000, newCm.Configuration.Csv.MaxRowsPerFile)
 	assert.EqualValues(t, 100, newCm.Configuration.Export.Action.Limit)
+	assert.EqualValues(t, 1000, newCm.Configuration.Export.Course.Progress.Limit)
+	assert.EqualValues(t, "COMPLETION_STATUS_COMPLETED", newCm.Configuration.Export.Course.Progress.CompletionStatus)
 	assert.True(t, newCm.Configuration.Export.Incremental)
 	assert.EqualValues(t, "false", newCm.Configuration.Export.Inspection.Archived)
 	assert.EqualValues(t, "true", newCm.Configuration.Export.Inspection.Completed)
@@ -185,6 +189,7 @@ func TestMapViperConfigToConfigurationOptions_ShouldRespectLimit(t *testing.T) {
 	require.NotNil(t, cm)
 	assert.EqualValues(t, 50, cm.Configuration.Export.Action.Limit)
 	assert.EqualValues(t, 50, cm.Configuration.Export.Issue.Limit)
+	assert.EqualValues(t, 50, cm.Configuration.Export.Course.Progress.Limit)
 }
 
 func TestMapViperConfigToConfigurationOptions_ShouldEnforceLimit(t *testing.T) {
@@ -193,6 +198,14 @@ func TestMapViperConfigToConfigurationOptions_ShouldEnforceLimit(t *testing.T) {
 	require.NotNil(t, cm)
 	assert.EqualValues(t, 100, cm.Configuration.Export.Action.Limit)
 	assert.EqualValues(t, 100, cm.Configuration.Export.Issue.Limit)
+	assert.EqualValues(t, 1000, cm.Configuration.Export.Course.Progress.Limit)
+}
+
+func TestMapViperConfigToConfigurationOptions_CustomValues(t *testing.T) {
+	cm, err := api.NewConfigurationManagerFromFile("", "fixtures/test_custom_values.yaml")
+	require.Nil(t, err)
+	require.NotNil(t, cm)
+	assert.EqualValues(t, "COMPLETION_STATUS_ALL", cm.Configuration.Export.Course.Progress.CompletionStatus)
 }
 
 func TestNewConfigurationManagerFromFile_WhenZeroLengthFile(t *testing.T) {
@@ -210,6 +223,8 @@ func TestNewConfigurationManagerFromFile_WhenZeroLengthFile(t *testing.T) {
 	assert.EqualValues(t, 100, cm.Configuration.Export.Issue.Limit)
 	assert.EqualValues(t, 100, cm.Configuration.Export.Inspection.Limit)
 	assert.EqualValues(t, 100, cm.Configuration.Export.Asset.Limit)
+	assert.EqualValues(t, 1000, cm.Configuration.Export.Course.Progress.Limit)
+	assert.EqualValues(t, "COMPLETION_STATUS_COMPLETED", cm.Configuration.Export.Course.Progress.CompletionStatus)
 	assert.EqualValues(t, "true", cm.Configuration.Export.Inspection.Completed)
 	assert.EqualValues(t, "false", cm.Configuration.Export.Inspection.Archived)
 	assert.EqualValues(t, "UTC", cm.Configuration.Export.TimeZone)

diff --git a/pkg/api/export_feeds_test.go b/pkg/api/export_feeds_test.go
@@ -41,6 +41,8 @@ func TestExporterFeedClient_ExportFeeds_should_create_all_schemas_to_file(t *tes
 	filesEqualish(t, "mocks/set_1/schemas/schedules.csv", filepath.Join(exporter.ExportPath, "schedules.csv"))
 	filesEqualish(t, "mocks/set_1/schemas/schedule_assignees.csv", filepath.Join(exporter.ExportPath, "schedule_assignees.csv"))
 	filesEqualish(t, "mocks/set_1/schemas/schedule_occurrences.csv", filepath.Join(exporter.ExportPath, "schedule_occurrences.csv"))
+
+	filesEqualish(t, "mocks/set_1/schemas/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv"))
 }
 
 func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testing.T) {
@@ -119,6 +121,8 @@ func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testi
 	filesEqualish(t, "mocks/set_1/outputs/sheqsy_shifts.csv", filepath.Join(exporter.ExportPath, "sheqsy_shifts.csv"))
 	filesEqualish(t, "mocks/set_1/outputs/sheqsy_activities.csv", filepath.Join(exporter.ExportPath, "sheqsy_activities.csv"))
 	filesEqualish(t, "mocks/set_1/outputs/sheqsy_departments.csv", filepath.Join(exporter.ExportPath, "sheqsy_departments.csv"))
+
+	filesEqualish(t, "mocks/set_1/outputs/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv"))
 }
 
 func TestExporterFeedClient_ExportFeeds_should_err_when_not_auth(t *testing.T) {

diff --git a/pkg/api/fixtures/test_custom_values.yaml b/pkg/api/fixtures/test_custom_values.yaml
@@ -0,0 +1,48 @@
+---
+access_token: "fake_token"
+api:
+  proxy_url: https://fake_proxy.com
+  sheqsy_url: https://app.sheqsy.com
+  tls_cert: ""
+  tls_skip_verify: false
+  url: https://api.safetyculture.io
+csv:
+  max_rows_per_file: 1000000
+db:
+  connection_string: "fake_connection_string"
+  dialect: mysql
+export:
+  action:
+    limit: 50
+  issue:
+    limit: 50
+  course:
+    progress:
+      limit: 50
+      completion_status: "COMPLETION_STATUS_ALL"
+  incremental: true
+  inspection:
+    archived: "false"
+    completed: "true"
+    included_inactive_items: false
+    limit: 100
+    skip_ids: ["ID1", "ID2"]
+    web_report_link: private
+  media: false
+  media_path: ./export/media/
+  modified_after: ""
+  path: ./export/
+  site:
+    include_deleted: false
+    include_full_hierarchy: false
+  tables: [TA1, TA2, TA3]
+  template_ids: []
+report:
+  filename_convention: INSPECTION_TITLE
+  format:
+    - PDF
+  preference_id: ""
+  retry_timeout: 15
+sheqsy_company_id: fake_company_id
+sheqsy_password: 123456
+sheqsy_username: fake_username
diff --git a/pkg/api/fixtures/test_limit_101.yaml b/pkg/api/fixtures/test_limit_101.yaml
@@ -16,6 +16,9 @@ export:
     limit: 101
   issue:
     limit: 101
+  course:
+    progress:
+      limit: 1001
   incremental: true
   inspection:
     archived: "false"

diff --git a/pkg/api/fixtures/test_limit_50.yaml b/pkg/api/fixtures/test_limit_50.yaml
@@ -16,6 +16,9 @@ export:
     limit: 50
   issue:
     limit: 50
+  course:
+    progress:
+      limit: 50
   incremental: true
   inspection:
     archived: "false"

diff --git a/pkg/api/mock_feeds_test.go b/pkg/api/mock_feeds_test.go
@@ -154,6 +154,11 @@ func initMockFeedsSet1(httpClient *http.Client) {
 		Get("/feed/assets").
 		Reply(200).
 		File("mocks/set_1/feed_assets_1.json")
+
+	gock.New("http://localhost:9999").
+		Get("/feed/training-course-progress").
+		Reply(200).
+		File("mocks/set_1/feed_training_course_progress_1.json")
 }
 
 func initMockFeedsSet2(httpClient *http.Client) {
@@ -259,6 +264,11 @@ func initMockFeedsSet2(httpClient *http.Client) {
 		Get("/feed/assets").
 		Reply(200).
 		File("mocks/set_1/feed_assets_1.json")
+
+	gock.New("http://localhost:9999").
+		Get("/feed/training-course-progress").
+		Reply(200).
+		File("mocks/set_1/feed_training_course_progress_1.json")
 }
 
 func initMockFeedsSet3(httpClient *http.Client) {
@@ -358,6 +368,11 @@ func initMockFeedsSet3(httpClient *http.Client) {
 		Get("/feed/assets").
 		Reply(200).
 		File("mocks/set_1/feed_assets_1.json")
+
+	gock.New("http://localhost:9999").
+		Get("/feed/training-course-progress").
+		Reply(200).
+		File("mocks/set_1/feed_training_course_progress_1.json")
 }
 
 func initMockIssuesFeed(httpClient *http.Client) {

diff --git a/pkg/api/mocks/set_1/feed_training_course_progress_1.json b/pkg/api/mocks/set_1/feed_training_course_progress_1.json
@@ -0,0 +1,93 @@
+{
+    "metadata": {
+        "next_page": null,
+        "next_page_token": null
+    },
+    "data": [
+        {
+            "opened_at": "2022-10-07T02:08:46.289Z",
+            "completed_at": "",
+            "total_lessons": 6,
+            "completed_lessons": 0,
+            "course_id": "733e547a8a7dfb438d27acc8",
+            "course_external_id": "",
+            "course_title": "Leading Under Pressure",
+            "user_email": "[email protected]",
+            "user_first_name": "Tony",
+            "user_last_name": "Blair",
+            "user_id": "user_4c5b19c1b75449a4ac3ea72d158a5a3d",
+            "user_external_id": "",
+            "progress_percent": 0,
+            "score": 0,
+            "due_at": ""
+        },
+        {
+            "opened_at": "2022-10-07T04:39:05.861Z",
+            "completed_at": "",
+            "total_lessons": 6,
+            "completed_lessons": 0,
+            "course_id": "733e547a8a7dfb438d27acc8",
+            "course_external_id": "",
+            "course_title": "Leading Under Pressure",
+            "user_email": "[email protected]",
+            "user_first_name": "John",
+            "user_last_name": "Murphy",
+            "user_id": "user_d64b89887cf44b8d83681103c319f922",
+            "user_external_id": "",
+            "progress_percent": 0,
+            "score": 0,
+            "due_at": ""
+        },
+        {
+            "opened_at": "2022-10-10T21:06:22.727Z",
+            "completed_at": "",
+            "total_lessons": 6,
+            "completed_lessons": 0,
+            "course_id": "733e547a8a7dfb438d27acc8",
+            "course_external_id": "",
+            "course_title": "Leading Under Pressure",
+            "user_email": "[email protected]",
+            "user_first_name": "Ryan",
+            "user_last_name": "Black",
+            "user_id": "user_839468bf0d574dd18c5cacb31fcfa6c1",
+            "user_external_id": "",
+            "progress_percent": 0,
+            "score": 0,
+            "due_at": ""
+        },
+        {
+            "opened_at": "2022-10-11T04:21:30.861Z",
+            "completed_at": "",
+            "total_lessons": 6,
+            "completed_lessons": 0,
+            "course_id": "733e547a8a7dfb438d27acc8",
+            "course_external_id": "",
+            "course_title": "Leading Under Pressure",
+            "user_email": "[email protected]",
+            "user_first_name": "Joe",
+            "user_last_name": "White",
+            "user_id": "user_3f2e6fc4924e3f049929a0684d670afc",
+            "user_external_id": "",
+            "progress_percent": 0,
+            "score": 0,
+            "due_at": ""
+        },
+        {
+            "opened_at": "2022-10-11T05:07:52.748Z",
+            "completed_at": "",
+            "total_lessons": 6,
+            "completed_lessons": 0,
+            "course_id": "733e547a8a7dfb438d27acc8",
+            "course_external_id": "",
+            "course_title": "Leading Under Pressure",
+            "user_email": "[email protected]",
+            "user_first_name": "Janet",
+            "user_last_name": "Polqa",
+            "user_id": "user_a424d7f87378442dbc22f4df684165c3",
+            "user_external_id": "",
+            "progress_percent": 0,
+            "score": 0,
+            "due_at": ""
+        }
+    ]
+}
diff --git a/pkg/api/mocks/set_1/outputs/training_course_progresses.csv b/pkg/api/mocks/set_1/outputs/training_course_progresses.csv
@@ -0,0 +1,6 @@
+opened_at,completed_at,total_lessons,completed_lessons,course_id,course_external_id,course_title,user_email,user_first_name,user_last_name,user_id,user_external_id,progress_percent,score,due_at
+--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,[email protected],Tony,Blair,user_4c5b19c1b75449a4ac3ea72d158a5a3d,,0,0,
+--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,[email protected],John,Murphy,user_d64b89887cf44b8d83681103c319f922,,0,0,
+--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,[email protected],Ryan,Black,user_839468bf0d574dd18c5cacb31fcfa6c1,,0,0,
+--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,[email protected],Joe,White,user_3f2e6fc4924e3f049929a0684d670afc,,0,0,
+--date--,,6,0,733e547a8a7dfb438d27acc8,,Leading Under Pressure,[email protected],Janet,Polqa,user_a424d7f87378442dbc22f4df684165c3,,0,0,
diff --git a/pkg/api/mocks/set_1/schemas/training_course_progresses.csv b/pkg/api/mocks/set_1/schemas/training_course_progresses.csv
@@ -0,0 +1 @@
+opened_at,completed_at,total_lessons,completed_lessons,course_id,course_external_id,course_title,user_email,user_first_name,user_last_name,user_id,user_external_id,progress_percent,score,due_at
diff --git a/pkg/internal/feed/drain_feed.go b/pkg/internal/feed/drain_feed.go
@@ -74,4 +74,8 @@ type GetFeedParams struct {
 	// Applicable only for sites
 	IncludeDeleted    bool  `url:"include_deleted,omitempty"`
 	ShowOnlyLeafNodes *bool `url:"show_only_leaf_nodes,omitempty"`
+
+	// Applicable only for course progress
+	Offset           int    `url:"offset,omitempty"`
+	CompletionStatus string `url:"completion_status,omitempty"`
 }
diff --git a/pkg/internal/feed/exporter_schema.go b/pkg/internal/feed/exporter_schema.go
@@ -38,7 +38,6 @@ func (e *SchemaExporter) WriteSchema(feed Feed) error {
 	e.Logger.With("feed", feed.Name()).Info("writing schema")
 
 	schema := &[]*schema{}
-
 	resp := e.DB.Raw(fmt.Sprintf("PRAGMA table_info('%s') ", feed.Name())).Scan(schema)
 	if resp.Error != nil {
 		return resp.Error

diff --git a/pkg/internal/feed/exporter_schema_test.go b/pkg/internal/feed/exporter_schema_test.go
@@ -24,9 +24,9 @@ func TestSchemaWriter_should_write_schema(t *testing.T) {
 		err = exporter.WriteSchema(f)
 		assert.Nil(t, err, fmt.Sprintf("something is wrong when writing schema %s, %v", f.Name(), err))
 
-		actual, err := os.ReadFile(fmt.Sprintf("fixtures/schemas/formatted/%s.txt", f.Name()))
+		expected, err := os.ReadFile(fmt.Sprintf("fixtures/schemas/formatted/%s.txt", f.Name()))
 		assert.Nil(t, err, fmt.Sprintf("something is wrong when reading file %s.txt, %v", f.Name(), err))
-		assert.Equal(t, strings.TrimSpace(buf.String()), strings.TrimSpace(string(actual)))
+		assert.Equal(t, strings.TrimSpace(string(expected)), strings.TrimSpace(buf.String()))
 
 		buf.Reset()
 	}
@@ -35,12 +35,10 @@ func TestSchemaWriter_should_write_schema(t *testing.T) {
 	exporterApp := feed.NewExporterApp(nil, nil, cfg.ToExporterConfig())
 
 	for _, f := range exporterApp.GetFeeds() {
-		fmt.Printf("TESTING FEED: %s\n", f.Name())
 		testSchema(f)
 	}
 
 	for _, f := range exporterApp.GetSheqsyFeeds() {
-		fmt.Printf("TESTING FEED: %s\n", f.Name())
 		testSchema(f)
 	}
 }

diff --git a/pkg/internal/feed/feed.go b/pkg/internal/feed/feed.go
@@ -21,6 +21,9 @@ type Feed interface {
 
 	CreateSchema(exporter Exporter) error
 	Export(ctx context.Context, apiClient *httpapi.Client, exporter Exporter, orgID string) error
+
+	// HasRemainingInformation - true if the feed data source provides remaining number of items
+	HasRemainingInformation() bool
 }
 
 // InitFeedOptions contains the options used when initialising a feed

diff --git a/pkg/internal/feed/feed_action.go b/pkg/internal/feed/feed_action.go
@@ -53,6 +53,11 @@ func (f *ActionFeed) Name() string {
 	return "actions"
 }
 
+// HasRemainingInformation returns true if the feed returns remaining items information
+func (f *ActionFeed) HasRemainingInformation() bool {
+	return true
+}
+
 // Model returns the model of the feed row
 func (f *ActionFeed) Model() interface{} {
 	return Action{}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		opened_at,completed_at,total_lessons,completed_lessons,course_id,course_external_id,course_title,user_email,user_first_name,user_last_name,user_id,user_external_id,progress_percent,score,due_at