perf: reduce write requests call to google spreadsheet via append_row…

…s method (#4423)
openedx · Sep 3, 2024 · 193a187 · 193a187
1 parent 8cc1d57
commit 193a187
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 15 deletions.
diff --git a/course_discovery/apps/course_metadata/gspread_client.py b/course_discovery/apps/course_metadata/gspread_client.py
@@ -80,17 +80,19 @@ def _write_rows(self, sheet_tab, headers, csv_data):
             csv_data: The data to be written in the worksheet, as a list of dictionaries, where
             each dictionary represents a row
         """
-        for row in csv_data:
-            sheet_tab.append_row(
-                [
-                    (
-                        row.get(header).replace('\"', '\"\"')  # double quote escape to preserve " in values
-                        if isinstance(row.get(header), str)
-                        else row.get(header)
-                    )
-                    for header in headers
-                ]
-            )
+        rows = [
+            [
+                row.get(header).replace('\"', '\"\"') if isinstance(row.get(header), str) else row.get(header)
+                for header in headers
+            ]
+            for row in csv_data
+        ]
+        try:
+            sheet_tab.append_rows(rows)
+        except gspread.exceptions.APIError as e:
+            logger.exception(f"[Spread Sheet Write Error]: APIError occurred while writing rows: {e}")
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception(f"[Spread Sheet Write Error]: Exception occurred while writing rows: {e}")
 
     def write_data(self, config, csv_headers, csv_data, overwrite):
         """

diff --git a/course_discovery/apps/course_metadata/tests/test_gspread.py b/course_discovery/apps/course_metadata/tests/test_gspread.py
@@ -75,12 +75,12 @@ def test_write_rows(self):
         headers = ['header1', 'header2']
         csv_data = [{'header1': 'value1', 'header2': 'value2'}, {'header1': 'value3', 'header2': 'value4'}]
 
+        expected_rows = [['value1', 'value2'], ['value3', 'value4']]
+
         client = GspreadClient()
-        client._write_rows(mock_sheet_tab, headers, csv_data)  # pylint: disable=protected-access
 
-        mock_sheet_tab.append_row.assert_any_call(['value1', 'value2'])
-        mock_sheet_tab.append_row.assert_any_call(['value3', 'value4'])
-        self.assertEqual(mock_sheet_tab.append_row.call_count, 2)
+        client._write_rows(mock_sheet_tab, headers, csv_data)  # pylint: disable=protected-access
+        mock_sheet_tab.append_rows.assert_called_once_with(expected_rows)
 
     @mock.patch('course_discovery.apps.course_metadata.gspread_client.GspreadClient._get_or_create_worksheet')
     @mock.patch('course_discovery.apps.course_metadata.gspread_client.GspreadClient._write_headers')