Merge pull request #23 from openedx/bmtcril/fix_unicode

fix: Unicode course data causes errors when sending to ClickHouse

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-latest]
         python-version: ['3.8']
         toxenv: [quality, docs, pii_check, django32, django40]
 

.github/workflows/pypi-publish.yml

@@ -7,7 +7,7 @@ on:
 jobs:
 
   push:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout

event_sink_clickhouse/__init__.py

@@ -2,4 +2,4 @@
 A sink for Open edX events to send them to ClickHouse.
 """
 
-__version__ = '0.1.0'
+__version__ = '0.1.1'

event_sink_clickhouse/sinks/course_published.py

@@ -245,7 +245,7 @@ def _send_course_overview(self, serialized_overview):
         request = requests.Request(
             'POST',
             self.ch_url,
-            data=output.getvalue(),
+            data=output.getvalue().encode("utf-8"),
             params=params,
             auth=self.ch_auth
         )
@@ -270,7 +270,7 @@ def _send_xblocks(self, serialized_xblocks):
         request = requests.Request(
             'POST',
             self.ch_url,
-            data=output.getvalue(),
+            data=output.getvalue().encode("utf-8"),
             params=params,
             auth=self.ch_auth
         )

test_utils/helpers.py

@@ -201,7 +201,7 @@ def check_overview_csv_matcher(course_overview):
     def match(request):
         body = request.body
 
-        f = StringIO(body)
+        f = StringIO(body.decode("utf-8"))
         reader = csv.reader(f)
 
         i = 0
@@ -251,7 +251,7 @@ def check_block_csv_matcher(course):
     that actually does the matching.
     """
     def match(request):
-        body = request.body
+        body = request.body.decode("utf-8")
         lines = body.split("\n")[:-1]
 
         # There should be one CSV line for each block in the test course