feat: add command to dump data to clickhouse

.github/workflows/integration-test.yml

@@ -65,7 +65,7 @@ jobs:
         run: tutor local do importdemocourse
       - name: Test commands
         run: |
-          tutor local do dump-courses-to-clickhouse --options "--force"
+          tutor local do dump-data-to-clickhouse --options "--object course_overviews"
           make extract_translations
       - name: Tutor stop
         run: tutor local stop
@@ -118,7 +118,7 @@ jobs:
       - name: Import demo course
         run: tutor dev do importdemocourse
       - name: Test commands
-        run: tutor dev do dump-courses-to-clickhouse --options "--force"
+        run: tutor dev do dump-data-to-clickhouse --options "--object course_overviews"
       - name: Tutor stop
         run: tutor dev stop
 
@@ -190,7 +190,7 @@ jobs:
       - name: Import demo course
         run: tutor k8s do importdemocourse
       - name: Test commands
-        run: tutor k8s do dump-courses-to-clickhouse --options "--force"
+        run: tutor k8s do dump-data-to-clickhouse --options "--object course_overviews"
       - name: Check failure logs
         if: failure()
         run: |

README.rst

@@ -67,8 +67,7 @@ At this point you should have a working Tutor / Aspects environment, but with no
 
    #. Sink course data from the LMS to clickhouse (see https://github.com/openedx/openedx-event-sink-clickhouse for more information)::
 
-       tutor local do dump-courses-to-clickhouse --options "--force"
-
+       tutor local do dump-data-to-clickhouse --options "--object course_overviews"
 
    #. Sink Historical event data to ClickHouse::
 

tutoraspects/commands_v0.py

@@ -93,20 +93,21 @@ def alembic(context, command) -> None:
     runner.run_job("aspects", command)
 
 
-@click.command(help="Dump courses to ClickHouse.")
+@click.command(help="Dump data to ClickHouse.")
+@click.option("--service", default="lms", help="The service to run the command on.")
 @click.option("--options", default="")
 @click.pass_obj
-def dump_courses_to_clickhouse(context, options) -> None:
+def dump_data_to_clickhouse(context, service, options) -> None:
     """
-    Job that proxies the dump_courses_to_clickhouse commands.
+    Job that proxies the dump_data_to_clickhouse commands.
     """
     config = tutor_config.load(context.root)
     runner = context.job_runner(config)
 
     command = f"""
-    ./manage.py cms dump_courses_to_clickhouse {options}
+    ./manage.py {service} dump_data_to_clickhouse {options}
     """
-    runner.run_job("cms", command)
+    runner.run_job(f"{service}", command)
 
 
 # pylint: disable=line-too-long
@@ -175,7 +176,7 @@ def dump_courses_to_clickhouse(context, options) -> None:
 @click.pass_obj
 def transform_tracking_logs(context, deduplicate, **kwargs) -> None:
     """
-    Job that proxies the dump_courses_to_clickhouse commands.
+    Job that proxies the transform_tracking_logs commands.
     """
     config = tutor_config.load(context.root)
     runner = context.job_runner(config)
@@ -210,6 +211,6 @@ def transform_tracking_logs(context, deduplicate, **kwargs) -> None:
     load_xapi_test_data,
     dbt,
     alembic,
-    dump_courses_to_clickhouse,
+    dump_data_to_clickhouse,
     transform_tracking_logs,
 )

tutoraspects/commands_v1.py

@@ -100,14 +100,20 @@ def alembic(command: string) -> list[tuple[str, str]]:
     ]
 
 
-# Ex: "tutor local do dump_courses_to_clickhouse "
+# Ex: "tutor local do dump_data_to_clickhouse "
 @click.command(context_settings={"ignore_unknown_options": True})
+@click.option(
+    "--service",
+    default="lms",
+    type=click.UNPROCESSED,
+    help="The service to run the command on.",
+)
 @click.option("--options", default="", type=click.UNPROCESSED)
-def dump_courses_to_clickhouse(options) -> list[tuple[str, str]]:
+def dump_data_to_clickhouse(service, options) -> list[tuple[str, str]]:
     """
-    Job that proxies the dump_courses_to_clickhouse commands.
+    Job that proxies the dump_data_to_clickhouse commands.
     """
-    return [("cms", f"./manage.py cms dump_courses_to_clickhouse {options}")]
+    return [(f"{service}", f"./manage.py {service} dump_data_to_clickhouse {options}")]
 
 
 # pylint: disable=line-too-long
@@ -272,7 +278,7 @@ def check_superset_assets():
     load_xapi_test_data,
     dbt,
     alembic,
-    dump_courses_to_clickhouse,
+    dump_data_to_clickhouse,
     transform_tracking_logs,
 )
 

tutoraspects/plugin.py

@@ -41,7 +41,7 @@
         (
             "OPENEDX_EXTRA_PIP_REQUIREMENTS",
             [
-                "openedx-event-sink-clickhouse==1.0.0",
+                "openedx-event-sink-clickhouse==1.1.0",
                 "edx-event-routing-backends==v7.2.0",
             ],
         ),