Merge pull request #182 from openedx/cag/backfill-course-data

feat: backfill course data command

README.rst

@@ -58,6 +58,10 @@ Usage
 
     tutor [dev|local] do load-xapi-test-data
 
+#. (Optional) Sink course data from the LMS to clickhouse (see  `https://github.com/openedx/openedx-event-sink-clickhouse` for more information)::
+
+    tutor [dev|local] do dump-courses-to-clickhouse --options "--force"
+
 
 Superset Assets
 ---------------

tutoraspects/commands_v0.py

@@ -92,4 +92,24 @@ def alembic(context, command) -> None:
     runner.run_job("aspects", command)
 
 
-COMMANDS = (load_xapi_test_data, dbt, alembic)
+@click.option("--options", default="")
+@click.pass_obj
+def dump_courses_to_clickhouse(context, options) -> None:
+    """
+    Job that proxies the dump_courses_to_clickhouse commands.
+    """
+    config = tutor_config.load(context.root)
+    runner = context.job_runner(config)
+
+    command = f"""
+    ./manage.py cms dump_courses_to_clickhouse {options}
+    """
+    runner.run_job("cms", command)
+
+
+COMMANDS = (
+    load_xapi_test_data,
+    dbt,
+    alembic,
+    dump_courses_to_clickhouse,
+)

tutoraspects/commands_v1.py

@@ -94,8 +94,19 @@ def alembic(command: string) -> list[tuple[str, str]]:
     ]
 
 
+# Ex: "tutor local do dump_courses_to_clickhouse "
+@click.command(context_settings={"ignore_unknown_options": True})
+@click.option("--options", default="", type=click.UNPROCESSED)
+def dump_courses_to_clickhouse(options) -> list[tuple[str, str]]:
+    """
+    Job that proxies the dump_courses_to_clickhouse commands.
+    """
+    return [("cms", f"./manage.py cms dump_courses_to_clickhouse {options}")]
+
+
 COMMANDS = (
     load_xapi_test_data,
     dbt,
     alembic,
+    dump_courses_to_clickhouse,
 )

tutoraspects/plugin.py

@@ -323,11 +323,11 @@
 # and then add it to the MY_INIT_TASKS list. Each task is in the format:
 # ("<service>", ("<path>", "<to>", "<script>", "<template>"))
 MY_INIT_TASKS: list[tuple[str, tuple[str, ...], int]] = [
-    ("mysql", ("aspects", "jobs", "init", "superset", "init-mysql.sh"), 92),
+    ("mysql", ("aspects", "jobs", "init", "init-mysql.sh"), 92),
     ("clickhouse", ("aspects", "jobs", "init", "clickhouse", "init-clickhouse.sh"), 93),
     ("aspects", ("aspects", "jobs", "init", "aspects", "init-aspects.sh"), 94),
     ("superset", ("aspects", "jobs", "init", "superset", "init-superset.sh"), 95),
-    ("lms", ("aspects", "jobs", "init", "superset", "init-openedx.sh"), 96),
+    ("lms", ("aspects", "jobs", "init", "init-lms.sh"), 96),
 ]
 
 # For each task added to MY_INIT_TASKS, we load the task template