diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index cf2fd4b33..bce18af73 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,7 +65,7 @@ jobs: run: tutor local do importdemocourse - name: Test commands run: | - tutor local do dump-courses-to-clickhouse --options "--force" + tutor local do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations - name: Tutor stop run: tutor local stop @@ -118,7 +118,7 @@ jobs: - name: Import demo course run: tutor dev do importdemocourse - name: Test commands - run: tutor dev do dump-courses-to-clickhouse --options "--force" + run: tutor dev do dump-data-to-clickhouse --options "--object course_overviews" - name: Tutor stop run: tutor dev stop @@ -190,7 +190,7 @@ jobs: - name: Import demo course run: tutor k8s do importdemocourse - name: Test commands - run: tutor k8s do dump-courses-to-clickhouse --options "--force" + run: tutor k8s do dump-data-to-clickhouse --options "--object course_overviews" - name: Check failure logs if: failure() run: | diff --git a/README.rst b/README.rst index e5c77c905..7756e6ceb 100644 --- a/README.rst +++ b/README.rst @@ -67,8 +67,7 @@ At this point you should have a working Tutor / Aspects environment, but with no #. Sink course data from the LMS to clickhouse (see https://github.com/openedx/openedx-event-sink-clickhouse for more information):: - tutor local do dump-courses-to-clickhouse --options "--force" - + tutor local do dump-data-to-clickhouse --options "--object course_overviews" #. Sink Historical event data to ClickHouse:: diff --git a/tutoraspects/commands_v0.py b/tutoraspects/commands_v0.py index c35578fbd..467165520 100644 --- a/tutoraspects/commands_v0.py +++ b/tutoraspects/commands_v0.py @@ -93,20 +93,21 @@ def alembic(context, command) -> None: runner.run_job("aspects", command) -@click.command(help="Dump courses to ClickHouse.") +@click.command(help="Dump data to ClickHouse.") +@click.option("--service", default="lms", help="The service to run the command on.") @click.option("--options", default="") @click.pass_obj -def dump_courses_to_clickhouse(context, options) -> None: +def dump_data_to_clickhouse(context, service, options) -> None: """ - Job that proxies the dump_courses_to_clickhouse commands. + Job that proxies the dump_data_to_clickhouse commands. """ config = tutor_config.load(context.root) runner = context.job_runner(config) command = f""" - ./manage.py cms dump_courses_to_clickhouse {options} + ./manage.py {service} dump_data_to_clickhouse {options} """ - runner.run_job("cms", command) + runner.run_job(f"{service}", command) # pylint: disable=line-too-long @@ -175,7 +176,7 @@ def dump_courses_to_clickhouse(context, options) -> None: @click.pass_obj def transform_tracking_logs(context, deduplicate, **kwargs) -> None: """ - Job that proxies the dump_courses_to_clickhouse commands. + Job that proxies the transform_tracking_logs commands. """ config = tutor_config.load(context.root) runner = context.job_runner(config) @@ -210,6 +211,6 @@ def transform_tracking_logs(context, deduplicate, **kwargs) -> None: load_xapi_test_data, dbt, alembic, - dump_courses_to_clickhouse, + dump_data_to_clickhouse, transform_tracking_logs, ) diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 95cc07870..04a1f4052 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -100,14 +100,20 @@ def alembic(command: string) -> list[tuple[str, str]]: ] -# Ex: "tutor local do dump_courses_to_clickhouse " +# Ex: "tutor local do dump_data_to_clickhouse " @click.command(context_settings={"ignore_unknown_options": True}) +@click.option( + "--service", + default="lms", + type=click.UNPROCESSED, + help="The service to run the command on.", +) @click.option("--options", default="", type=click.UNPROCESSED) -def dump_courses_to_clickhouse(options) -> list[tuple[str, str]]: +def dump_data_to_clickhouse(service, options) -> list[tuple[str, str]]: """ - Job that proxies the dump_courses_to_clickhouse commands. + Job that proxies the dump_data_to_clickhouse commands. """ - return [("cms", f"./manage.py cms dump_courses_to_clickhouse {options}")] + return [(f"{service}", f"./manage.py {service} dump_data_to_clickhouse {options}")] # pylint: disable=line-too-long @@ -272,7 +278,7 @@ def check_superset_assets(): load_xapi_test_data, dbt, alembic, - dump_courses_to_clickhouse, + dump_data_to_clickhouse, transform_tracking_logs, ) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 1e8051773..af50fdfe8 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -41,7 +41,7 @@ ( "OPENEDX_EXTRA_PIP_REQUIREMENTS", [ - "openedx-event-sink-clickhouse==1.0.0", + "openedx-event-sink-clickhouse==1.1.0", "edx-event-routing-backends==v7.2.0", ], ),