From 019218f9e5814013f5e03a80d0acf4fd63f34f74 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 6 Feb 2024 17:31:09 -0500 Subject: [PATCH 1/3] feat: add command to dump data to clickhouse --- .github/workflows/integration-test.yml | 6 +++--- README.rst | 3 +-- tutoraspects/commands_v0.py | 15 ++++++++------- tutoraspects/commands_v1.py | 11 ++++++----- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index cf2fd4b33..bce18af73 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,7 +65,7 @@ jobs: run: tutor local do importdemocourse - name: Test commands run: | - tutor local do dump-courses-to-clickhouse --options "--force" + tutor local do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations - name: Tutor stop run: tutor local stop @@ -118,7 +118,7 @@ jobs: - name: Import demo course run: tutor dev do importdemocourse - name: Test commands - run: tutor dev do dump-courses-to-clickhouse --options "--force" + run: tutor dev do dump-data-to-clickhouse --options "--object course_overviews" - name: Tutor stop run: tutor dev stop @@ -190,7 +190,7 @@ jobs: - name: Import demo course run: tutor k8s do importdemocourse - name: Test commands - run: tutor k8s do dump-courses-to-clickhouse --options "--force" + run: tutor k8s do dump-data-to-clickhouse --options "--object course_overviews" - name: Check failure logs if: failure() run: | diff --git a/README.rst b/README.rst index e5c77c905..7756e6ceb 100644 --- a/README.rst +++ b/README.rst @@ -67,8 +67,7 @@ At this point you should have a working Tutor / Aspects environment, but with no #. Sink course data from the LMS to clickhouse (see https://github.com/openedx/openedx-event-sink-clickhouse for more information):: - tutor local do dump-courses-to-clickhouse --options "--force" - + tutor local do dump-data-to-clickhouse --options "--object course_overviews" #. Sink Historical event data to ClickHouse:: diff --git a/tutoraspects/commands_v0.py b/tutoraspects/commands_v0.py index c35578fbd..467165520 100644 --- a/tutoraspects/commands_v0.py +++ b/tutoraspects/commands_v0.py @@ -93,20 +93,21 @@ def alembic(context, command) -> None: runner.run_job("aspects", command) -@click.command(help="Dump courses to ClickHouse.") +@click.command(help="Dump data to ClickHouse.") +@click.option("--service", default="lms", help="The service to run the command on.") @click.option("--options", default="") @click.pass_obj -def dump_courses_to_clickhouse(context, options) -> None: +def dump_data_to_clickhouse(context, service, options) -> None: """ - Job that proxies the dump_courses_to_clickhouse commands. + Job that proxies the dump_data_to_clickhouse commands. """ config = tutor_config.load(context.root) runner = context.job_runner(config) command = f""" - ./manage.py cms dump_courses_to_clickhouse {options} + ./manage.py {service} dump_data_to_clickhouse {options} """ - runner.run_job("cms", command) + runner.run_job(f"{service}", command) # pylint: disable=line-too-long @@ -175,7 +176,7 @@ def dump_courses_to_clickhouse(context, options) -> None: @click.pass_obj def transform_tracking_logs(context, deduplicate, **kwargs) -> None: """ - Job that proxies the dump_courses_to_clickhouse commands. + Job that proxies the transform_tracking_logs commands. """ config = tutor_config.load(context.root) runner = context.job_runner(config) @@ -210,6 +211,6 @@ def transform_tracking_logs(context, deduplicate, **kwargs) -> None: load_xapi_test_data, dbt, alembic, - dump_courses_to_clickhouse, + dump_data_to_clickhouse, transform_tracking_logs, ) diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 95cc07870..075fa3c2f 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -100,14 +100,15 @@ def alembic(command: string) -> list[tuple[str, str]]: ] -# Ex: "tutor local do dump_courses_to_clickhouse " +# Ex: "tutor local do dump_data_to_clickhouse " @click.command(context_settings={"ignore_unknown_options": True}) +@click.option("--service", default="lms", type=click.UNPROCESSED, help="The service to run the command on.") @click.option("--options", default="", type=click.UNPROCESSED) -def dump_courses_to_clickhouse(options) -> list[tuple[str, str]]: +def dump_data_to_clickhouse(service, options) -> list[tuple[str, str]]: """ - Job that proxies the dump_courses_to_clickhouse commands. + Job that proxies the dump_data_to_clickhouse commands. """ - return [("cms", f"./manage.py cms dump_courses_to_clickhouse {options}")] + return [(f"{service}", f"./manage.py {service} dump_data_to_clickhouse {options}")] # pylint: disable=line-too-long @@ -272,7 +273,7 @@ def check_superset_assets(): load_xapi_test_data, dbt, alembic, - dump_courses_to_clickhouse, + dump_data_to_clickhouse, transform_tracking_logs, ) From 71af3f7975bd755d7a7c684de70558f8464ba69f Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Wed, 7 Feb 2024 10:04:46 -0500 Subject: [PATCH 2/3] chore: reformat all files --- tutoraspects/commands_v1.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 075fa3c2f..04a1f4052 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -102,7 +102,12 @@ def alembic(command: string) -> list[tuple[str, str]]: # Ex: "tutor local do dump_data_to_clickhouse " @click.command(context_settings={"ignore_unknown_options": True}) -@click.option("--service", default="lms", type=click.UNPROCESSED, help="The service to run the command on.") +@click.option( + "--service", + default="lms", + type=click.UNPROCESSED, + help="The service to run the command on.", +) @click.option("--options", default="", type=click.UNPROCESSED) def dump_data_to_clickhouse(service, options) -> list[tuple[str, str]]: """ From 447e052f9c2e64a9d9b01d947b64dcb743a42d55 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 12 Feb 2024 13:21:13 -0500 Subject: [PATCH 3/3] feat: upgrade event-sink-clickhouse to v1.1.0 --- tutoraspects/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 1e8051773..af50fdfe8 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -41,7 +41,7 @@ ( "OPENEDX_EXTRA_PIP_REQUIREMENTS", [ - "openedx-event-sink-clickhouse==1.0.0", + "openedx-event-sink-clickhouse==1.1.0", "edx-event-routing-backends==v7.2.0", ], ),