From f929b38d259122dbe2ca2521e52271bd524b3347 Mon Sep 17 00:00:00 2001 From: Fang Lin Date: Thu, 21 Dec 2023 17:03:35 -0800 Subject: [PATCH 1/5] adjust the schedule of two clean up jobs --- docker/prod-values.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/prod-values.yml b/docker/prod-values.yml index 9b2c382ea..7f7b7197f 100644 --- a/docker/prod-values.yml +++ b/docker/prod-values.yml @@ -178,7 +178,7 @@ cronjob: cpu: 25m memory: 32Mi - name: course-cleanup - schedule: "1 3 * 1,4,7,9 1" + schedule: "1 3 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "course"] resources: @@ -189,7 +189,7 @@ cronjob: cpu: 25m memory: 128Mi - name: seenreg-cleanup - schedule: "1 3 * 1,4,7,9 2" + schedule: "1 5 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "seenreg"] resources: From c5d92a7c732c76ffca9495d447b25e245ec1e853 Mon Sep 17 00:00:00 2001 From: Fang Lin Date: Thu, 21 Dec 2023 20:35:46 -0800 Subject: [PATCH 2/5] reschedule --- docker/prod-values.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/prod-values.yml b/docker/prod-values.yml index 7f7b7197f..3da9fcec4 100644 --- a/docker/prod-values.yml +++ b/docker/prod-values.yml @@ -178,7 +178,7 @@ cronjob: cpu: 25m memory: 32Mi - name: course-cleanup - schedule: "1 3 * * 5" + schedule: "1 7 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "course"] resources: From 78db1d4c2811ebf3ff94e802a2caa9af067b640a Mon Sep 17 00:00:00 2001 From: Fang Lin Date: Fri, 22 Dec 2023 11:38:36 -0800 Subject: [PATCH 3/5] Fix/term switch (#3039) Apply MyUW term switch, reduce data in the two largest tables --- docker/prod-values.yml | 8 ++++---- myuw/management/commands/db_cleanup.py | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docker/prod-values.yml b/docker/prod-values.yml index 3da9fcec4..6493fcef0 100644 --- a/docker/prod-values.yml +++ b/docker/prod-values.yml @@ -178,7 +178,7 @@ cronjob: cpu: 25m memory: 32Mi - name: course-cleanup - schedule: "1 7 * * 5" + schedule: "1 20 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "course"] resources: @@ -189,7 +189,7 @@ cronjob: cpu: 25m memory: 128Mi - name: seenreg-cleanup - schedule: "1 5 * * 5" + schedule: "31 20 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "seenreg"] resources: @@ -200,7 +200,7 @@ cronjob: cpu: 25m memory: 128Mi - name: notice-cleanup - schedule: "1 3 * * 3" + schedule: "1 21 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "notice"] resources: @@ -211,7 +211,7 @@ cronjob: cpu: 25m memory: 512Mi - name: linkvisit-cleanup - schedule: "1 3 * * 4" + schedule: "1 22 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "linkvisit"] resources: diff --git a/myuw/management/commands/db_cleanup.py b/myuw/management/commands/db_cleanup.py index a7c5cfe32..44747b6cc 100644 --- a/myuw/management/commands/db_cleanup.py +++ b/myuw/management/commands/db_cleanup.py @@ -14,7 +14,7 @@ from uw_sws import sws_now, SWS_TIMEZONE from myuw.models import ( VisitedLinkNew, SeenRegistration, UserNotices, UserCourseDisplay) -from myuw.dao.term import get_term_by_date, get_term_before +from myuw.dao.term import get_term_by_date, get_term_before, get_term_after from myuw.util.settings import get_cronjob_recipient, get_cronjob_sender from myuw.logger.timer import Timer @@ -65,11 +65,19 @@ def deletion(self, ids_to_delete, queryf): ["{}@uw.edu".format(get_cronjob_recipient())]) raise CommandError(msg) + def get_cur_term(self): + comparison_date = sws_now().date() + term = get_term_by_date(comparison_date) + # Match MyUW quarter switchS + if comparison_date > term.grade_submission_deadline.date(): + return get_term_after(term) + return term + def course_display(self): # clean up after one year timer = Timer() queryf = "DELETE FROM user_course_display_pref WHERE id IN ({})" - term = get_term_by_date(sws_now().date()) + term = self.get_cur_term() y = term.year - 1 q = term.quarter qset = UserCourseDisplay.objects.filter(year=y, quarter=q) @@ -88,7 +96,7 @@ def notice_read(self): # clean up after 180 days timer = Timer() queryf = "DELETE FROM myuw_mobile_usernotices WHERE id IN ({})" - cut_off_dt = self.get_cut_off_date() + cut_off_dt = self.get_cut_off_date(90) qset = UserNotices.objects.filter(first_viewed__lt=cut_off_dt) if qset.exists(): ids_to_delete = qset.values_list('id', flat=True) @@ -105,7 +113,7 @@ def registration_seen(self): # clean up previous quarters' timer = Timer() queryf = "DELETE FROM myuw_mobile_seenregistration WHERE id IN ({})" - term = get_term_before(get_term_by_date(sws_now().date())) + term = get_term_before(self.get_cur_term()) qset = SeenRegistration.objects.filter( year=term.year, quarter=term.quarter) if qset.exists(): @@ -123,7 +131,7 @@ def link_visited(self): # clean up after 180 days timer = Timer() queryf = "DELETE FROM myuw_visitedlinknew WHERE id IN ({})" - cut_off_dt = self.get_cut_off_date() + cut_off_dt = self.get_cut_off_date(90) qset = VisitedLinkNew.objects.filter(visit_date__lt=cut_off_dt) if qset.exists(): ids_to_delete = qset.values_list('id', flat=True) From 182c6b7b0185b6e44d216f657e0bd8404fbf9254 Mon Sep 17 00:00:00 2001 From: Fang Lin Date: Fri, 22 Dec 2023 11:49:39 -0800 Subject: [PATCH 4/5] reschdule --- docker/prod-values.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/prod-values.yml b/docker/prod-values.yml index 6493fcef0..4100c79fd 100644 --- a/docker/prod-values.yml +++ b/docker/prod-values.yml @@ -178,7 +178,7 @@ cronjob: cpu: 25m memory: 32Mi - name: course-cleanup - schedule: "1 20 * * 5" + schedule: "21 20 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "course"] resources: @@ -189,7 +189,7 @@ cronjob: cpu: 25m memory: 128Mi - name: seenreg-cleanup - schedule: "31 20 * * 5" + schedule: "1 23 * * 5" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "seenreg"] resources: From a6f1d8cc8f4f8d5df033f6805e096581af03e7ea Mon Sep 17 00:00:00 2001 From: Fang Lin Date: Thu, 28 Dec 2023 14:18:44 -0800 Subject: [PATCH 5/5] Resolve MUWM-5196: switch over to postgres DB (#3034) Resolve MUWM-5196: switch over to postgres DB --- Dockerfile | 4 +-- docker/dev-values.yml | 6 ++-- docker/prod-values.yml | 34 +++++++++++------------ docker/test-values.yml | 38 +++++++++++++------------- docker/vue-values.yml | 6 ++-- myuw/management/commands/db_cleanup.py | 4 +-- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index a474fe667..5703d8d28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG DJANGO_CONTAINER_VERSION=1.4.2 FROM us-docker.pkg.dev/uwit-mci-axdd/containers/django-container:${DJANGO_CONTAINER_VERSION} as app-prewebpack-container USER root -RUN apt-get update && apt-get install mysql-client libmysqlclient-dev -y +RUN apt-get update && apt-get install -y postgresql-client libpq-dev USER acait ADD --chown=acait:acait . /app/ @@ -11,7 +11,7 @@ ADD --chown=acait:acait docker/app_start.sh /scripts RUN chmod u+x /scripts/app_start.sh RUN /app/bin/pip install -r requirements.txt -RUN /app/bin/pip install mysqlclient +RUN /app/bin/pip install psycopg2 FROM node:16.3-stretch AS node-bundler diff --git a/docker/dev-values.yml b/docker/dev-values.yml index 9c202df83..2bada3faf 100644 --- a/docker/dev-values.yml +++ b/docker/dev-values.yml @@ -29,10 +29,10 @@ externalService: enabled: true name: dev-my-uw-db type: ClusterIP - serviceAddress: 172.18.0.19 - servicePort: 3306 + serviceAddress: 172.18.1.210 + servicePort: 5432 database: - engine: mysql + engine: postgres name: myuw_dev hostname: dev-my-uw-db secretName: dev.my.uw.edu-sql-secrets diff --git a/docker/prod-values.yml b/docker/prod-values.yml index 4100c79fd..af705a190 100644 --- a/docker/prod-values.yml +++ b/docker/prod-values.yml @@ -61,10 +61,10 @@ externalService: enabled: true name: my-uw-db type: ClusterIP - serviceAddress: 172.18.0.19 - servicePort: 3306 + serviceAddress: 172.18.1.210 + servicePort: 5432 database: - engine: mysql + engine: postgres name: myuw hostname: my-uw-db secretName: my.uw.edu-sql-secrets @@ -178,49 +178,49 @@ cronjob: cpu: 25m memory: 32Mi - name: course-cleanup - schedule: "21 20 * * 5" + schedule: "1 3 21,26,30 3,6,8,12 *" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "course"] resources: limits: - cpu: 75m - memory: 768Mi + cpu: 50m + memory: 512Mi requests: cpu: 25m memory: 128Mi - name: seenreg-cleanup - schedule: "1 23 * * 5" + schedule: "1 4 21,26,30 3,6,8,12 *" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "seenreg"] resources: limits: - cpu: 75m - memory: 768Mi + cpu: 50m + memory: 512Mi requests: cpu: 25m memory: 128Mi - name: notice-cleanup - schedule: "1 21 * * 5" + schedule: "1 5 * * 0" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "notice"] resources: limits: - cpu: 75m - memory: 1024Mi + cpu: 50m + memory: 768Mi requests: cpu: 25m - memory: 512Mi + memory: 256Mi - name: linkvisit-cleanup - schedule: "1 22 * * 5" + schedule: "1 6 * * 0" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "linkvisit"] resources: limits: - cpu: 100m - memory: 1280Mi + cpu: 50m + memory: 768Mi requests: cpu: 25m - memory: 512Mi + memory: 256Mi environmentVariables: - name: AUTH value: SAML diff --git a/docker/test-values.yml b/docker/test-values.yml index 68622d3b5..180f955df 100644 --- a/docker/test-values.yml +++ b/docker/test-values.yml @@ -51,10 +51,10 @@ externalService: enabled: true name: test-my-uw-db type: ClusterIP - serviceAddress: 172.18.0.19 - servicePort: 3306 + serviceAddress: 172.18.1.210 + servicePort: 5432 database: - engine: mysql + engine: postgres name: myuw_test hostname: test-my-uw-db secretName: test.my.uw.edu-sql-secrets @@ -114,49 +114,49 @@ cronjob: cpu: 25m memory: 32Mi - name: course-cleanup - schedule: "1 3 * 1,4,7,9 1" + schedule: "1 3 21,26,30 3,6,8,12 *" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "course"] resources: limits: - cpu: 75m - memory: 128Mi + cpu: 50m + memory: 512Mi requests: cpu: 25m - memory: 32Mi + memory: 128Mi - name: seenreg-cleanup - schedule: "1 3 * 1,4,7,9 2" + schedule: "1 4 21,26,30 3,6,8,12 *" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "seenreg"] resources: limits: - cpu: 75m - memory: 128Mi + cpu: 50m + memory: 512Mi requests: cpu: 25m - memory: 32Mi + memory: 128Mi - name: notice-cleanup - schedule: "1 3 * * 3" + schedule: "1 5 * * 0" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "notice"] resources: limits: - cpu: 75m - memory: 128Mi + cpu: 50m + memory: 512Mi requests: cpu: 25m - memory: 32Mi + memory: 256Mi - name: linkvisit-cleanup - schedule: "1 3 * * 4" + schedule: "1 6 * * 0" command: ["/scripts/management_command.sh"] args: ["db_cleanup", "linkvisit"] resources: limits: - cpu: 75m - memory: 128Mi + cpu: 50m + memory: 512Mi requests: cpu: 25m - memory: 32Mi + memory: 256Mi environmentVariables: - name: AUTH value: SAML diff --git a/docker/vue-values.yml b/docker/vue-values.yml index ba11fbc29..2eb1e48d5 100644 --- a/docker/vue-values.yml +++ b/docker/vue-values.yml @@ -29,10 +29,10 @@ externalService: enabled: true name: vue-my-uw-db type: ClusterIP - serviceAddress: 172.18.0.19 - servicePort: 3306 + serviceAddress: 172.18.1.210 + servicePort: 5432 database: - engine: mysql + engine: postgres name: myuw_test hostname: vue-my-uw-db secretName: vue.my.uw.edu-sql-secrets diff --git a/myuw/management/commands/db_cleanup.py b/myuw/management/commands/db_cleanup.py index 44747b6cc..58b6e6496 100644 --- a/myuw/management/commands/db_cleanup.py +++ b/myuw/management/commands/db_cleanup.py @@ -96,7 +96,7 @@ def notice_read(self): # clean up after 180 days timer = Timer() queryf = "DELETE FROM myuw_mobile_usernotices WHERE id IN ({})" - cut_off_dt = self.get_cut_off_date(90) + cut_off_dt = self.get_cut_off_date() qset = UserNotices.objects.filter(first_viewed__lt=cut_off_dt) if qset.exists(): ids_to_delete = qset.values_list('id', flat=True) @@ -131,7 +131,7 @@ def link_visited(self): # clean up after 180 days timer = Timer() queryf = "DELETE FROM myuw_visitedlinknew WHERE id IN ({})" - cut_off_dt = self.get_cut_off_date(90) + cut_off_dt = self.get_cut_off_date() qset = VisitedLinkNew.objects.filter(visit_date__lt=cut_off_dt) if qset.exists(): ids_to_delete = qset.values_list('id', flat=True)