diff --git a/Makefile b/Makefile index 160c4b8..d6655a0 100644 --- a/Makefile +++ b/Makefile @@ -37,12 +37,12 @@ migrate: migrations: @CMD="poetry run python manage.py makemigrations" $(MAKE) exec-backend-command +owasp-aggregate-projects-data: + @CMD="poetry run python manage.py owasp_aggregate_projects_data" $(MAKE) exec-backend-command + owasp-scrape-site-data: @CMD="poetry run python manage.py owasp_scrape_site_data" $(MAKE) exec-backend-command -owasp-update-projects: - @CMD="poetry run python manage.py owasp_update_projects" $(MAKE) exec-backend-command - pre-commit: @pre-commit run -a @@ -63,8 +63,8 @@ sync: \ github-sync-owasp-organization \ owasp-scrape-site-data \ github-sync-related-repositories \ - github_summarize_issues \ - owasp-update-projects + github-summarize-issues \ + owasp-aggregate-projects-data test: @docker build -f backend/Dockerfile.test backend -t nest-backend-test 2>/dev/null diff --git a/backend/apps/common/models.py b/backend/apps/common/models.py index 8978c2a..e398ac6 100644 --- a/backend/apps/common/models.py +++ b/backend/apps/common/models.py @@ -12,12 +12,12 @@ class Meta: abstract = True @staticmethod - def bulk_save(model, objects): + def bulk_save(model, objects, fields=None): """Bulk save objects.""" model.objects.bulk_create((o for o in objects if not o.id), BATCH_SIZE) model.objects.bulk_update( (o for o in objects if o.id), - fields=[field.name for field in model._meta.fields if not field.primary_key], # noqa: SLF001 + fields=fields or [field.name for field in model._meta.fields if not field.primary_key], # noqa: SLF001 batch_size=BATCH_SIZE, ) objects.clear() diff --git a/backend/apps/github/management/commands/github_summarize_issues.py b/backend/apps/github/management/commands/github_summarize_issues.py index 66f6614..2a62bd0 100644 --- a/backend/apps/github/management/commands/github_summarize_issues.py +++ b/backend/apps/github/management/commands/github_summarize_issues.py @@ -41,8 +41,8 @@ def handle(self, *args, **options): ) ) - issue.summary = open_ai.set_input(f"{issue.title}\r\n{issue.body}").complete() + issue.summary = open_ai.set_input(f"{issue.title}\r\n{issue.body}").complete() or "" issues.append(issue) # Bulk save data. - Issue.bulk_save(issues) + Issue.bulk_save(issues, fields=("summary",)) diff --git a/backend/apps/github/models/issue.py b/backend/apps/github/models/issue.py index e506352..bc0e41c 100644 --- a/backend/apps/github/models/issue.py +++ b/backend/apps/github/models/issue.py @@ -126,10 +126,9 @@ def from_github(self, gh_issue, author=None, repository=None): self.repository = repository @staticmethod - def bulk_save(issues): + def bulk_save(issues, fields=None): """Bulk save issues.""" - BulkSaveModel.bulk_save(Issue, issues) - issues.clear() + BulkSaveModel.bulk_save(Issue, issues, fields=fields) @staticmethod def update_data(gh_issue, author=None, repository=None, save=True): diff --git a/backend/apps/github/models/label.py b/backend/apps/github/models/label.py index 124fca8..1e21ec0 100644 --- a/backend/apps/github/models/label.py +++ b/backend/apps/github/models/label.py @@ -44,7 +44,6 @@ def from_github(self, gh_label): def bulk_save(labels): """Bulk save labels.""" BulkSaveModel.bulk_save(Label, labels) - labels.clear() @staticmethod def update_data(gh_label, save=True): diff --git a/backend/apps/github/models/organization.py b/backend/apps/github/models/organization.py index 53b5a1a..c5f52d9 100644 --- a/backend/apps/github/models/organization.py +++ b/backend/apps/github/models/organization.py @@ -44,7 +44,6 @@ def from_github(self, gh_organization): def bulk_save(organizations): """Bulk save organizations.""" BulkSaveModel.bulk_save(Organization, organizations) - organizations.clear() @staticmethod def update_data(gh_organization, save=True): diff --git a/backend/apps/github/models/release.py b/backend/apps/github/models/release.py index 7342cdf..ee0a997 100644 --- a/backend/apps/github/models/release.py +++ b/backend/apps/github/models/release.py @@ -67,7 +67,6 @@ def from_github(self, gh_release, author=None, repository=None): def bulk_save(releases): """Bulk save releases.""" BulkSaveModel.bulk_save(Release, releases) - releases.clear() @staticmethod def update_data(gh_release, author=None, repository=None, save=True): diff --git a/backend/apps/owasp/management/commands/owasp_update_projects.py b/backend/apps/owasp/management/commands/owasp_aggregate_projects_data.py similarity index 100% rename from backend/apps/owasp/management/commands/owasp_update_projects.py rename to backend/apps/owasp/management/commands/owasp_aggregate_projects_data.py