From 719df124a083afca6931db655e4f1a14ceb2e4d8 Mon Sep 17 00:00:00 2001 From: Arkadii Yakovets Date: Sun, 2 Mar 2025 17:30:30 -0800 Subject: [PATCH] Update code --- backend/apps/github/admin.py | 2 +- backend/apps/github/common.py | 124 ++++++++---------- .../apps/github/models/generic_issue_model.py | 5 + backend/apps/github/models/issue.py | 9 +- backend/apps/github/models/repository.py | 5 + backend/apps/owasp/models/project.py | 2 +- 6 files changed, 70 insertions(+), 77 deletions(-) diff --git a/backend/apps/github/admin.py b/backend/apps/github/admin.py index 3eef6af8d..32dfb747d 100644 --- a/backend/apps/github/admin.py +++ b/backend/apps/github/admin.py @@ -147,10 +147,10 @@ class UserAdmin(admin.ModelAdmin): search_fields = ("login", "name") -admin.site.register(PullRequest, PullRequestAdmin) admin.site.register(Issue, IssueAdmin) admin.site.register(Label, LabelAdmin) admin.site.register(Organization, OrganizationAdmin) +admin.site.register(PullRequest, PullRequestAdmin) admin.site.register(Release, ReleaseAdmin) admin.site.register(Repository, RepositoryAdmin) admin.site.register(RepositoryContributor, RepositoryContributorAdmin) diff --git a/backend/apps/github/common.py b/backend/apps/github/common.py index ce7b4ea08..3aa28fef7 100644 --- a/backend/apps/github/common.py +++ b/backend/apps/github/common.py @@ -1,7 +1,9 @@ """GitHub app common module.""" import logging +from datetime import timedelta as td +from django.utils import timezone from github.GithubException import UnknownObjectException from apps.github.models.issue import Issue @@ -46,90 +48,62 @@ def sync_repository(gh_repository, organization=None, user=None): user=user, ) - # GitHub repository issues. - if ( - not repository.is_archived - and repository.track_issues - and repository.project - and repository.project.track_issues - ): + if not repository.is_archived: + # GitHub repository issues. + project_track_issues = repository.project.track_issues if repository.project else True + if repository.track_issues and project_track_issues: + kwargs = { + "direction": "asc", + "sort": "created", + "state": "all", + } + if latest_updated_issue := repository.latest_updated_issue: + # Get only what has been updated after the latest sync. + kwargs.update({"since": latest_updated_issue.updated_at}) + + for gh_issue in gh_repository.get_issues(**kwargs): + if gh_issue.pull_request: # Skip pull requests. + continue + + author = User.update_data(gh_issue.user) + issue = Issue.update_data(gh_issue, author=author, repository=repository) + + # Assignees. + issue.assignees.clear() + for gh_issue_assignee in gh_issue.assignees: + issue.assignees.add(User.update_data(gh_issue_assignee)) + + # Labels. + issue.labels.clear() + for gh_issue_label in gh_issue.labels: + try: + issue.labels.add(Label.update_data(gh_issue_label)) + except UnknownObjectException: + logger.info("Couldn't get GitHub issue label %s", issue.url) + else: + logger.info("Skipping issues sync for %s", repository.name) + + # GitHub repository pull requests. kwargs = { - "direction": "asc", - "sort": "created", + "direction": "desc", + "sort": "updated", "state": "all", } - if latest_updated_issue := repository.latest_updated_issue: - # Get only what has been updated after the latest sync. - kwargs.update({"since": latest_updated_issue.updated_at}) - - for gh_issue in gh_repository.get_issues(**kwargs): - if gh_issue.pull_request: # Skip pull requests. - continue + pull_request_cut_off_at = timezone.now() - td(days=30) + latest_updated_pull_request = repository.latest_updated_pull_request + for gh_pull_request in gh_repository.get_pulls(**kwargs): author = User.update_data(gh_issue.user) - issue = Issue.update_data(gh_issue, author=author, repository=repository) - - # Assignees. - issue.assignees.clear() - for gh_issue_assignee in gh_issue.assignees: - issue.assignees.add(User.update_data(gh_issue_assignee)) - - # Labels. - issue.labels.clear() - for gh_issue_label in gh_issue.labels: - try: - issue.labels.add(Label.update_data(gh_issue_label)) - except UnknownObjectException: - logger.info("Couldn't get GitHub issue label %s", issue.url) - else: - logger.info("Skipping issues sync for %s", repository.name) - - if not repository.is_archived and repository.project: - # Fetch both open and closed PRs from GitHub - kwargs = { - "direction": "desc", - "sort": "created", - "state": "open", - } - - latest_pull_request = repository.latest_pull_request - if latest_pull_request: - gh_first_pr = PullRequest.objects.order_by("created_at").first().created_at - kwargs["state"] = "all" - - gh_pull_requests = gh_repository.get_pulls(**kwargs) - - for gh_pull_request in gh_pull_requests: - if latest_pull_request and gh_pull_request.state == "closed": - # Skipping closed PR before first sync - if gh_first_pr > gh_pull_request.created_at: - break - # Check if this PR already exists in the database and is open - existing_open_pr = PullRequest.objects.filter( - repository=repository, state="open", number=gh_pull_request.number - ).first() - - if not existing_open_pr: - continue # Skip closed PRs from previous syncs - - # Extract author details - author = ( - User.update_data(gh_pull_request.user) - if gh_pull_request.user and gh_pull_request.user.type != "Bot" - else None - ) - - # Update PR data pull_request = PullRequest.update_data( gh_pull_request, author=author, repository=repository ) - # Clear and update assignees + # Assignees. pull_request.assignees.clear() for gh_pull_request_assignee in gh_pull_request.assignees: pull_request.assignees.add(User.update_data(gh_pull_request_assignee)) - # Clear and update labels + # Labels. pull_request.labels.clear() for gh_pull_request_label in gh_pull_request.labels: try: @@ -137,6 +111,14 @@ def sync_repository(gh_repository, organization=None, user=None): except UnknownObjectException: logger.info("Couldn't get GitHub pull request label %s", pull_request.url) + pull_request_cut_off = pull_request.updated_at <= pull_request_cut_off_at + pull_request_seen = ( + latest_updated_pull_request + and pull_request.updated_at <= latest_updated_pull_request.updated_at + ) + if pull_request_seen or pull_request_cut_off: + break + # GitHub repository releases. releases = [] if not is_owasp_site_repository: diff --git a/backend/apps/github/models/generic_issue_model.py b/backend/apps/github/models/generic_issue_model.py index 6485aa985..89267ee8d 100644 --- a/backend/apps/github/models/generic_issue_model.py +++ b/backend/apps/github/models/generic_issue_model.py @@ -36,6 +36,11 @@ def __str__(self): """Issue human readable representation.""" return f"{self.title} by {self.author}" + @property + def is_open(self): + """Return whether issue is open.""" + return self.state == self.State.OPEN + @property def project(self): """Return project.""" diff --git a/backend/apps/github/models/issue.py b/backend/apps/github/models/issue.py index 125ad7ebb..e672ef01c 100644 --- a/backend/apps/github/models/issue.py +++ b/backend/apps/github/models/issue.py @@ -133,11 +133,12 @@ def generate_summary(self, open_ai=None, max_tokens=500): def save(self, *args, **kwargs): """Save issue.""" - if not self.hint: - self.generate_hint() + if self.is_open: + if not self.hint: + self.generate_hint() - if not self.summary: - self.generate_summary() + if not self.summary: + self.generate_summary() super().save(*args, **kwargs) diff --git a/backend/apps/github/models/repository.py b/backend/apps/github/models/repository.py index 405821473..36045802d 100644 --- a/backend/apps/github/models/repository.py +++ b/backend/apps/github/models/repository.py @@ -116,6 +116,11 @@ def latest_updated_issue(self): """Repository latest updated issue.""" return self.issues.order_by("-updated_at").first() + @property + def latest_updated_pull_request(self): + """Repository latest updated pull request.""" + return self.pull_requests.order_by("-updated_at").first() + @property def nest_key(self): """Return repository Nest key.""" diff --git a/backend/apps/owasp/models/project.py b/backend/apps/owasp/models/project.py index cb21c23dd..c2c581a09 100644 --- a/backend/apps/owasp/models/project.py +++ b/backend/apps/owasp/models/project.py @@ -224,7 +224,7 @@ def from_github(self, repository): def save(self, *args, **kwargs): """Save project.""" - if not self.summary and (prompt := Prompt.get_owasp_project_summary()): + if self.is_active and not self.summary and (prompt := Prompt.get_owasp_project_summary()): self.generate_summary(prompt=prompt) super().save(*args, **kwargs)