From 3c7aa880908965b649c7d5d3d2b04670597c4412 Mon Sep 17 00:00:00 2001 From: Arkadii Yakovets Date: Wed, 4 Sep 2024 10:37:08 -0700 Subject: [PATCH] Add GitHub issues. Refactor sync process. --- .gitignore | 1 + backend/apps/common/models.py | 16 +++ backend/apps/github/admin.py | 33 ++++- .../github_sync_owasp_organization.py | 102 +++----------- .../github_sync_related_repositories.py | 31 ++--- ...4_label_alter_release_sequence_id_issue.py | 128 ++++++++++++++++++ .../migrations/0035_alter_issue_labels.py | 19 +++ .../migrations/0036_alter_issue_state.py | 22 +++ .../github/migrations/0037_issue_assignees.py | 19 +++ ...issue_assignee_issue_closed_at_and_more.py | 26 ++++ backend/apps/github/models/__init__.py | 97 +++++++------ backend/apps/github/models/common.py | 5 + backend/apps/github/models/issue.py | 123 +++++++++++++++++ backend/apps/github/models/label.py | 62 +++++++++ backend/apps/github/models/organization.py | 25 +++- backend/apps/github/models/release.py | 27 +++- backend/apps/github/models/repository.py | 30 ++++ backend/apps/github/models/user.py | 15 ++ backend/apps/github/utils.py | 5 - backend/apps/owasp/models/chapter.py | 25 +++- backend/apps/owasp/models/committee.py | 25 +++- backend/apps/owasp/models/event.py | 25 +++- backend/apps/owasp/models/project.py | 25 +++- backend/pyproject.toml | 26 +++- backend/settings/base.py | 3 + 25 files changed, 742 insertions(+), 173 deletions(-) create mode 100644 backend/apps/github/migrations/0034_label_alter_release_sequence_id_issue.py create mode 100644 backend/apps/github/migrations/0035_alter_issue_labels.py create mode 100644 backend/apps/github/migrations/0036_alter_issue_state.py create mode 100644 backend/apps/github/migrations/0037_issue_assignees.py create mode 100644 backend/apps/github/migrations/0038_remove_issue_assignee_issue_closed_at_and_more.py create mode 100644 backend/apps/github/models/issue.py create mode 100644 backend/apps/github/models/label.py diff --git a/.gitignore b/.gitignore index 122dee1ce..e06f2ad21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ __pycache__ .bash_history .cache +.coverage .env .local .python_history diff --git a/backend/apps/common/models.py b/backend/apps/common/models.py index 71a8826e3..274ad4ffb 100644 --- a/backend/apps/common/models.py +++ b/backend/apps/common/models.py @@ -3,6 +3,22 @@ from django.db import models +class BulkSaveModel(models.Model): + """Base model for bulk save action.""" + + class Meta: + abstract = True + + @staticmethod + def bulk_save(model, objects): + """Bulk save objects.""" + model.objects.bulk_create(o for o in objects if not o.id) + model.objects.bulk_update( + [o for o in objects if o.id], + fields=[field.name for field in model._meta.fields if not field.primary_key], # noqa: SLF001 + ) + + class TimestampedModel(models.Model): """Base model with auto created_at and updated_at fields.""" diff --git a/backend/apps/github/admin.py b/backend/apps/github/admin.py index 39b3ec59b..6a319a92e 100644 --- a/backend/apps/github/admin.py +++ b/backend/apps/github/admin.py @@ -3,7 +3,36 @@ from django.contrib import admin from django.utils.safestring import mark_safe -from apps.github.models import Organization, Release, Repository, User +from apps.github.models import Issue, Label, Organization, Release, Repository, User + + +class LabelAdmin(admin.ModelAdmin): + search_fields = ("name", "description") + + +class IssueAdmin(admin.ModelAdmin): + autocomplete_fields = ( + "repository", + "author", + "assignees", + "labels", + ) + list_display = ( + "repository", + "title", + "custom_field_github_url", + ) + list_filter = ( + "state", + "is_locked", + ) + search_fields = ("title", "description") + + def custom_field_github_url(self, obj): + """Issue GitHub URL.""" + return mark_safe(f"↗️") # noqa: S308 + + custom_field_github_url.short_description = "GitHub 🔗" class RepositoryAdmin(admin.ModelAdmin): @@ -69,6 +98,8 @@ class UserAdmin(admin.ModelAdmin): search_fields = ("name",) +admin.site.register(Issue, IssueAdmin) +admin.site.register(Label, LabelAdmin) admin.site.register(Organization, OrganizationAdmin) admin.site.register(Release, ReleaseAdmin) admin.site.register(Repository, RepositoryAdmin) diff --git a/backend/apps/github/management/commands/github_sync_owasp_organization.py b/backend/apps/github/management/commands/github_sync_owasp_organization.py index 0224f0c0e..058aa718f 100644 --- a/backend/apps/github/management/commands/github_sync_owasp_organization.py +++ b/backend/apps/github/management/commands/github_sync_owasp_organization.py @@ -6,11 +6,11 @@ from django.core.management.base import BaseCommand from apps.github.constants import GITHUB_ITEMS_PER_PAGE -from apps.github.models import Organization, Release, Repository, fetch_repository_data +from apps.github.models import Issue, Organization, Release, Repository, sync_repository from apps.owasp.constants import OWASP_ORGANIZATION_NAME from apps.owasp.models import Chapter, Committee, Event, Project -BATCH_SIZE = 100 +BATCH_SIZE = 10 class Command(BaseCommand): @@ -19,62 +19,14 @@ class Command(BaseCommand): def handle(self, *_args, **_options): def save_data(): """Save data to DB.""" - # Organizations. - Organization.objects.bulk_create(o for o in organizations if not o.id) - Organization.objects.bulk_update( - (o for o in organizations if o.id), - fields=[ - field.name - for field in Organization._meta.fields # noqa: SLF001 - if not field.primary_key - ], - ) - organizations.clear() - - # Repositories. - Repository.objects.bulk_create(r for r in repositories if not r.id) - Repository.objects.bulk_update( - [r for r in repositories if r.id], - fields=[field.name for field in Repository._meta.fields if not field.primary_key], # noqa: SLF001 - ) - repositories.clear() + Organization.bulk_save(organizations) + Issue.bulk_save(issues) + Release.bulk_save(releases) - # Releases. - Release.objects.bulk_create(r for r in releases if not r.id) - Release.objects.bulk_update( - (r for r in releases if r.id), - fields=[field.name for field in Release._meta.fields if not field.primary_key], # noqa: SLF001 - ) - releases.clear() - - # Chapters. - Chapter.objects.bulk_create(c for c in chapters if not c.id) - Chapter.objects.bulk_update( - (c for c in chapters if c.id), - fields=[field.name for field in Chapter._meta.fields if not field.primary_key], # noqa: SLF001 - ) - chapters.clear() - - # Committees. - Committee.objects.bulk_create(c for c in committees if not c.id) - Committee.objects.bulk_update( - (c for c in committees if c.id), - fields=[field.name for field in Committee._meta.fields if not field.primary_key], # noqa: SLF001 - ) - - # Events. - Event.objects.bulk_create(e for e in events if not e.id) - Event.objects.bulk_update( - [e for e in events if not e.id], - fields=[field.name for field in Event._meta.fields if not field.primary_key], # noqa: SLF001 - ) - - # Projects. - Project.objects.bulk_create(p for p in projects if not p.id) - Project.objects.bulk_update( - (p for p in projects if p.id), - fields=[field.name for field in Project._meta.fields if not field.primary_key], # noqa: SLF001 - ) + Chapter.bulk_save(chapters) + Committee.bulk_save(committees) + Event.bulk_save(events) + Project.bulk_save(projects) gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE) gh_owasp_organization = gh.get_organization(OWASP_ORGANIZATION_NAME) @@ -86,59 +38,39 @@ def save_data(): chapters = [] committees = [] events = [] + issues = [] organizations = [] projects = [] releases = [] - repositories = [] for idx, gh_repository in enumerate( - gh_owasp_organization.get_repos(type="public", sort="created", direction="desc")[0:] + gh_owasp_organization.get_repos(type="public", sort="created", direction="asc") ): print(f"{idx + 1:<3} {gh_repository.name}") - owasp_organization, new_repository, new_releases = fetch_repository_data( + owasp_organization, repository, new_releases = sync_repository( gh_repository, organization=owasp_organization, user=owasp_user ) if not owasp_organization.id: owasp_organization.save() - repositories.append(new_repository) + releases.extend(new_releases) entity_key = gh_repository.name.lower() # OWASP chapters. if entity_key.startswith("www-chapter-"): - try: - chapter = Chapter.objects.get(key=entity_key) - except Chapter.DoesNotExist: - chapter = Chapter(key=entity_key) - chapter.from_github(gh_repository, new_repository) - chapters.append(chapter) + chapters.append(Chapter.update_data(gh_repository, repository, save=False)) # OWASP projects. elif entity_key.startswith("www-project-"): - try: - project = Project.objects.get(key=entity_key) - except Project.DoesNotExist: - project = Project(key=entity_key) - project.from_github(gh_repository, new_repository) - projects.append(project) + projects.append(Project.update_data(gh_repository, repository, save=False)) # OWASP events. elif entity_key.startswith("www-event-"): - try: - event = Event.objects.get(key=entity_key) - except Event.DoesNotExist: - event = Event(key=entity_key) - event.from_github(gh_repository, new_repository) - events.append(event) + events.append(Event.update_data(gh_repository, repository, save=False)) # OWASP committees. elif entity_key.startswith("www-committee-"): - try: - committee = Committee.objects.get(key=entity_key) - except Committee.DoesNotExist: - committee = Committee(key=entity_key) - committee.from_github(gh_repository, new_repository) - committees.append(committee) + committees.append(Committee.update_data(gh_repository, repository, save=False)) if idx % BATCH_SIZE == 0: save_data() diff --git a/backend/apps/github/management/commands/github_sync_related_repositories.py b/backend/apps/github/management/commands/github_sync_related_repositories.py index cbe80191d..8ab3f2545 100644 --- a/backend/apps/github/management/commands/github_sync_related_repositories.py +++ b/backend/apps/github/management/commands/github_sync_related_repositories.py @@ -8,13 +8,13 @@ from github.GithubException import UnknownObjectException from apps.github.constants import GITHUB_ITEMS_PER_PAGE -from apps.github.models import Release, fetch_repository_data +from apps.github.models import Issue, Release, sync_repository from apps.github.utils import get_repository_path from apps.owasp.models import Project logger = logging.getLogger(__name__) -BATCH_SIZE = 100 +BATCH_SIZE = 10 class Command(BaseCommand): @@ -23,27 +23,14 @@ class Command(BaseCommand): def handle(self, *args, **_options): def save_data(): """Save data to DB.""" - # Releases. - Release.objects.bulk_create(r for r in releases if not r.id) - Release.objects.bulk_update( - (r for r in releases if r.id), - fields=[field.name for field in Release._meta.fields if not field.primary_key], # noqa: SLF001 - ) - releases.clear() - - # Projects. - Project.objects.bulk_update( - (p for p in projects), - fields=[field.name for field in Project._meta.fields if not field.primary_key], # noqa: SLF001 - ) - projects.clear() - - active_projects = Project.objects.filter(is_active=True).order_by( - "owasp_repository__created_at" - ) + Issue.bulk_save(issues) + Release.bulk_save(releases) + Project.bulk_save(projects) + active_projects = Project.objects.filter(is_active=True).order_by("created_at") gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE) + issues = [] projects = [] releases = [] for idx, project in enumerate(active_projects): @@ -64,10 +51,10 @@ def save_data(): project.save(update_fields=("repositories_raw",)) continue - organization, repository, new_releases = fetch_repository_data(gh_repository) + organization, repository, new_releases = sync_repository(gh_repository) if organization is not None: organization.save() - repository.save() + project.repositories.add(repository) releases.extend(new_releases) diff --git a/backend/apps/github/migrations/0034_label_alter_release_sequence_id_issue.py b/backend/apps/github/migrations/0034_label_alter_release_sequence_id_issue.py new file mode 100644 index 000000000..7b9e2baf3 --- /dev/null +++ b/backend/apps/github/migrations/0034_label_alter_release_sequence_id_issue.py @@ -0,0 +1,128 @@ +# Generated by Django 5.1.1 on 2024-09-04 00:00 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0033_alter_release_repository"), + ] + + operations = [ + migrations.CreateModel( + name="Label", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("node_id", models.CharField(unique=True, verbose_name="Node ID")), + ("name", models.CharField(max_length=100, verbose_name="Name")), + ("description", models.CharField(max_length=200, verbose_name="Description")), + ("color", models.CharField(default="", max_length=6, verbose_name="Color")), + ( + "sequence_id", + models.PositiveBigIntegerField(default=0, verbose_name="Label ID"), + ), + ("is_default", models.BooleanField(default=False, verbose_name="Is default")), + ], + options={ + "verbose_name_plural": "Labels", + "db_table": "github_labels", + }, + ), + migrations.AlterField( + model_name="release", + name="sequence_id", + field=models.PositiveBigIntegerField(default=0, verbose_name="Release ID"), + ), + migrations.CreateModel( + name="Issue", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("node_id", models.CharField(unique=True, verbose_name="Node ID")), + ("title", models.CharField(max_length=200, verbose_name="Title")), + ("body", models.TextField(default="", verbose_name="Body")), + ( + "state", + models.CharField( + choices=[("open", "Open")], + default="open", + max_length=20, + verbose_name="State", + ), + ), + ("url", models.URLField(default="", verbose_name="URL")), + ("number", models.PositiveBigIntegerField(default=0, verbose_name="Number")), + ( + "sequence_id", + models.PositiveBigIntegerField(default=0, verbose_name="Issue ID"), + ), + ("is_locked", models.BooleanField(default=False, verbose_name="Is locked")), + ( + "lock_reason", + models.CharField(default="", max_length=200, verbose_name="Lock reason"), + ), + ( + "comments_count", + models.PositiveIntegerField(default=0, verbose_name="Comments"), + ), + ("created_at", models.DateTimeField(verbose_name="Created at")), + ("updated_at", models.DateTimeField(verbose_name="Updated at")), + ( + "assignee", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="assigned_issues", + to="github.user", + verbose_name="Assignee", + ), + ), + ( + "author", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_issues", + to="github.user", + verbose_name="Author", + ), + ), + ( + "repository", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="issues", + to="github.repository", + ), + ), + ( + "labels", + models.ManyToManyField( + blank=True, related_name="+", to="github.label", verbose_name="Labels" + ), + ), + ], + options={ + "verbose_name_plural": "Issues", + "db_table": "github_issues", + }, + ), + ] diff --git a/backend/apps/github/migrations/0035_alter_issue_labels.py b/backend/apps/github/migrations/0035_alter_issue_labels.py new file mode 100644 index 000000000..00202f9d7 --- /dev/null +++ b/backend/apps/github/migrations/0035_alter_issue_labels.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.1 on 2024-09-04 00:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0034_label_alter_release_sequence_id_issue"), + ] + + operations = [ + migrations.AlterField( + model_name="issue", + name="labels", + field=models.ManyToManyField( + blank=True, related_name="issue", to="github.label", verbose_name="Labels" + ), + ), + ] diff --git a/backend/apps/github/migrations/0036_alter_issue_state.py b/backend/apps/github/migrations/0036_alter_issue_state.py new file mode 100644 index 000000000..2b7d7079c --- /dev/null +++ b/backend/apps/github/migrations/0036_alter_issue_state.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.1 on 2024-09-04 00:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0035_alter_issue_labels"), + ] + + operations = [ + migrations.AlterField( + model_name="issue", + name="state", + field=models.CharField( + choices=[("open", "Open"), ("closed", "Closed")], + default="open", + max_length=20, + verbose_name="State", + ), + ), + ] diff --git a/backend/apps/github/migrations/0037_issue_assignees.py b/backend/apps/github/migrations/0037_issue_assignees.py new file mode 100644 index 000000000..f61bd606f --- /dev/null +++ b/backend/apps/github/migrations/0037_issue_assignees.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.1 on 2024-09-04 01:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0036_alter_issue_state"), + ] + + operations = [ + migrations.AddField( + model_name="issue", + name="assignees", + field=models.ManyToManyField( + blank=True, related_name="issue", to="github.user", verbose_name="Assignees" + ), + ), + ] diff --git a/backend/apps/github/migrations/0038_remove_issue_assignee_issue_closed_at_and_more.py b/backend/apps/github/migrations/0038_remove_issue_assignee_issue_closed_at_and_more.py new file mode 100644 index 000000000..f0264d668 --- /dev/null +++ b/backend/apps/github/migrations/0038_remove_issue_assignee_issue_closed_at_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 5.1.1 on 2024-09-04 15:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0037_issue_assignees"), + ] + + operations = [ + migrations.RemoveField( + model_name="issue", + name="assignee", + ), + migrations.AddField( + model_name="issue", + name="closed_at", + field=models.DateTimeField(blank=True, null=True, verbose_name="Closed at"), + ), + migrations.AddField( + model_name="issue", + name="state_reason", + field=models.CharField(default="", max_length=200, verbose_name="State reason"), + ), + ] diff --git a/backend/apps/github/models/__init__.py b/backend/apps/github/models/__init__.py index a3ffbb61f..e1dd667a2 100644 --- a/backend/apps/github/models/__init__.py +++ b/backend/apps/github/models/__init__.py @@ -1,14 +1,18 @@ """Github app.""" +from django.conf import settings + +from apps.github.models.issue import Issue +from apps.github.models.label import Label from apps.github.models.organization import Organization from apps.github.models.release import Release from apps.github.models.repository import Repository from apps.github.models.user import User -from apps.github.utils import check_owasp_site_repository, get_node_id +from apps.github.utils import check_owasp_site_repository -def fetch_repository_data(gh_repository, organization=None, user=None): - """Fetch GitHub repository data.""" +def sync_repository(gh_repository, organization=None, user=None): + """Sync GitHub repository data.""" entity_key = gh_repository.name.lower() is_owasp_site_repository = check_owasp_site_repository(entity_key) @@ -16,36 +20,18 @@ def fetch_repository_data(gh_repository, organization=None, user=None): if organization is None: gh_organization = gh_repository.organization if gh_organization is not None: - organization_node_id = get_node_id(gh_organization) - try: - organization = Organization.objects.get(node_id=organization_node_id) - except Organization.DoesNotExist: - organization = Organization(node_id=organization_node_id) - organization.from_github(gh_organization) + organization = Organization.update_data(gh_organization, save=False) # GitHub repository owner. if user is None: - gh_user = gh_repository.owner - # if gh_user is not None: - user_node_id = get_node_id(gh_user) - try: - user = User.objects.get(node_id=user_node_id) - except User.DoesNotExist: - user = User(node_id=user_node_id) - user.from_github(gh_user) - user.save() + user = User.update_data(gh_repository.owner) # GitHub repository. commits = gh_repository.get_commits() contributors = gh_repository.get_contributors() languages = None if is_owasp_site_repository else gh_repository.get_languages() - repository_node_id = get_node_id(gh_repository) - try: - repository = Repository.objects.get(node_id=repository_node_id) - except Repository.DoesNotExist: - repository = Repository(node_id=repository_node_id) - repository.from_github( + repository = Repository.update_data( gh_repository, commits=commits, contributors=contributors, @@ -54,6 +40,44 @@ def fetch_repository_data(gh_repository, organization=None, user=None): user=user, ) + # GitHub repository issues. + if not repository.is_archived: + # Sync open issues for the first run. + kwargs = { + "direction": "asc", + "sort": "created", + "state": "open", + } + latest_issue = Issue.objects.filter(repository=repository).order_by("-updated_at").first() + if latest_issue: + # Sync open/closed issues for subsequent runs. + kwargs.update( + { + "since": latest_issue.updated_at, + "state": "all", + } + ) + for gh_issue in gh_repository.get_issues(**kwargs): + # Skip pull requests. + if gh_issue.pull_request: + continue + + # GitHub issue author. + if gh_issue.user is not None: + author = User.update_data(gh_issue.user) + + issue = Issue.update_data(gh_issue, author=author, repository=repository) + + # Assignees. + issue.assignees.clear() + for gh_issue_assignee in gh_issue.assignees: + issue.assignees.add(User.update_data(gh_issue_assignee)) + + # Labels. + issue.labels.clear() + for gh_issue_label in gh_issue.labels: + issue.labels.add(Label.update_data(gh_issue_label)) + # GitHub repository releases. releases = [] if not is_owasp_site_repository: @@ -63,32 +87,15 @@ def fetch_repository_data(gh_repository, organization=None, user=None): else () ) for gh_release in gh_repository.get_releases(): - release_node_id = get_node_id(gh_release) + release_node_id = Release.get_node_id(gh_release) if release_node_id in existing_release_node_ids: break # GitHub release author. - gh_user = gh_release.author - if gh_user is not None: - author_node_id = get_node_id(gh_user) - try: - author = User.objects.get(node_id=author_node_id) - except User.DoesNotExist: - author = User(node_id=author_node_id) - author.from_github(gh_user) - author.save() - - # if author_node_id not in updated_users: - # updated_users.add(author_node_id) - # author.from_github(gh_user) - # author.save() + if gh_release.author is not None: + author = User.update_data(gh_release.author) # GitHub release. - try: - release = Release.objects.get(node_id=release_node_id) - except Release.DoesNotExist: - release = Release(node_id=release_node_id) - release.from_github(gh_release, author=author, repository=repository) - releases.append(release) + releases.append(Release.update_data(gh_release, author=author, repository=repository)) return organization, repository, releases diff --git a/backend/apps/github/models/common.py b/backend/apps/github/models/common.py index 640769e7b..8eac123ab 100644 --- a/backend/apps/github/models/common.py +++ b/backend/apps/github/models/common.py @@ -66,3 +66,8 @@ class Meta: abstract = True node_id = models.CharField(verbose_name="Node ID", unique=True) + + @staticmethod + def get_node_id(node): + """Extract node_id.""" + return node.raw_data["node_id"] diff --git a/backend/apps/github/models/issue.py b/backend/apps/github/models/issue.py new file mode 100644 index 000000000..7bc1af31b --- /dev/null +++ b/backend/apps/github/models/issue.py @@ -0,0 +1,123 @@ +"""Github app issue model.""" + +from django.db import models + +from apps.common.models import BulkSaveModel, TimestampedModel +from apps.github.models.common import NodeModel + + +class Issue(BulkSaveModel, NodeModel, TimestampedModel): + """Issue model.""" + + class Meta: + db_table = "github_issues" + verbose_name_plural = "Issues" + + class State(models.TextChoices): + OPEN = "open", "Open" + CLOSED = "closed", "Closed" + + title = models.CharField(verbose_name="Title", max_length=200) + body = models.TextField(verbose_name="Body", default="") + state = models.CharField( + verbose_name="State", max_length=20, choices=State, default=State.OPEN + ) + state_reason = models.CharField(verbose_name="State reason", max_length=200, default="") + url = models.URLField(verbose_name="URL", max_length=200, default="") + number = models.PositiveBigIntegerField(verbose_name="Number", default=0) + sequence_id = models.PositiveBigIntegerField(verbose_name="Issue ID", default=0) + + is_locked = models.BooleanField(verbose_name="Is locked", default=False) + lock_reason = models.CharField(verbose_name="Lock reason", max_length=200, default="") + + comments_count = models.PositiveIntegerField(verbose_name="Comments", default=0) + + closed_at = models.DateTimeField(verbose_name="Closed at", blank=True, null=True) + created_at = models.DateTimeField(verbose_name="Created at") + updated_at = models.DateTimeField(verbose_name="Updated at") + + # FKs. + author = models.ForeignKey( + "github.User", + verbose_name="Author", + on_delete=models.SET_NULL, + blank=True, + null=True, + related_name="created_issues", + ) + repository = models.ForeignKey( + "github.Repository", + on_delete=models.SET_NULL, + blank=True, + null=True, + related_name="issues", + ) + + # M2Ms. + assignees = models.ManyToManyField( + "github.User", + verbose_name="Assignees", + related_name="issue", + blank=True, + ) + labels = models.ManyToManyField( + "github.Label", + verbose_name="Labels", + related_name="issue", + blank=True, + ) + + def __str__(self): + """Issue human readable representation.""" + return f"{self.title} by {self.author}" + + def from_github(self, gh_issue, author=None, assignee=None, repository=None): + """Update instance based on GitHub issue data.""" + field_mapping = { + "body": "body", + "comments_count": "comments", + "closed_at": "closed_at", + "created_at": "created_at", + "is_locked": "locked", + "lock_reason": "active_lock_reason", + "number": "number", + "sequence_id": "id", + "state": "state", + "state_reason": "state_reason", + "title": "title", + "updated_at": "updated_at", + "url": "html_url", + } + + # Direct fields. + for model_field, gh_field in field_mapping.items(): + value = getattr(gh_issue, gh_field) + if value is not None: + setattr(self, model_field, value) + + # Author. + self.author = author + + # Repository. + self.repository = repository + + @staticmethod + def bulk_save(issues): + """Bulk save issues.""" + BulkSaveModel.bulk_save(Issue, issues) + issues.clear() + + @staticmethod + def update_data(gh_issue, author=None, repository=None, save=True): + """Update issue data.""" + issue_node_id = Issue.get_node_id(gh_issue) + try: + issue = Issue.objects.get(node_id=issue_node_id) + except Issue.DoesNotExist: + issue = Issue(node_id=issue_node_id) + + issue.from_github(gh_issue, author=author, repository=repository) + if save: + issue.save() + + return issue diff --git a/backend/apps/github/models/label.py b/backend/apps/github/models/label.py new file mode 100644 index 000000000..124fca8fe --- /dev/null +++ b/backend/apps/github/models/label.py @@ -0,0 +1,62 @@ +"""Github app label model.""" + +from django.db import models + +from apps.common.models import BulkSaveModel, TimestampedModel +from apps.github.models.common import NodeModel + + +class Label(BulkSaveModel, NodeModel, TimestampedModel): + """Label model.""" + + class Meta: + db_table = "github_labels" + verbose_name_plural = "Labels" + + name = models.CharField(verbose_name="Name", max_length=100) + description = models.CharField(verbose_name="Description", max_length=200) + color = models.CharField(verbose_name="Color", max_length=6, default="") + sequence_id = models.PositiveBigIntegerField(verbose_name="Label ID", default=0) + is_default = models.BooleanField(verbose_name="Is default", default=False) + + def __str__(self): + """Label human readable representation.""" + return f"{self.name} ({self.description})" if self.description else self.name + + def from_github(self, gh_label): + """Update instance based on GitHub label data.""" + # TODO(arkid15r): uncomment after PyGithub supports all fields. + field_mapping = { + "color": "color", + "description": "description", + # "is_default": "default", + "name": "name", + # "sequence_id": "id", + } + + # Direct fields. + for model_field, gh_field in field_mapping.items(): + value = getattr(gh_label, gh_field) + if value is not None: + setattr(self, model_field, value) + + @staticmethod + def bulk_save(labels): + """Bulk save labels.""" + BulkSaveModel.bulk_save(Label, labels) + labels.clear() + + @staticmethod + def update_data(gh_label, save=True): + """Update label data.""" + label_node_id = Label.get_node_id(gh_label) + try: + label = Label.objects.get(node_id=label_node_id) + except Label.DoesNotExist: + label = Label(node_id=label_node_id) + + label.from_github(gh_label) + if save: + label.save() + + return label diff --git a/backend/apps/github/models/organization.py b/backend/apps/github/models/organization.py index 06d5f365a..b5df59d5d 100644 --- a/backend/apps/github/models/organization.py +++ b/backend/apps/github/models/organization.py @@ -2,11 +2,11 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from apps.github.models.common import GenericUserModel, NodeModel -class Organization(NodeModel, GenericUserModel, TimestampedModel): +class Organization(BulkSaveModel, NodeModel, GenericUserModel, TimestampedModel): """Organization model.""" class Meta: @@ -32,3 +32,24 @@ def from_github(self, gh_organization): value = getattr(gh_organization, gh_field) if value is not None: setattr(self, model_field, value) + + @staticmethod + def bulk_save(organizations): + """Bulk save organizations.""" + BulkSaveModel.bulk_save(Organization, organizations) + organizations.clear() + + @staticmethod + def update_data(gh_organization, save=True): + """Update organization data.""" + organization_node_id = Organization.get_node_id(gh_organization) + try: + organization = Organization.objects.get(node_id=organization_node_id) + except Organization.DoesNotExist: + organization = Organization(node_id=organization_node_id) + + organization.from_github(gh_organization) + if save: + organization.save() + + return organization diff --git a/backend/apps/github/models/release.py b/backend/apps/github/models/release.py index 4960595f6..7342cdf75 100644 --- a/backend/apps/github/models/release.py +++ b/backend/apps/github/models/release.py @@ -2,11 +2,11 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from apps.github.models.common import NodeModel -class Release(NodeModel, TimestampedModel): +class Release(BulkSaveModel, NodeModel, TimestampedModel): """Release model.""" class Meta: @@ -20,7 +20,7 @@ class Meta: is_draft = models.BooleanField(verbose_name="Is draft", default=False) is_pre_release = models.BooleanField(verbose_name="Is pre-release", default=False) - sequence_id = models.PositiveBigIntegerField(verbose_name="Release internal ID", default=0) + sequence_id = models.PositiveBigIntegerField(verbose_name="Release ID", default=0) created_at = models.DateTimeField(verbose_name="Created at") published_at = models.DateTimeField(verbose_name="Published at") @@ -62,3 +62,24 @@ def from_github(self, gh_release, author=None, repository=None): # Repository. self.repository = repository + + @staticmethod + def bulk_save(releases): + """Bulk save releases.""" + BulkSaveModel.bulk_save(Release, releases) + releases.clear() + + @staticmethod + def update_data(gh_release, author=None, repository=None, save=True): + """Update release data.""" + release_node_id = Release.get_node_id(gh_release) + try: + release = Release.objects.get(node_id=release_node_id) + except Release.DoesNotExist: + release = Release(node_id=release_node_id) + + release.from_github(gh_release, author=author, repository=repository) + if save: + release.save() + + return release diff --git a/backend/apps/github/models/repository.py b/backend/apps/github/models/repository.py index 3dd40ef6b..ea21f9e21 100644 --- a/backend/apps/github/models/repository.py +++ b/backend/apps/github/models/repository.py @@ -194,3 +194,33 @@ def from_github( # FKs. self.organization = organization self.owner = user + + @staticmethod + def update_data( + gh_repository, + commits=None, + contributors=None, + languages=None, + organization=None, + user=None, + save=True, + ): + """Update repository data.""" + repository_node_id = Repository.get_node_id(gh_repository) + try: + repository = Repository.objects.get(node_id=repository_node_id) + except Repository.DoesNotExist: + repository = Repository(node_id=repository_node_id) + + repository.from_github( + gh_repository, + commits=commits, + contributors=contributors, + languages=languages, + organization=organization, + user=user, + ) + if save: + repository.save() + + return repository diff --git a/backend/apps/github/models/user.py b/backend/apps/github/models/user.py index 43dbaa572..98bd2a157 100644 --- a/backend/apps/github/models/user.py +++ b/backend/apps/github/models/user.py @@ -36,3 +36,18 @@ def from_github(self, gh_user): value = getattr(gh_user, gh_field) if value is not None: setattr(self, model_field, value) + + @staticmethod + def update_data(gh_user, save=True): + """Update GitHub user data.""" + user_node_id = User.get_node_id(gh_user) + try: + user = User.objects.get(node_id=user_node_id) + except User.DoesNotExist: + user = User(node_id=user_node_id) + + user.from_github(gh_user) + if save: + user.save() + + return user diff --git a/backend/apps/github/utils.py b/backend/apps/github/utils.py index 16d311237..ec54a6fdd 100644 --- a/backend/apps/github/utils.py +++ b/backend/apps/github/utils.py @@ -27,11 +27,6 @@ def check_funding_policy_compliance(platform, target): return False -def get_node_id(node): - """Extract node_id.""" - return node.raw_data["node_id"] - - def get_repository_path(url): """Parse repository URL to owner and repository name.""" match = GITHUB_REPOSITORY_RE.search(url.split("#")[0]) diff --git a/backend/apps/owasp/models/chapter.py b/backend/apps/owasp/models/chapter.py index d4ab72070..e07b79a44 100644 --- a/backend/apps/owasp/models/chapter.py +++ b/backend/apps/owasp/models/chapter.py @@ -2,11 +2,11 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from apps.owasp.models.common import OwaspEntity -class Chapter(OwaspEntity, TimestampedModel): +class Chapter(BulkSaveModel, OwaspEntity, TimestampedModel): """Chapter model.""" class Meta: @@ -49,3 +49,24 @@ def from_github(self, gh_repository, repository): # FKs. self.owasp_repository = repository + + @staticmethod + def bulk_save(chapters): + """Bulk save chapters.""" + BulkSaveModel.bulk_save(Chapter, chapters) + chapters.clear() + + @staticmethod + def update_data(gh_repository, repository, save=True): + """Update chapter data.""" + key = gh_repository.name.lower() + try: + chapter = Chapter.objects.get(key=key) + except Chapter.DoesNotExist: + chapter = Chapter(key=key) + + chapter.from_github(gh_repository, repository) + if save: + chapter.save() + + return chapter diff --git a/backend/apps/owasp/models/committee.py b/backend/apps/owasp/models/committee.py index 39bdfa7ac..ac6112e7b 100644 --- a/backend/apps/owasp/models/committee.py +++ b/backend/apps/owasp/models/committee.py @@ -2,11 +2,11 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from apps.owasp.models.common import OwaspEntity -class Committee(OwaspEntity, TimestampedModel): +class Committee(BulkSaveModel, OwaspEntity, TimestampedModel): """Committee model.""" class Meta: @@ -38,3 +38,24 @@ def from_github(self, gh_repository, repository): # FKs. self.owasp_repository = repository + + @staticmethod + def bulk_save(committees): + """Bulk save committees.""" + BulkSaveModel.bulk_save(Committee, committees) + committees.clear() + + @staticmethod + def update_data(gh_repository, repository, save=True): + """Update committee data.""" + key = gh_repository.name.lower() + try: + committee = Committee.objects.get(key=key) + except Committee.DoesNotExist: + committee = Committee(key=key) + + committee.from_github(gh_repository, repository) + if save: + committee.save() + + return committee diff --git a/backend/apps/owasp/models/event.py b/backend/apps/owasp/models/event.py index 50984f089..c22859c7c 100644 --- a/backend/apps/owasp/models/event.py +++ b/backend/apps/owasp/models/event.py @@ -2,11 +2,11 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from apps.owasp.models.common import OwaspEntity -class Event(OwaspEntity, TimestampedModel): +class Event(BulkSaveModel, OwaspEntity, TimestampedModel): """Event model.""" class Meta: @@ -44,3 +44,24 @@ def from_github(self, gh_repository, repository): # FKs. self.owasp_repository = repository + + @staticmethod + def bulk_save(events): + """Bulk save events.""" + BulkSaveModel.bulk_save(Event, events) + events.clear() + + @staticmethod + def update_data(gh_repository, repository, save=True): + """Update event data.""" + key = gh_repository.name.lower() + try: + event = Event.objects.get(key=key) + except Event.DoesNotExist: + event = Event(key=key) + + event.from_github(gh_repository, repository) + if save: + event.save() + + return event diff --git a/backend/apps/owasp/models/project.py b/backend/apps/owasp/models/project.py index 57d440303..765eb550a 100644 --- a/backend/apps/owasp/models/project.py +++ b/backend/apps/owasp/models/project.py @@ -2,11 +2,11 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from apps.owasp.models.common import OwaspEntity -class Project(OwaspEntity, TimestampedModel): +class Project(BulkSaveModel, OwaspEntity, TimestampedModel): """Project model.""" class Meta: @@ -138,3 +138,24 @@ def from_github(self, gh_repository, repository): # FKs. self.owasp_repository = repository + + @staticmethod + def bulk_save(projects): + """Bulk save projects.""" + BulkSaveModel.bulk_save(Project, projects) + projects.clear() + + @staticmethod + def update_data(gh_repository, repository, save=True): + """Update project data.""" + key = gh_repository.name.lower() + try: + project = Project.objects.get(key=key) + except Project.DoesNotExist: + project = Project(key=key) + + project.from_github(gh_repository, repository) + if save: + project.save() + + return project diff --git a/backend/pyproject.toml b/backend/pyproject.toml index ca84a5f31..099b1bff1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -7,10 +7,23 @@ readme = "README.md" packages = [{ include = "apps" }] +[tool.coverage.run] +branch = true +omit = [ + "__init__.py", + "**/admin.py", + "**/apps.py", + "**/migrations/*", + "manage.py", + "settings/*", + "tests/*", + "wsgi.py", +] + [tool.poetry.dependencies] django = "^5.1" django-configurations = "^2.5.1" -django-storages = {extras = ["s3"], version = "^1.14.4"} +django-storages = { extras = ["s3"], version = "^1.14.4" } gunicorn = "^23.0.0" lxml = "^5.3.0" psycopg2 = "^2.9.9" @@ -39,7 +52,14 @@ multi_line_output = 3 profile = "black" [tool.pytest.ini_options] -addopts = ["--numprocesses=auto"] +addopts = [ + "--cov-config=pyproject.toml", + "--cov-report=term-missing", + "--cov=.", + "--dist=loadscope", + "--no-cov-on-fail", + "--numprocesses=auto", +] log_level = "INFO" python_files = ["*_tests.py"] @@ -55,6 +75,7 @@ ignore = [ "COM812", "DJ012", "ERA001", + "FBT002", "FIX002", "PD", "PLC0414", @@ -76,6 +97,7 @@ select = ["ALL"] "**/models/*.py" = ["D106"] "tests/**" = ["D100", "D103", "S101"] + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/backend/settings/base.py b/backend/settings/base.py index e81937bee..ccf04fea6 100644 --- a/backend/settings/base.py +++ b/backend/settings/base.py @@ -1,5 +1,6 @@ """OWASP Nest base configuration.""" +import os from pathlib import Path from configurations import Configuration, values @@ -12,6 +13,8 @@ class Base(Configuration): ALLOWED_HOSTS = values.ListValue() DEBUG = False + ENVIRONMENT = os.environ.get("DJANGO_CONFIGURATION") + DJANGO_APPS = ( "django.contrib.admin", "django.contrib.auth",