Add GitHub issues. Refactor sync process.

.gitignore

@@ -1,6 +1,7 @@
 __pycache__
 .bash_history
 .cache
+.coverage
 .env
 .local
 .python_history

backend/apps/common/models.py

@@ -3,6 +3,22 @@
 from django.db import models
 
 
+class BulkSaveModel(models.Model):
+    """Base model for bulk save action."""
+
+    class Meta:
+        abstract = True
+
+    @staticmethod
+    def bulk_save(model, objects):
+        """Bulk save objects."""
+        model.objects.bulk_create(o for o in objects if not o.id)
+        model.objects.bulk_update(
+            [o for o in objects if o.id],
+            fields=[field.name for field in model._meta.fields if not field.primary_key],  # noqa: SLF001
+        )
+
+
 class TimestampedModel(models.Model):
     """Base model with auto created_at and updated_at fields."""
 

backend/apps/github/admin.py

@@ -3,7 +3,36 @@
 from django.contrib import admin
 from django.utils.safestring import mark_safe
 
-from apps.github.models import Organization, Release, Repository, User
+from apps.github.models import Issue, Label, Organization, Release, Repository, User
+
+
+class LabelAdmin(admin.ModelAdmin):
+    search_fields = ("name", "description")
+
+
+class IssueAdmin(admin.ModelAdmin):
+    autocomplete_fields = (
+        "repository",
+        "author",
+        "assignees",
+        "labels",
+    )
+    list_display = (
+        "repository",
+        "title",
+        "custom_field_github_url",
+    )
+    list_filter = (
+        "state",
+        "is_locked",
+    )
+    search_fields = ("title", "description")
+
+    def custom_field_github_url(self, obj):
+        """Issue GitHub URL."""
+        return mark_safe(f"<a href='{obj.url}' target='_blank'>↗️</a>")  # noqa: S308
+
+    custom_field_github_url.short_description = "GitHub 🔗"
 
 
 class RepositoryAdmin(admin.ModelAdmin):
@@ -69,6 +98,8 @@ class UserAdmin(admin.ModelAdmin):
     search_fields = ("name",)
 
 
+admin.site.register(Issue, IssueAdmin)
+admin.site.register(Label, LabelAdmin)
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Release, ReleaseAdmin)
 admin.site.register(Repository, RepositoryAdmin)

backend/apps/github/management/commands/github_sync_owasp_organization.py

@@ -6,11 +6,11 @@
 from django.core.management.base import BaseCommand
 
 from apps.github.constants import GITHUB_ITEMS_PER_PAGE
-from apps.github.models import Organization, Release, Repository, fetch_repository_data
+from apps.github.models import Issue, Organization, Release, Repository, sync_repository
 from apps.owasp.constants import OWASP_ORGANIZATION_NAME
 from apps.owasp.models import Chapter, Committee, Event, Project
 
-BATCH_SIZE = 100
+BATCH_SIZE = 10
 
 
 class Command(BaseCommand):
@@ -19,62 +19,14 @@ class Command(BaseCommand):
     def handle(self, *_args, **_options):
         def save_data():
             """Save data to DB."""
-            # Organizations.
-            Organization.objects.bulk_create(o for o in organizations if not o.id)
-            Organization.objects.bulk_update(
-                (o for o in organizations if o.id),
-                fields=[
-                    field.name
-                    for field in Organization._meta.fields  # noqa: SLF001
-                    if not field.primary_key
-                ],
-            )
-            organizations.clear()
-
-            # Repositories.
-            Repository.objects.bulk_create(r for r in repositories if not r.id)
-            Repository.objects.bulk_update(
-                [r for r in repositories if r.id],
-                fields=[field.name for field in Repository._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-            repositories.clear()
+            Organization.bulk_save(organizations)
+            Issue.bulk_save(issues)
+            Release.bulk_save(releases)
 
-            # Releases.
-            Release.objects.bulk_create(r for r in releases if not r.id)
-            Release.objects.bulk_update(
-                (r for r in releases if r.id),
-                fields=[field.name for field in Release._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-            releases.clear()
-
-            # Chapters.
-            Chapter.objects.bulk_create(c for c in chapters if not c.id)
-            Chapter.objects.bulk_update(
-                (c for c in chapters if c.id),
-                fields=[field.name for field in Chapter._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-            chapters.clear()
-
-            # Committees.
-            Committee.objects.bulk_create(c for c in committees if not c.id)
-            Committee.objects.bulk_update(
-                (c for c in committees if c.id),
-                fields=[field.name for field in Committee._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-
-            # Events.
-            Event.objects.bulk_create(e for e in events if not e.id)
-            Event.objects.bulk_update(
-                [e for e in events if not e.id],
-                fields=[field.name for field in Event._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-
-            # Projects.
-            Project.objects.bulk_create(p for p in projects if not p.id)
-            Project.objects.bulk_update(
-                (p for p in projects if p.id),
-                fields=[field.name for field in Project._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
+            Chapter.bulk_save(chapters)
+            Committee.bulk_save(committees)
+            Event.bulk_save(events)
+            Project.bulk_save(projects)
 
         gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE)
         gh_owasp_organization = gh.get_organization(OWASP_ORGANIZATION_NAME)
@@ -86,59 +38,39 @@ def save_data():
         chapters = []
         committees = []
         events = []
+        issues = []
         organizations = []
         projects = []
         releases = []
-        repositories = []
         for idx, gh_repository in enumerate(
-            gh_owasp_organization.get_repos(type="public", sort="created", direction="desc")[0:]
+            gh_owasp_organization.get_repos(type="public", sort="created", direction="asc")
         ):
             print(f"{idx + 1:<3} {gh_repository.name}")
 
-            owasp_organization, new_repository, new_releases = fetch_repository_data(
+            owasp_organization, repository, new_releases = sync_repository(
                 gh_repository, organization=owasp_organization, user=owasp_user
             )
             if not owasp_organization.id:
                 owasp_organization.save()
-            repositories.append(new_repository)
+
             releases.extend(new_releases)
 
             entity_key = gh_repository.name.lower()
             # OWASP chapters.
             if entity_key.startswith("www-chapter-"):
-                try:
-                    chapter = Chapter.objects.get(key=entity_key)
-                except Chapter.DoesNotExist:
-                    chapter = Chapter(key=entity_key)
-                chapter.from_github(gh_repository, new_repository)
-                chapters.append(chapter)
+                chapters.append(Chapter.update_data(gh_repository, repository, save=False))
 
             # OWASP projects.
             elif entity_key.startswith("www-project-"):
-                try:
-                    project = Project.objects.get(key=entity_key)
-                except Project.DoesNotExist:
-                    project = Project(key=entity_key)
-                project.from_github(gh_repository, new_repository)
-                projects.append(project)
+                projects.append(Project.update_data(gh_repository, repository, save=False))
 
             # OWASP events.
             elif entity_key.startswith("www-event-"):
-                try:
-                    event = Event.objects.get(key=entity_key)
-                except Event.DoesNotExist:
-                    event = Event(key=entity_key)
-                event.from_github(gh_repository, new_repository)
-                events.append(event)
+                events.append(Event.update_data(gh_repository, repository, save=False))
 
             # OWASP committees.
             elif entity_key.startswith("www-committee-"):
-                try:
-                    committee = Committee.objects.get(key=entity_key)
-                except Committee.DoesNotExist:
-                    committee = Committee(key=entity_key)
-                committee.from_github(gh_repository, new_repository)
-                committees.append(committee)
+                committees.append(Committee.update_data(gh_repository, repository, save=False))
 
             if idx % BATCH_SIZE == 0:
                 save_data()

backend/apps/github/management/commands/github_sync_related_repositories.py

@@ -8,13 +8,13 @@
 from github.GithubException import UnknownObjectException
 
 from apps.github.constants import GITHUB_ITEMS_PER_PAGE
-from apps.github.models import Release, fetch_repository_data
+from apps.github.models import Issue, Release, sync_repository
 from apps.github.utils import get_repository_path
 from apps.owasp.models import Project
 
 logger = logging.getLogger(__name__)
 
-BATCH_SIZE = 100
+BATCH_SIZE = 10
 
 
 class Command(BaseCommand):
@@ -23,27 +23,14 @@ class Command(BaseCommand):
     def handle(self, *args, **_options):
         def save_data():
             """Save data to DB."""
-            # Releases.
-            Release.objects.bulk_create(r for r in releases if not r.id)
-            Release.objects.bulk_update(
-                (r for r in releases if r.id),
-                fields=[field.name for field in Release._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-            releases.clear()
-
-            # Projects.
-            Project.objects.bulk_update(
-                (p for p in projects),
-                fields=[field.name for field in Project._meta.fields if not field.primary_key],  # noqa: SLF001
-            )
-            projects.clear()
-
-        active_projects = Project.objects.filter(is_active=True).order_by(
-            "owasp_repository__created_at"
-        )
+            Issue.bulk_save(issues)
+            Release.bulk_save(releases)
+            Project.bulk_save(projects)
 
+        active_projects = Project.objects.filter(is_active=True).order_by("created_at")
         gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE)
 
+        issues = []
         projects = []
         releases = []
         for idx, project in enumerate(active_projects):
@@ -64,10 +51,10 @@ def save_data():
                         project.save(update_fields=("repositories_raw",))
                         continue
 
-                organization, repository, new_releases = fetch_repository_data(gh_repository)
+                organization, repository, new_releases = sync_repository(gh_repository)
                 if organization is not None:
                     organization.save()
-                repository.save()
+
                 project.repositories.add(repository)
                 releases.extend(new_releases)