Add project top contributors

Makefile

@@ -13,7 +13,7 @@ dump-data:
 enrich-data: github-enrich-issues owasp-enrich-projects
 
 exec-backend-command:
-	@docker exec -i nest-backend $(CMD) 2>/dev/null
+	@docker exec -i nest-backend $(CMD)
 
 exec-backend-command-it:
 	@docker exec -it nest-backend $(CMD) 2>/dev/null

backend/apps/github/admin.py

@@ -8,6 +8,7 @@
 from apps.github.models.organization import Organization
 from apps.github.models.release import Release
 from apps.github.models.repository import Repository
+from apps.github.models.repository_contributor import RepositoryContributor
 from apps.github.models.user import User
 
 
@@ -82,6 +83,14 @@ def custom_field_title(self, obj):
     custom_field_github_url.short_description = "GitHub 🔗"
 
 
+class RepositoryContributorAdmin(admin.ModelAdmin):
+    autocomplete_fields = (
+        "repository",
+        "user",
+    )
+    search_fields = ("user__name",)
+
+
 class OrganizationAdmin(admin.ModelAdmin):
     list_display = (
         "title",
@@ -110,4 +119,5 @@ class UserAdmin(admin.ModelAdmin):
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Release, ReleaseAdmin)
 admin.site.register(Repository, RepositoryAdmin)
+admin.site.register(RepositoryContributor, RepositoryContributorAdmin)
 admin.site.register(User, UserAdmin)

backend/apps/github/common.py

@@ -9,6 +9,7 @@
 from apps.github.models.organization import Organization
 from apps.github.models.release import Release
 from apps.github.models.repository import Repository
+from apps.github.models.repository_contributor import RepositoryContributor
 from apps.github.models.user import User
 from apps.github.utils import check_owasp_site_repository
 
@@ -105,5 +106,20 @@ def sync_repository(gh_repository, organization=None, user=None):
                 else None
             )
             releases.append(Release.update_data(gh_release, author=author, repository=repository))
+    Release.bulk_save(releases)
+
+    # GitHub repository contributors.
+    repository_contributors = []
+    for gh_contributor in gh_repository.get_contributors():
+        user = (
+            User.update_data(gh_contributor)
+            if gh_contributor and gh_contributor.type != "Bot"
+            else None
+        )
+        if user:
+            repository_contributors.append(
+                RepositoryContributor.update_data(gh_contributor, repository=repository, user=user)
+            )
+    RepositoryContributor.bulk_save(repository_contributors)
 
-    return organization, repository, releases
+    return organization, repository

backend/apps/github/constants.py

@@ -6,3 +6,6 @@
 GITHUB_ITEMS_PER_PAGE = 100
 GITHUB_REPOSITORY_RE = re.compile("^https://github.com/([^/]+)/([^/]+)(/.*)?$")
 GITHUB_USER_RE = re.compile("^https://github.com/([^/]+)/?$")
+
+OWASP_FOUNDATION_LOGIN = "OWASPFoundation"
+OWASP_LOGIN = "owasp"

backend/apps/github/management/commands/github_update_owasp_organization.py

@@ -9,7 +9,6 @@
 
 from apps.github.common import sync_repository
 from apps.github.constants import GITHUB_ITEMS_PER_PAGE
-from apps.github.models.release import Release
 from apps.github.models.repository import Repository
 from apps.owasp.constants import OWASP_ORGANIZATION_NAME
 from apps.owasp.models.chapter import Chapter
@@ -45,7 +44,6 @@ def handle(self, *_args, **options):
         committees = []
         events = []
         projects = []
-        releases = []
 
         offset = options["offset"]
         gh_repositories = gh_owasp_organization.get_repos(
@@ -59,10 +57,9 @@ def handle(self, *_args, **options):
             entity_key = gh_repository.name.lower()
             print(f"{prefix:<12} https://owasp.org/{entity_key}")
 
-            owasp_organization, repository, new_releases = sync_repository(
+            owasp_organization, repository = sync_repository(
                 gh_repository, organization=owasp_organization, user=owasp_user
             )
-            releases.extend(new_releases)
 
             # OWASP chapters.
             if entity_key.startswith("www-chapter-"):
@@ -80,9 +77,6 @@ def handle(self, *_args, **options):
             elif entity_key.startswith("www-committee-"):
                 committees.append(Committee.update_data(gh_repository, repository, save=False))
 
-        # Bulk save data.
-        Release.bulk_save(releases)
-
         Chapter.bulk_save(chapters)
         Committee.bulk_save(committees)
         Event.bulk_save(events)

backend/apps/github/management/commands/github_update_related_repositories.py

@@ -10,7 +10,6 @@
 from apps.github.common import sync_repository
 from apps.github.constants import GITHUB_ITEMS_PER_PAGE
 from apps.github.models.issue import Issue
-from apps.github.models.release import Release
 from apps.github.utils import get_repository_path
 from apps.owasp.models.project import Project
 
@@ -30,7 +29,6 @@ def handle(self, *args, **options):
 
         issues = []
         projects = []
-        releases = []
 
         offset = options["offset"]
         for idx, project in enumerate(active_projects[offset:]):
@@ -53,16 +51,14 @@ def handle(self, *args, **options):
                         project.save(update_fields=("invalid_urls", "related_urls"))
                         continue
 
-                organization, repository, new_releases = sync_repository(gh_repository)
+                organization, repository = sync_repository(gh_repository)
                 if organization is not None:
                     organization.save()
 
                 project.repositories.add(repository)
-                releases.extend(new_releases)
 
             projects.append(project)
 
         # Bulk save data.
         Issue.bulk_save(issues)
-        Release.bulk_save(releases)
         Project.bulk_save(projects)

backend/apps/github/migrations/0007_repositorycontributor.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.1.1 on 2024-09-21 19:07
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("github", "0006_alter_issue_state_reason"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RepositoryContributor",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("nest_created_at", models.DateTimeField(auto_now_add=True)),
+                ("nest_updated_at", models.DateTimeField(auto_now=True)),
+                ("node_id", models.CharField(unique=True, verbose_name="Node ID")),
+                (
+                    "contributions_count",
+                    models.PositiveIntegerField(default=0, verbose_name="Contributions"),
+                ),
+                (
+                    "repository",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="github.repository",
+                        verbose_name="Repository",
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="github.user",
+                        verbose_name="User",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name_plural": "Contributors",
+                "db_table": "github_repository_contributors",
+            },
+        ),
+    ]

backend/apps/github/migrations/0008_alter_repositorycontributor_options.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.1.1 on 2024-09-21 19:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("github", "0007_repositorycontributor"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="repositorycontributor",
+            options={"verbose_name_plural": "Repository contributors"},
+        ),
+    ]

backend/apps/github/migrations/0009_remove_repositorycontributor_node_id.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.1.1 on 2024-09-21 19:13
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("github", "0008_alter_repositorycontributor_options"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="repositorycontributor",
+            name="node_id",
+        ),
+    ]

backend/apps/github/models/common.py

@@ -30,10 +30,16 @@ class Meta:
     created_at = models.DateTimeField(verbose_name="Created at")
     updated_at = models.DateTimeField(verbose_name="Updated at")
 
+    @property
     def title(self):
-        """User title."""
+        """Entity title."""
         return f"{self.name or self.login}"
 
+    @property
+    def url(self):
+        """Entity URL."""
+        return f"https://github.com/{self.login.lower()}"
+
     def from_github(self, data):
         """Update instance based on GitHub data."""
         field_mapping = {

backend/apps/github/models/repository.py

@@ -7,6 +7,7 @@
 from github.GithubException import GithubException
 
 from apps.common.models import TimestampedModel
+from apps.github.constants import OWASP_LOGIN
 from apps.github.models.common import NodeModel
 from apps.github.models.mixins import RepositoryIndexMixin
 from apps.github.utils import (
@@ -167,7 +168,7 @@ def from_github(
         # Key and OWASP repository flags.
         self.key = self.name.lower()
         self.is_owasp_repository = (
-            organization is not None and organization.login.lower() == "owasp"
+            organization is not None and organization.login.lower() == OWASP_LOGIN
         )
         self.is_owasp_site_repository = check_owasp_site_repository(self.key)
 
@@ -207,7 +208,10 @@ def from_github(
                 for target in targets if isinstance(targets, list) else [targets]:
                     if not target:
                         continue
-                    is_funding_policy_compliant = check_funding_policy_compliance(platform, target)
+                    is_funding_policy_compliant = check_funding_policy_compliance(
+                        platform,
+                        target,
+                    )
 
                     if not is_funding_policy_compliant:
                         break

backend/apps/github/models/repository_contributor.py

@@ -0,0 +1,74 @@
+"""Github app label model."""
+
+from django.db import models
+from django.template.defaultfilters import pluralize
+
+from apps.common.models import BulkSaveModel, TimestampedModel
+
+TOP_CONTRIBUTORS_LIMIT = 20
+
+
+class RepositoryContributor(BulkSaveModel, TimestampedModel):
+    """Repository contributor model."""
+
+    class Meta:
+        db_table = "github_repository_contributors"
+        verbose_name_plural = "Repository contributors"
+
+    contributions_count = models.PositiveIntegerField(verbose_name="Contributions", default=0)
+
+    # FKs.
+    repository = models.ForeignKey(
+        "github.Repository",
+        verbose_name="Repository",
+        on_delete=models.CASCADE,
+    )
+    user = models.ForeignKey(
+        "github.User",
+        verbose_name="User",
+        on_delete=models.CASCADE,
+    )
+
+    def __str__(self):
+        """Repository contributor human readable representation."""
+        return (
+            f"{self.user} has made {self.contributions_count} "
+            f"contribution{pluralize(self.contributions_count)} to {self.repository}"
+        )
+
+    def from_github(self, gh_label):
+        """Update instance based on GitHub contributor data."""
+        field_mapping = {
+            "contributions_count": "contributions",
+        }
+
+        # Direct fields.
+        for model_field, gh_field in field_mapping.items():
+            value = getattr(gh_label, gh_field)
+            if value is not None:
+                setattr(self, model_field, value)
+
+    @staticmethod
+    def bulk_save(repository_contributors):
+        """Bulk save repository contributors."""
+        BulkSaveModel.bulk_save(RepositoryContributor, repository_contributors)
+
+    @staticmethod
+    def update_data(gh_contributor, repository, user, save=True):
+        """Update repository contributor data."""
+        try:
+            repository_contributor = RepositoryContributor.objects.get(
+                repository=repository,
+                user=user,
+            )
+        except RepositoryContributor.DoesNotExist:
+            repository_contributor = RepositoryContributor(
+                repository=repository,
+                user=user,
+            )
+        repository_contributor.from_github(gh_contributor)
+
+        if save:
+            repository_contributor.save()
+
+        return repository_contributor

backend/apps/owasp/admin.py

@@ -35,10 +35,11 @@ class EventAdmin(admin.ModelAdmin):
 
 class ProjectAdmin(admin.ModelAdmin):
     autocomplete_fields = (
-        "owasp_repository",
         "organizations",
+        "owasp_repository",
         "owners",
         "repositories",
+        "top_contributors",
     )
     list_display = (
         "custom_field_name",
@@ -57,6 +58,7 @@ class ProjectAdmin(admin.ModelAdmin):
         "level",
         "type",
     )
+    ordering = ("-created_at",)
     search_fields = (
         "description",
         "key",

backend/apps/owasp/api/search/project.py

@@ -19,6 +19,7 @@ def get_projects(query, attributes=None, limit=25):
             "idx_name",
             "idx_stars_count",
             "idx_summary",
+            "idx_top_contributors",
             "idx_topics",
             "idx_type",
             "idx_updated_at",