From 69c1ba30a7153b7a3b482e0c960dceb9694ee756 Mon Sep 17 00:00:00 2001 From: Arkadii Yakovets Date: Sat, 21 Sep 2024 19:34:38 -0700 Subject: [PATCH] Add project top contributors --- Makefile | 2 +- backend/apps/github/admin.py | 10 +++ backend/apps/github/common.py | 18 ++++- backend/apps/github/constants.py | 3 + .../github_update_owasp_organization.py | 8 +- .../github_update_related_repositories.py | 6 +- .../migrations/0007_repositorycontributor.py | 51 +++++++++++++ ...008_alter_repositorycontributor_options.py | 16 ++++ ...09_remove_repositorycontributor_node_id.py | 16 ++++ backend/apps/github/models/common.py | 8 +- backend/apps/github/models/repository.py | 8 +- .../github/models/repository_contributor.py | 74 +++++++++++++++++++ backend/apps/owasp/admin.py | 4 +- backend/apps/owasp/api/search/project.py | 1 + backend/apps/owasp/index/project.py | 4 +- .../commands/owasp_aggregate_projects.py | 14 ++++ .../0003_project_top_contributors.py | 20 +++++ backend/apps/owasp/models/mixins/project.py | 12 +++ backend/apps/owasp/models/project.py | 5 ++ .../apps/owasp/templates/search/issue.html | 4 +- .../apps/owasp/templates/search/project.html | 54 +++++++++++--- 21 files changed, 307 insertions(+), 31 deletions(-) create mode 100644 backend/apps/github/migrations/0007_repositorycontributor.py create mode 100644 backend/apps/github/migrations/0008_alter_repositorycontributor_options.py create mode 100644 backend/apps/github/migrations/0009_remove_repositorycontributor_node_id.py create mode 100644 backend/apps/github/models/repository_contributor.py create mode 100644 backend/apps/owasp/migrations/0003_project_top_contributors.py diff --git a/Makefile b/Makefile index 7406b4df7..7abebe469 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ dump-data: enrich-data: github-enrich-issues owasp-enrich-projects exec-backend-command: - @docker exec -i nest-backend $(CMD) 2>/dev/null + @docker exec -i nest-backend $(CMD) exec-backend-command-it: @docker exec -it nest-backend $(CMD) 2>/dev/null diff --git a/backend/apps/github/admin.py b/backend/apps/github/admin.py index 51839dbe8..8b3533779 100644 --- a/backend/apps/github/admin.py +++ b/backend/apps/github/admin.py @@ -8,6 +8,7 @@ from apps.github.models.organization import Organization from apps.github.models.release import Release from apps.github.models.repository import Repository +from apps.github.models.repository_contributor import RepositoryContributor from apps.github.models.user import User @@ -82,6 +83,14 @@ def custom_field_title(self, obj): custom_field_github_url.short_description = "GitHub 🔗" +class RepositoryContributorAdmin(admin.ModelAdmin): + autocomplete_fields = ( + "repository", + "user", + ) + search_fields = ("user__name",) + + class OrganizationAdmin(admin.ModelAdmin): list_display = ( "title", @@ -110,4 +119,5 @@ class UserAdmin(admin.ModelAdmin): admin.site.register(Organization, OrganizationAdmin) admin.site.register(Release, ReleaseAdmin) admin.site.register(Repository, RepositoryAdmin) +admin.site.register(RepositoryContributor, RepositoryContributorAdmin) admin.site.register(User, UserAdmin) diff --git a/backend/apps/github/common.py b/backend/apps/github/common.py index 59749f689..ef8f79f00 100644 --- a/backend/apps/github/common.py +++ b/backend/apps/github/common.py @@ -9,6 +9,7 @@ from apps.github.models.organization import Organization from apps.github.models.release import Release from apps.github.models.repository import Repository +from apps.github.models.repository_contributor import RepositoryContributor from apps.github.models.user import User from apps.github.utils import check_owasp_site_repository @@ -105,5 +106,20 @@ def sync_repository(gh_repository, organization=None, user=None): else None ) releases.append(Release.update_data(gh_release, author=author, repository=repository)) + Release.bulk_save(releases) + + # GitHub repository contributors. + repository_contributors = [] + for gh_contributor in gh_repository.get_contributors(): + user = ( + User.update_data(gh_contributor) + if gh_contributor and gh_contributor.type != "Bot" + else None + ) + if user: + repository_contributors.append( + RepositoryContributor.update_data(gh_contributor, repository=repository, user=user) + ) + RepositoryContributor.bulk_save(repository_contributors) - return organization, repository, releases + return organization, repository diff --git a/backend/apps/github/constants.py b/backend/apps/github/constants.py index 741693e44..641be8077 100644 --- a/backend/apps/github/constants.py +++ b/backend/apps/github/constants.py @@ -6,3 +6,6 @@ GITHUB_ITEMS_PER_PAGE = 100 GITHUB_REPOSITORY_RE = re.compile("^https://github.com/([^/]+)/([^/]+)(/.*)?$") GITHUB_USER_RE = re.compile("^https://github.com/([^/]+)/?$") + +OWASP_FOUNDATION_LOGIN = "OWASPFoundation" +OWASP_LOGIN = "owasp" diff --git a/backend/apps/github/management/commands/github_update_owasp_organization.py b/backend/apps/github/management/commands/github_update_owasp_organization.py index 2829d7d2a..e5bd90d99 100644 --- a/backend/apps/github/management/commands/github_update_owasp_organization.py +++ b/backend/apps/github/management/commands/github_update_owasp_organization.py @@ -9,7 +9,6 @@ from apps.github.common import sync_repository from apps.github.constants import GITHUB_ITEMS_PER_PAGE -from apps.github.models.release import Release from apps.github.models.repository import Repository from apps.owasp.constants import OWASP_ORGANIZATION_NAME from apps.owasp.models.chapter import Chapter @@ -45,7 +44,6 @@ def handle(self, *_args, **options): committees = [] events = [] projects = [] - releases = [] offset = options["offset"] gh_repositories = gh_owasp_organization.get_repos( @@ -59,10 +57,9 @@ def handle(self, *_args, **options): entity_key = gh_repository.name.lower() print(f"{prefix:<12} https://owasp.org/{entity_key}") - owasp_organization, repository, new_releases = sync_repository( + owasp_organization, repository = sync_repository( gh_repository, organization=owasp_organization, user=owasp_user ) - releases.extend(new_releases) # OWASP chapters. if entity_key.startswith("www-chapter-"): @@ -80,9 +77,6 @@ def handle(self, *_args, **options): elif entity_key.startswith("www-committee-"): committees.append(Committee.update_data(gh_repository, repository, save=False)) - # Bulk save data. - Release.bulk_save(releases) - Chapter.bulk_save(chapters) Committee.bulk_save(committees) Event.bulk_save(events) diff --git a/backend/apps/github/management/commands/github_update_related_repositories.py b/backend/apps/github/management/commands/github_update_related_repositories.py index 79b64d302..38b3286cb 100644 --- a/backend/apps/github/management/commands/github_update_related_repositories.py +++ b/backend/apps/github/management/commands/github_update_related_repositories.py @@ -10,7 +10,6 @@ from apps.github.common import sync_repository from apps.github.constants import GITHUB_ITEMS_PER_PAGE from apps.github.models.issue import Issue -from apps.github.models.release import Release from apps.github.utils import get_repository_path from apps.owasp.models.project import Project @@ -30,7 +29,6 @@ def handle(self, *args, **options): issues = [] projects = [] - releases = [] offset = options["offset"] for idx, project in enumerate(active_projects[offset:]): @@ -53,16 +51,14 @@ def handle(self, *args, **options): project.save(update_fields=("invalid_urls", "related_urls")) continue - organization, repository, new_releases = sync_repository(gh_repository) + organization, repository = sync_repository(gh_repository) if organization is not None: organization.save() project.repositories.add(repository) - releases.extend(new_releases) projects.append(project) # Bulk save data. Issue.bulk_save(issues) - Release.bulk_save(releases) Project.bulk_save(projects) diff --git a/backend/apps/github/migrations/0007_repositorycontributor.py b/backend/apps/github/migrations/0007_repositorycontributor.py new file mode 100644 index 000000000..85b17dc9a --- /dev/null +++ b/backend/apps/github/migrations/0007_repositorycontributor.py @@ -0,0 +1,51 @@ +# Generated by Django 5.1.1 on 2024-09-21 19:07 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0006_alter_issue_state_reason"), + ] + + operations = [ + migrations.CreateModel( + name="RepositoryContributor", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("node_id", models.CharField(unique=True, verbose_name="Node ID")), + ( + "contributions_count", + models.PositiveIntegerField(default=0, verbose_name="Contributions"), + ), + ( + "repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="github.repository", + verbose_name="Repository", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="github.user", + verbose_name="User", + ), + ), + ], + options={ + "verbose_name_plural": "Contributors", + "db_table": "github_repository_contributors", + }, + ), + ] diff --git a/backend/apps/github/migrations/0008_alter_repositorycontributor_options.py b/backend/apps/github/migrations/0008_alter_repositorycontributor_options.py new file mode 100644 index 000000000..2628e96cb --- /dev/null +++ b/backend/apps/github/migrations/0008_alter_repositorycontributor_options.py @@ -0,0 +1,16 @@ +# Generated by Django 5.1.1 on 2024-09-21 19:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0007_repositorycontributor"), + ] + + operations = [ + migrations.AlterModelOptions( + name="repositorycontributor", + options={"verbose_name_plural": "Repository contributors"}, + ), + ] diff --git a/backend/apps/github/migrations/0009_remove_repositorycontributor_node_id.py b/backend/apps/github/migrations/0009_remove_repositorycontributor_node_id.py new file mode 100644 index 000000000..84d43b68d --- /dev/null +++ b/backend/apps/github/migrations/0009_remove_repositorycontributor_node_id.py @@ -0,0 +1,16 @@ +# Generated by Django 5.1.1 on 2024-09-21 19:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0008_alter_repositorycontributor_options"), + ] + + operations = [ + migrations.RemoveField( + model_name="repositorycontributor", + name="node_id", + ), + ] diff --git a/backend/apps/github/models/common.py b/backend/apps/github/models/common.py index 8eac123ab..7eb11d9ab 100644 --- a/backend/apps/github/models/common.py +++ b/backend/apps/github/models/common.py @@ -30,10 +30,16 @@ class Meta: created_at = models.DateTimeField(verbose_name="Created at") updated_at = models.DateTimeField(verbose_name="Updated at") + @property def title(self): - """User title.""" + """Entity title.""" return f"{self.name or self.login}" + @property + def url(self): + """Entity URL.""" + return f"https://github.com/{self.login.lower()}" + def from_github(self, data): """Update instance based on GitHub data.""" field_mapping = { diff --git a/backend/apps/github/models/repository.py b/backend/apps/github/models/repository.py index b7b0e9f56..d432e39f7 100644 --- a/backend/apps/github/models/repository.py +++ b/backend/apps/github/models/repository.py @@ -7,6 +7,7 @@ from github.GithubException import GithubException from apps.common.models import TimestampedModel +from apps.github.constants import OWASP_LOGIN from apps.github.models.common import NodeModel from apps.github.models.mixins import RepositoryIndexMixin from apps.github.utils import ( @@ -167,7 +168,7 @@ def from_github( # Key and OWASP repository flags. self.key = self.name.lower() self.is_owasp_repository = ( - organization is not None and organization.login.lower() == "owasp" + organization is not None and organization.login.lower() == OWASP_LOGIN ) self.is_owasp_site_repository = check_owasp_site_repository(self.key) @@ -207,7 +208,10 @@ def from_github( for target in targets if isinstance(targets, list) else [targets]: if not target: continue - is_funding_policy_compliant = check_funding_policy_compliance(platform, target) + is_funding_policy_compliant = check_funding_policy_compliance( + platform, + target, + ) if not is_funding_policy_compliant: break diff --git a/backend/apps/github/models/repository_contributor.py b/backend/apps/github/models/repository_contributor.py new file mode 100644 index 000000000..3bb5bfe73 --- /dev/null +++ b/backend/apps/github/models/repository_contributor.py @@ -0,0 +1,74 @@ +"""Github app label model.""" + +from django.db import models +from django.template.defaultfilters import pluralize + +from apps.common.models import BulkSaveModel, TimestampedModel + +TOP_CONTRIBUTORS_LIMIT = 20 + + +class RepositoryContributor(BulkSaveModel, TimestampedModel): + """Repository contributor model.""" + + class Meta: + db_table = "github_repository_contributors" + verbose_name_plural = "Repository contributors" + + contributions_count = models.PositiveIntegerField(verbose_name="Contributions", default=0) + + # FKs. + repository = models.ForeignKey( + "github.Repository", + verbose_name="Repository", + on_delete=models.CASCADE, + ) + user = models.ForeignKey( + "github.User", + verbose_name="User", + on_delete=models.CASCADE, + ) + + def __str__(self): + """Repository contributor human readable representation.""" + return ( + f"{self.user} has made {self.contributions_count} " + f"contribution{pluralize(self.contributions_count)} to {self.repository}" + ) + + def from_github(self, gh_label): + """Update instance based on GitHub contributor data.""" + field_mapping = { + "contributions_count": "contributions", + } + + # Direct fields. + for model_field, gh_field in field_mapping.items(): + value = getattr(gh_label, gh_field) + if value is not None: + setattr(self, model_field, value) + + @staticmethod + def bulk_save(repository_contributors): + """Bulk save repository contributors.""" + BulkSaveModel.bulk_save(RepositoryContributor, repository_contributors) + + @staticmethod + def update_data(gh_contributor, repository, user, save=True): + """Update repository contributor data.""" + try: + repository_contributor = RepositoryContributor.objects.get( + repository=repository, + user=user, + ) + except RepositoryContributor.DoesNotExist: + repository_contributor = RepositoryContributor( + repository=repository, + user=user, + ) + repository_contributor.from_github(gh_contributor) + + if save: + repository_contributor.save() + + return repository_contributor diff --git a/backend/apps/owasp/admin.py b/backend/apps/owasp/admin.py index 7021607de..1706b3bf6 100644 --- a/backend/apps/owasp/admin.py +++ b/backend/apps/owasp/admin.py @@ -35,10 +35,11 @@ class EventAdmin(admin.ModelAdmin): class ProjectAdmin(admin.ModelAdmin): autocomplete_fields = ( - "owasp_repository", "organizations", + "owasp_repository", "owners", "repositories", + "top_contributors", ) list_display = ( "custom_field_name", @@ -57,6 +58,7 @@ class ProjectAdmin(admin.ModelAdmin): "level", "type", ) + ordering = ("-created_at",) search_fields = ( "description", "key", diff --git a/backend/apps/owasp/api/search/project.py b/backend/apps/owasp/api/search/project.py index ba7054371..03d87bec0 100644 --- a/backend/apps/owasp/api/search/project.py +++ b/backend/apps/owasp/api/search/project.py @@ -19,6 +19,7 @@ def get_projects(query, attributes=None, limit=25): "idx_name", "idx_stars_count", "idx_summary", + "idx_top_contributors", "idx_topics", "idx_type", "idx_updated_at", diff --git a/backend/apps/owasp/index/project.py b/backend/apps/owasp/index/project.py index 0be2271cb..686d8fc78 100644 --- a/backend/apps/owasp/index/project.py +++ b/backend/apps/owasp/index/project.py @@ -14,6 +14,7 @@ class ProjectIndex(AlgoliaIndex): fields = ( "idx_companies", + "idx_top_contributors", "idx_contributors_count", "idx_description", "idx_forks_count", @@ -54,7 +55,8 @@ class ProjectIndex(AlgoliaIndex): "unordered(idx_name)", "unordered(idx_languages, idx_tags, idx_topics)", "unordered(idx_description)", - "unordered(idx_companies, idx_leaders, idx_organizations)", + "unordered(idx_companies, idx_organizations)", + "unordered(idx_leaders, idx_top_contributors.login, idx_top_contributors.name)", "unordered(idx_level)", ], } diff --git a/backend/apps/owasp/management/commands/owasp_aggregate_projects.py b/backend/apps/owasp/management/commands/owasp_aggregate_projects.py index 4f3f9fb98..337361a40 100644 --- a/backend/apps/owasp/management/commands/owasp_aggregate_projects.py +++ b/backend/apps/owasp/management/commands/owasp_aggregate_projects.py @@ -2,6 +2,11 @@ from django.core.management.base import BaseCommand +from apps.github.constants import OWASP_FOUNDATION_LOGIN +from apps.github.models.repository_contributor import ( + TOP_CONTRIBUTORS_LIMIT, + RepositoryContributor, +) from apps.owasp.models.project import Project @@ -51,6 +56,15 @@ def handle(self, *_args, **options): project.organizations.add(repository.organization) project.owners.add(repository.owner) + # Top contributors. + excluded_contributor_logins = [OWASP_FOUNDATION_LOGIN] + project.top_contributors.set( + RepositoryContributor.objects.filter(repository__project=project) + .exclude(user__login__in=excluded_contributor_logins) + .order_by("-contributions_count") + .distinct()[:TOP_CONTRIBUTORS_LIMIT] + ) + # Pushed at. pushed_at.append(repository.pushed_at) diff --git a/backend/apps/owasp/migrations/0003_project_top_contributors.py b/backend/apps/owasp/migrations/0003_project_top_contributors.py new file mode 100644 index 000000000..27f42c59e --- /dev/null +++ b/backend/apps/owasp/migrations/0003_project_top_contributors.py @@ -0,0 +1,20 @@ +# Generated by Django 5.1.1 on 2024-09-21 19:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0009_remove_repositorycontributor_node_id"), + ("owasp", "0002_project_summary"), + ] + + operations = [ + migrations.AddField( + model_name="project", + name="top_contributors", + field=models.ManyToManyField( + blank=True, to="github.repositorycontributor", verbose_name="Top contributors" + ), + ), + ] diff --git a/backend/apps/owasp/models/mixins/project.py b/backend/apps/owasp/models/mixins/project.py index c213b848a..6fbb583f5 100644 --- a/backend/apps/owasp/models/mixins/project.py +++ b/backend/apps/owasp/models/mixins/project.py @@ -71,6 +71,18 @@ def idx_tags(self): """Return tags for indexing.""" return self.tags + @property + def idx_top_contributors(self): + """Return top contributors for indexing.""" + return [ + { + "avatar_url": tc.user.avatar_url, + "login": tc.user.login, + "name": tc.user.name, + } + for tc in self.top_contributors.order_by("-contributions_count") + ] + @property def idx_topics(self): """Return topics for indexing.""" diff --git a/backend/apps/owasp/models/project.py b/backend/apps/owasp/models/project.py index f7d2d6da0..c2bc24989 100644 --- a/backend/apps/owasp/models/project.py +++ b/backend/apps/owasp/models/project.py @@ -129,6 +129,11 @@ class ProjectType(models.TextChoices): verbose_name="Repositories", blank=True, ) + top_contributors = models.ManyToManyField( + "github.RepositoryContributor", + verbose_name="Top contributors", + blank=True, + ) def __str__(self): """Project human readable representation.""" diff --git a/backend/apps/owasp/templates/search/issue.html b/backend/apps/owasp/templates/search/issue.html index ff8818823..9c436e263 100644 --- a/backend/apps/owasp/templates/search/issue.html +++ b/backend/apps/owasp/templates/search/issue.html @@ -97,7 +97,7 @@
@@ -90,19 +90,44 @@

- ${project.idx_name} + ${project.idx_name}

-
- Leaders: +
+ Leaders: ${leader} ${leader} ,
+
+ Top contributors: +
+ + + +
+
@@ -111,7 +136,7 @@

} }).mount('#app'); - {% endblock content %}