Implement Slack bot app

Makefile

@@ -8,7 +8,7 @@ django-shell:
 	@CMD="poetry run python manage.py shell" $(MAKE) exec-backend-command
 
 dump-data:
-	@CMD="poetry run python manage.py dumpdata github owasp --indent=2 --output=data/nest.json" $(MAKE) exec-backend-command
+	@CMD="poetry run python manage.py dumpdata github owasp --indent=2" $(MAKE) exec-backend-command > data/nest.json
 
 exec-backend-command:
 	@docker exec -it nest-backend $(CMD) 2>/dev/null
@@ -19,8 +19,8 @@ github-sync-owasp-organization:
 github-sync-related-repositories:
 	@CMD="poetry run python manage.py github_sync_related_repositories" $(MAKE) exec-backend-command
 
-github-summarize-issues:
-	@CMD="poetry run python manage.py github_summarize_issues" $(MAKE) exec-backend-command
+github-enrich-issues:
+	@CMD="poetry run python manage.py github_enrich_issues" $(MAKE) exec-backend-command
 
 index:
 	@CMD="poetry run python manage.py algolia_reindex" $(MAKE) exec-backend-command
@@ -63,11 +63,11 @@ sync: \
 	github-sync-owasp-organization \
 	owasp-scrape-site-data \
 	github-sync-related-repositories \
-	github-summarize-issues \
+	github-enrich-issues \
 	owasp-aggregate-projects-data
 
 test:
 	@docker build -f backend/Dockerfile.test backend -t nest-backend-test 2>/dev/null
-	@docker run -e DJANGO_CONFIGURATION=Test nest-backend-test poetry run pytest 2>/dev/null
+	@docker run -e DJANGO_CONFIGURATION=Test nest-backend-test poetry run pytest
 
 update: sync index

backend/.env/template

@@ -11,4 +11,6 @@ DJANGO_DB_PORT="None"
 DJANGO_DB_USER="None"
 DJANGO_OPEN_AI_SECRET_KEY="None"
 DJANGO_SECRET_KEY="None"
+DJANGO_SLACK_APP_TOKEN="None"
+DJANGO_SLACK_BOT_TOKEN="None"
 GITHUB_TOKEN="None"

backend/apps/common/open_ai.py

@@ -11,7 +11,7 @@
 class OpenAi:
     """Open AI communication class."""
 
-    def __init__(self, model="gpt-4o-mini", max_tokens=1000, temperature=0.7, prompt=None):
+    def __init__(self, model="gpt-4o-mini", max_tokens=1000, temperature=0.7):
         """OpenAi constructor."""
         self.client = openai.OpenAI(
             api_key=settings.OPEN_AI_SECRET_KEY,
@@ -22,18 +22,21 @@ def __init__(self, model="gpt-4o-mini", max_tokens=1000, temperature=0.7, prompt
         self.model = model
         self.temperature = temperature
 
-        if prompt:
-            self.set_prompt(prompt)
-
-    def set_prompt(self, content):
+    def set_input(self, content):
         """Set system role content."""
-        self.prompt = content
+        self.input = content
 
         return self
 
-    def set_input(self, content):
+    def set_max_tokens(self, max_tokens):
+        """Set max tokens."""
+        self.max_tokens = max_tokens
+
+        return self
+
+    def set_prompt(self, content):
         """Set system role content."""
-        self.input = content
+        self.prompt = content
 
         return self
 

backend/apps/common/utils.py

@@ -1,5 +1,13 @@
 """Common app utils."""
 
+from django.conf import settings
+from django.urls import reverse
+
+
+def get_absolute_url(view_name):
+    """Return absolute URL for a view."""
+    return f"{settings.SITE_URL}{reverse(view_name)}"
+
 
 def join_values(fields, delimiter=" "):
     """Join non-empty field values using the delimiter."""

backend/apps/github/admin.py

@@ -3,7 +3,12 @@
 from django.contrib import admin
 from django.utils.safestring import mark_safe
 
-from apps.github.models import Issue, Label, Organization, Release, Repository, User
+from apps.github.models.issue import Issue
+from apps.github.models.label import Label
+from apps.github.models.organization import Organization
+from apps.github.models.release import Release
+from apps.github.models.repository import Repository
+from apps.github.models.user import User
 
 
 class LabelAdmin(admin.ModelAdmin):
@@ -26,7 +31,7 @@ class IssueAdmin(admin.ModelAdmin):
         "state",
         "is_locked",
     )
-    search_fields = ("title", "body", "summary")
+    search_fields = ("title",)
 
     def custom_field_github_url(self, obj):
         """Issue GitHub URL."""

backend/apps/github/api/__init__.py

@@ -1,8 +1 @@
 """GitHub app API."""
-
-from apps.github.api.issue import IssueViewSet
-from apps.github.api.label import LabelViewSet
-from apps.github.api.organization import OrganizationViewSet
-from apps.github.api.release import ReleaseViewSet
-from apps.github.api.repository import RepositoryViewSet
-from apps.github.api.user import UserViewSet

backend/apps/github/api/issue.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers, viewsets
 
-from apps.github.models import Issue
+from apps.github.models.issue import Issue
 
 
 # Serializers define the API representation.

backend/apps/github/api/label.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers, viewsets
 
-from apps.github.models import Label
+from apps.github.models.label import Label
 
 
 # Serializers define the API representation.

backend/apps/github/api/organization.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers, viewsets
 
-from apps.github.models import Organization
+from apps.github.models.organization import Organization
 
 
 # Serializers define the API representation.

backend/apps/github/api/release.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers, viewsets
 
-from apps.github.models import Release
+from apps.github.models.release import Release
 
 
 # Serializers define the API representation.

backend/apps/github/api/repository.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers, viewsets
 
-from apps.github.models import Repository
+from apps.github.models.repository import Repository
 
 
 # Serializers define the API representation.

backend/apps/github/api/urls.py

@@ -2,14 +2,12 @@
 
 from rest_framework import routers
 
-from apps.github.api import (
-    IssueViewSet,
-    LabelViewSet,
-    OrganizationViewSet,
-    ReleaseViewSet,
-    RepositoryViewSet,
-    UserViewSet,
-)
+from apps.github.api.issue import IssueViewSet
+from apps.github.api.label import LabelViewSet
+from apps.github.api.organization import OrganizationViewSet
+from apps.github.api.release import ReleaseViewSet
+from apps.github.api.repository import RepositoryViewSet
+from apps.github.api.user import UserViewSet
 
 router = routers.SimpleRouter()
 

backend/apps/github/api/user.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers, viewsets
 
-from apps.github.models import User
+from apps.github.models.user import User
 
 
 # Serializers define the API representation.

backend/apps/github/common.py

@@ -0,0 +1,108 @@
+"""GitHub app common module."""
+
+import logging
+
+from github.GithubException import UnknownObjectException
+
+from apps.github.models.issue import Issue
+from apps.github.models.label import Label
+from apps.github.models.organization import Organization
+from apps.github.models.release import Release
+from apps.github.models.repository import Repository
+from apps.github.models.user import User
+from apps.github.utils import check_owasp_site_repository
+
+logger = logging.getLogger(__name__)
+
+
+def sync_repository(gh_repository, organization=None, user=None):
+    """Sync GitHub repository data."""
+    entity_key = gh_repository.name.lower()
+    is_owasp_site_repository = check_owasp_site_repository(entity_key)
+
+    # GitHub repository organization.
+    if organization is None:
+        gh_organization = gh_repository.organization
+        if gh_organization is not None:
+            organization = Organization.update_data(gh_organization)
+
+    # GitHub repository owner.
+    if user is None:
+        user = User.update_data(gh_repository.owner)
+
+    # GitHub repository.
+    commits = gh_repository.get_commits()
+    contributors = gh_repository.get_contributors()
+    languages = None if is_owasp_site_repository else gh_repository.get_languages()
+
+    repository = Repository.update_data(
+        gh_repository,
+        commits=commits,
+        contributors=contributors,
+        languages=languages,
+        organization=organization,
+        user=user,
+    )
+
+    # GitHub repository issues.
+    if not repository.is_archived:
+        # Sync open issues for the first run.
+        kwargs = {
+            "direction": "asc",
+            "sort": "created",
+            "state": "open",
+        }
+        latest_issue = Issue.objects.filter(repository=repository).order_by("-updated_at").first()
+        if latest_issue:
+            # Sync open/closed issues for subsequent runs.
+            kwargs.update(
+                {
+                    "since": latest_issue.updated_at,
+                    "state": "all",
+                }
+            )
+        for gh_issue in gh_repository.get_issues(**kwargs):
+            # Skip pull requests.
+            if gh_issue.pull_request:
+                continue
+
+            # GitHub issue author.
+            if gh_issue.user is not None:
+                author = User.update_data(gh_issue.user)
+
+            issue = Issue.update_data(gh_issue, author=author, repository=repository)
+
+            # Assignees.
+            issue.assignees.clear()
+            for gh_issue_assignee in gh_issue.assignees:
+                issue.assignees.add(User.update_data(gh_issue_assignee))
+
+            # Labels.
+            issue.labels.clear()
+            for gh_issue_label in gh_issue.labels:
+                try:
+                    issue.labels.add(Label.update_data(gh_issue_label))
+                except UnknownObjectException:
+                    logger.info("Couldn't get GitHub issue label %s", issue.url)
+
+    # GitHub repository releases.
+    releases = []
+    if not is_owasp_site_repository:
+        existing_release_node_ids = set(
+            Release.objects.filter(repository=repository).values_list("node_id", flat=True)
+            if repository.id
+            else ()
+        )
+        for gh_release in gh_repository.get_releases():
+            release_node_id = Release.get_node_id(gh_release)
+            if release_node_id in existing_release_node_ids:
+                break
+
+            # GitHub release author.
+            if gh_release.author is not None:
+                author = User.update_data(gh_release.author)
+
+            # GitHub release.
+            releases.append(Release.update_data(gh_release, author=author, repository=repository))
+
+    return organization, repository, releases

backend/apps/github/index/issue.py

@@ -40,6 +40,7 @@ class IssueIndex(AlgoliaIndex):
     )
 
     settings = {
+        "attributeForDistinct": "idx_project_name",
         "minProximity": 4,
         "indexLanguages": ["en"],
         "customRanking": [

backend/apps/github/management/commands/github_summarize_issues.py → backend/apps/github/management/commands/github_enrich_issues.py

@@ -5,7 +5,7 @@
 from django.core.management.base import BaseCommand
 
 from apps.common.open_ai import OpenAi
-from apps.github.models import Issue
+from apps.github.models.issue import Issue
 
 logger = logging.getLogger(__name__)
 
@@ -27,22 +27,33 @@ def handle(self, *args, **options):
             prefix = f"{idx + offset + 1} of {open_issues_count - offset}"
             print(f"{prefix:<10} {issue.title}")
 
-            open_ai.set_prompt(
+            open_ai.set_input(f"{issue.title}\r\n{issue.body}")
+
+            # Generate summary
+            open_ai.set_max_tokens(500).set_prompt(
                 (
                     "Summarize the following GitHub issue using imperative mood."
-                    "Add a good amount of technical details."
-                    "Include possible first steps of tackling the problem."
+                    "Use a good amount technical details."
+                    "Avoid using lists for description."
+                    "Do not use mardown in output."
                 )
                 if issue.project.is_code_type or issue.project.is_tool_type
                 else (
                     "Summarize the following GitHub issue."
                     "Avoid mentioning author's name or issue creation date."
-                    "Add a hint of what needs to be done if possible."
+                    "Avoid using lists for description."
+                    "Do not use markdown in output."
                 )
             )
+            issue.summary = open_ai.complete() or ""
+
+            # Generate hint
+            open_ai.set_max_tokens(1000).set_prompt(
+                "Describe possible first steps of approaching the problem."
+            )
+            issue.hint = open_ai.complete() or ""
 
-            issue.summary = open_ai.set_input(f"{issue.title}\r\n{issue.body}").complete() or ""
             issues.append(issue)
 
         # Bulk save data.
-        Issue.bulk_save(issues, fields=("summary",))
+        Issue.bulk_save(issues, fields=("hint", "summary"))

backend/apps/github/management/commands/github_sync_owasp_organization.py

@@ -7,10 +7,15 @@
 from django.core.management.base import BaseCommand
 from github.GithubException import BadCredentialsException
 
+from apps.github.common import sync_repository
 from apps.github.constants import GITHUB_ITEMS_PER_PAGE
-from apps.github.models import Release, Repository, sync_repository
+from apps.github.models.release import Release
+from apps.github.models.repository import Repository
 from apps.owasp.constants import OWASP_ORGANIZATION_NAME
-from apps.owasp.models import Chapter, Committee, Event, Project
+from apps.owasp.models.chapter import Chapter
+from apps.owasp.models.committee import Committee
+from apps.owasp.models.event import Event
+from apps.owasp.models.project import Project
 
 logger = logging.getLogger(__name__)