Update bot projects command, optimize sync

Makefile

@@ -5,14 +5,17 @@ collect-static:
 	@CMD="poetry run python manage.py collectstatic --noinput" $(MAKE) exec-backend-command
 
 django-shell:
-	@CMD="poetry run python manage.py shell" $(MAKE) exec-backend-command
+	@CMD="poetry run python manage.py shell" $(MAKE) exec-backend-command-it
 
 dump-data:
 	@CMD="poetry run python manage.py dumpdata github owasp --indent=2" $(MAKE) exec-backend-command > data/nest.json
 
 enrich-data: github-enrich-issues owasp-enrich-projects
 
 exec-backend-command:
+	@docker exec -i nest-backend $(CMD) 2>/dev/null
+
+exec-backend-command-it:
 	@docker exec -it nest-backend $(CMD) 2>/dev/null
 
 github-enrich-issues:
@@ -72,7 +75,7 @@ setup:
 	@CMD="poetry run python manage.py createsuperuser" $(MAKE) exec-backend-command
 
 shell:
-	@CMD="/bin/bash" $(MAKE) exec-backend-command
+	@CMD="/bin/bash" $(MAKE) exec-backend-command-it
 
 sync: update-data enrich-data index-data
 

backend/apps/common/utils.py

@@ -1,7 +1,11 @@
 """Common app utils."""
 
+from datetime import datetime, timezone
+
 from django.conf import settings
+from django.template.defaultfilters import pluralize
 from django.urls import reverse
+from humanize import intword, naturaltime
 
 
 def get_absolute_url(view_name):
@@ -12,3 +16,14 @@ def get_absolute_url(view_name):
 def join_values(fields, delimiter=" "):
     """Join non-empty field values using the delimiter."""
     delimiter.join(field for field in fields if field)
+
+
+def natural_date(value):
+    """Return humanized version of a date."""
+    return naturaltime(datetime.fromtimestamp(value, tz=timezone.utc))
+
+
+def natural_number(value, unit=None):
+    """Return humanized version of a number."""
+    number = intword(value)
+    return f"{number} {unit}{pluralize(value)}" if unit else number

backend/apps/owasp/models/chapter.py

@@ -33,7 +33,7 @@ def __str__(self):
         """Chapter human readable representation."""
         return f"{self.name or self.key}"
 
-    def from_github(self, gh_repository, repository):
+    def from_github(self, repository):
         """Update instance based on GitHub repository data."""
         field_mapping = {
             "country": "country",
@@ -45,7 +45,7 @@ def from_github(self, gh_repository, repository):
             "region": "region",
             "tags": "tags",
         }
-        OwaspEntity.from_github(self, field_mapping, gh_repository, repository)
+        OwaspEntity.from_github(self, field_mapping, repository)
 
         # FKs.
         self.owasp_repository = repository
@@ -64,7 +64,7 @@ def update_data(gh_repository, repository, save=True):
         except Chapter.DoesNotExist:
             chapter = Chapter(key=key)
 
-        chapter.from_github(gh_repository, repository)
+        chapter.from_github(repository)
         if save:
             chapter.save()
 

backend/apps/owasp/models/committee.py

@@ -27,14 +27,14 @@ def __str__(self):
         """Committee human readable representation."""
         return f"{self.name}"
 
-    def from_github(self, gh_repository, repository):
+    def from_github(self, repository):
         """Update instance based on GitHub repository data."""
         field_mapping = {
             "description": "pitch",
             "name": "title",
             "tags": "tags",
         }
-        OwaspEntity.from_github(self, field_mapping, gh_repository, repository)
+        OwaspEntity.from_github(self, field_mapping, repository)
 
         # FKs.
         self.owasp_repository = repository
@@ -53,7 +53,7 @@ def update_data(gh_repository, repository, save=True):
         except Committee.DoesNotExist:
             committee = Committee(key=key)
 
-        committee.from_github(gh_repository, repository)
+        committee.from_github(repository)
         if save:
             committee.save()
 

backend/apps/owasp/models/common.py

@@ -4,7 +4,6 @@
 import re
 
 import yaml
-from github.GithubException import GithubException, UnknownObjectException
 
 from apps.github.constants import GITHUB_REPOSITORY_RE, GITHUB_USER_RE
 from apps.github.utils import get_repository_file_content
@@ -28,38 +27,34 @@ def owasp_url(self):
         """Get OWASP URL."""
         return f"https://owasp.org/{self.key}"
 
-    @property
-    def index_md_raw_url(self):
-        """Return project's raw index.md GitHub URL."""
-        return (
-            "https://raw.githubusercontent.com/OWASP/"
-            f"{self.owasp_repository.key}/{self.owasp_repository.default_branch}/index.md"
-        )
-
-    def from_github(self, field_mapping, gh_repository, repository):
+    def from_github(self, field_mapping, repository):
         """Update instance based on GitHub repository data."""
         # Fetch project metadata from index.md file.
         project_metadata = {}
-        try:
-            index_md_content = get_repository_file_content(self.index_md_raw_url)
-            yaml_content = re.search(r"^---\n(.*?)\n---", index_md_content, re.DOTALL)
-            project_metadata = yaml.safe_load(yaml_content.group(1)) or {} if yaml_content else {}
-
-            # Direct fields.
-            for model_field, gh_field in field_mapping.items():
-                value = project_metadata.get(gh_field)
-                if value:
-                    setattr(self, model_field, value)
-        except yaml.scanner.ScannerError:
-            logger.exception("Unable to parse metadata", extra={"repository": gh_repository.name})
-        except GithubException as e:
-            if e.data["status"] == "404" and "This repository is empty" in e.data["message"]:
-                repository.is_empty = True
-        except UnknownObjectException:
-            pass
+        index_md_content = get_repository_file_content(
+            self.get_index_md_raw_url(repository=repository)
+        )
+        yaml_content = re.search(r"^---\n(.*?)\n---", index_md_content, re.DOTALL)
+        project_metadata = yaml.safe_load(yaml_content.group(1)) or {} if yaml_content else {}
+
+        # Direct fields.
+        for model_field, gh_field in field_mapping.items():
+            value = project_metadata.get(gh_field)
+            if value:
+                setattr(self, model_field, value)
 
         return project_metadata
 
+    def get_index_md_raw_url(self, repository=None):
+        """Return project's raw index.md GitHub URL."""
+        owasp_repository = repository or self.owasp_repository
+        return (
+            "https://raw.githubusercontent.com/OWASP/"
+            f"{owasp_repository.key}/{owasp_repository.default_branch}/index.md"
+            if owasp_repository
+            else None
+        )
+
     def get_related_url(self, url):
         """Get OWASP entity related URL."""
         if url in {self.github_url, self.owasp_url}:

backend/apps/owasp/models/event.py

@@ -32,15 +32,15 @@ def __str__(self):
         """Event human readable representation."""
         return f"{self.name or self.key}"
 
-    def from_github(self, gh_repository, repository):
+    def from_github(self, repository):
         """Update instance based on GitHub repository data."""
         field_mapping = {
             "description": "pitch",
             "level": "level",
             "name": "title",
             "tags": "tags",
         }
-        OwaspEntity.from_github(self, field_mapping, gh_repository, repository)
+        OwaspEntity.from_github(self, field_mapping, repository)
 
         # FKs.
         self.owasp_repository = repository
@@ -59,7 +59,7 @@ def update_data(gh_repository, repository, save=True):
         except Event.DoesNotExist:
             event = Event(key=key)
 
-        event.from_github(gh_repository, repository)
+        event.from_github(repository)
         if save:
             event.save()
 

backend/apps/owasp/models/project.py

@@ -159,14 +159,14 @@ def deactivate(self):
         self.is_active = False
         self.save(update_fields=("is_active",))
 
-    def from_github(self, gh_repository, repository):
+    def from_github(self, repository):
         """Update instance based on GitHub repository data."""
         field_mapping = {
             "description": "pitch",
             "name": "title",
             "tags": "tags",
         }
-        project_metadata = OwaspEntity.from_github(self, field_mapping, gh_repository, repository)
+        project_metadata = OwaspEntity.from_github(self, field_mapping, repository)
 
         # Normalize tags.
         self.tags = (
@@ -218,7 +218,7 @@ def update_data(gh_repository, repository, save=True):
         except Project.DoesNotExist:
             project = Project(key=key)
 
-        project.from_github(gh_repository, repository)
+        project.from_github(repository)
         if save:
             project.save()
 

backend/apps/slack/commands/projects.py

@@ -1,9 +1,10 @@
 """Slack bot contribute command."""
 
 from django.conf import settings
+from django.template.defaultfilters import pluralize
 from django.utils.text import Truncator
 
-from apps.common.utils import get_absolute_url
+from apps.common.utils import get_absolute_url, natural_date, natural_number
 from apps.slack.apps import SlackConfig
 from apps.slack.blocks import markdown
 from apps.slack.constants import FEEDBACK_CHANNEL_MESSAGE
@@ -29,7 +30,13 @@ def handler(ack, command, client):
     ]
 
     attributes = [
+        "idx_contributors_count",
+        "idx_created_at",
+        "idx_forks_count",
+        "idx_leaders",
+        "idx_level",
         "idx_name",
+        "idx_stars_count",
         "idx_summary",
         "idx_url",
     ]
@@ -53,11 +60,30 @@ def handler(ack, command, client):
             name_truncated = Truncator(escape(project["idx_name"])).chars(
                 NAME_TRUNCATION_LIMIT, truncate="..."
             )
+            contributors_count = (
+                f", {natural_number(project['idx_contributors_count'], unit='contributor')}"
+                if project["idx_contributors_count"]
+                else ""
+            )
+            forks_count = (
+                f", {natural_number(project['idx_forks_count'], unit='fork')}"
+                if project["idx_forks_count"]
+                else ""
+            )
+            stars_count = (
+                f", {natural_number(project['idx_stars_count'], unit='star')}"
+                if project["idx_stars_count"]
+                else ""
+            )
+            leaders = project["idx_leaders"]
             blocks.append(
                 markdown(
                     f"\n*{idx + 1}.* <{project['idx_url']}|*{name_truncated}*>\n"
+                    f"_Created {natural_date(project['idx_created_at'])}"
+                    f"{stars_count}{forks_count}{contributors_count}_\n"
+                    f"_Leader{pluralize(len(leaders))}: {', '.join(leaders)}_\n"
                     f"{escape(project['idx_summary'])}\n"
-                ),
+                )
             )
 
         blocks.append(

backend/pyproject.toml

@@ -37,14 +37,12 @@ python = "^3.12"
 pyyaml = "^6.0.2"
 requests = "^2.32.3"
 slack-bolt = "^1.20.1"
-djlint = "^1.35.2"
 
 [tool.poetry.group.dev.dependencies]
+djlint = "^1.35.2"
+isort = "^5.13.2"
 pre-commit = "^3.8.0"
 ruff = "^0.5.7"
-isort = "^5.13.2"
-djlint = "^1.35.2"
-
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.2"