Merge remote-tracking branch 'upstream/main'

.github/ansible/production/deploy.yaml

@@ -86,3 +86,9 @@
     - name: Retart services
       shell:
         cmd: docker compose up -d
+
+    - name: Index data
+      async: 1800 # 30 minutes
+      poll: 0
+      shell: |
+        make index-data > /var/log/nest/index-data.log 2>&1

.github/ansible/staging/deploy.yaml

@@ -104,3 +104,9 @@
     - name: Retart services
       shell:
         cmd: docker compose up -d
+
+    - name: Index data
+      async: 1800 # 30 minutes
+      poll: 0
+      shell: |
+        make index-data > /var/log/nest/index-data.log 2>&1

backend/Makefile

@@ -35,6 +35,7 @@ github-update-project-related-repositories:
 index-data:
 	@echo "Indexing Nest data"
 	@CMD="poetry run python manage.py algolia_reindex" $(MAKE) exec-backend-command
+	@CMD="poetry run python manage.py algolia_update_replicas" $(MAKE) exec-backend-command
 	@CMD="poetry run python manage.py algolia_update_synonyms" $(MAKE) exec-backend-command
 	@CMD="poetry run python manage.py algolia_update_suggestions" $(MAKE) exec-backend-command
 

backend/apps/common/index.py

@@ -20,7 +20,7 @@ class IndexBase:
     """Nest index synonyms mixin and record count."""
 
     @staticmethod
-    def _get_client():
+    def get_client():
         """Get the Algolia client."""
         return SearchClientSync(
             settings.ALGOLIA_APPLICATION_ID,
@@ -73,7 +73,7 @@ def reindex_synonyms(app_name, index_name):
         if not (synonyms := IndexBase._parse_synonyms_file(file_path)):
             return None
 
-        client = IndexBase._get_client()
+        client = IndexBase.get_client()
         index_name = f"{settings.ENVIRONMENT.lower()}_{index_name}"
 
         try:
@@ -91,7 +91,7 @@ def reindex_synonyms(app_name, index_name):
     @lru_cache(maxsize=1024)
     def get_total_count(index_name):
         """Get total count of records in index."""
-        client = IndexBase._get_client()
+        client = IndexBase.get_client()
         try:
             return client.search_single_index(
                 index_name=f"{settings.ENVIRONMENT.lower()}_{index_name}",

backend/apps/common/management/commands/algolia_update_replicas.py

@@ -0,0 +1,14 @@
+"""A command to update OWASP Nest index replicas."""
+
+from django.core.management.base import BaseCommand
+
+from apps.owasp.index.project import ProjectIndex
+
+
+class Command(BaseCommand):
+    help = "Update OWASP Nest index replicas."
+
+    def handle(self, *_args, **_options):
+        print("\n Starting replica configuration...")
+        ProjectIndex.configure_replicas()
+        print("\n Replica have been Successfully created.")

backend/apps/owasp/index/chapter.py

@@ -16,6 +16,7 @@ class ChapterIndex(AlgoliaIndex):
     fields = (
         "idx_country",
         "idx_created_at",
+        "idx_is_active",
         "idx_key",
         "idx_leaders",
         "idx_name",

backend/apps/owasp/index/project.py

@@ -2,6 +2,7 @@
 
 from algoliasearch_django import AlgoliaIndex
 from algoliasearch_django.decorators import register
+from django.conf import settings
 
 from apps.common.index import IS_LOCAL_BUILD, LOCAL_INDEX_LIMIT, IndexBase
 from apps.owasp.models.project import Project
@@ -21,6 +22,7 @@ class ProjectIndex(AlgoliaIndex, IndexBase):
         "idx_forks_count",
         "idx_issues",
         "idx_issues_count",
+        "idx_is_active",
         "idx_key",
         "idx_languages",
         "idx_leaders",
@@ -92,3 +94,24 @@ def get_queryset(self):
     def update_synonyms():
         """Update synonyms."""
         return ProjectIndex.reindex_synonyms("owasp", "projects")
+
+    @staticmethod
+    def configure_replicas():
+        """Configure the settings for project replicas."""
+        env = settings.ENVIRONMENT.lower()
+        client = IndexBase.get_client()
+        replicas = {
+            f"{env}_projects_name_asc": ["asc(idx_name)"],
+            f"{env}_projects_name_desc": ["desc(idx_name)"],
+            f"{env}_projects_stars_count_asc": ["asc(idx_stars_count)"],
+            f"{env}_projects_stars_count_desc": ["desc(idx_stars_count)"],
+            f"{env}_projects_contributors_count_asc": ["asc(idx_contributors_count)"],
+            f"{env}_projects_contributors_count_desc": ["desc(idx_contributors_count)"],
+            f"{env}_projects_forks_count_asc": ["asc(idx_forks_count)"],
+            f"{env}_projects_forks_count_desc": ["desc(idx_forks_count)"],
+        }
+
+        client.set_settings(f"{env}_projects", {"replicas": list(replicas.keys())})
+
+        for replica_name, ranking in replicas.items():
+            client.set_settings(replica_name, {"ranking": ranking})

backend/apps/owasp/models/chapter.py

@@ -72,11 +72,9 @@ def active_chapters_count():
     def is_indexable(self):
         """Chapters to index."""
         return (
-            self.is_active
-            and self.latitude is not None
+            self.latitude is not None
             and self.longitude is not None
             and not self.owasp_repository.is_empty
-            and not self.owasp_repository.is_archived
         )
 
     def from_github(self, repository):
@@ -114,9 +112,6 @@ def generate_geo_location(self):
 
     def generate_suggested_location(self, open_ai=None, max_tokens=100):
         """Generate project summary."""
-        if not self.is_active:
-            return
-
         if not (prompt := Prompt.get_owasp_chapter_suggested_location()):
             return
 

backend/apps/owasp/models/common.py

@@ -52,7 +52,7 @@ class Meta:
     @property
     def is_indexable(self):
         """Entities to index."""
-        return self.is_active and self.has_active_repositories
+        return self.has_active_repositories
 
     @property
     def github_url(self):
@@ -105,7 +105,7 @@ def from_github(self, field_mapping, repository):
 
     def generate_summary(self, prompt, open_ai=None, max_tokens=500):
         """Generate entity summary."""
-        if not self.is_active or not prompt:
+        if not prompt:
             return
 
         open_ai = open_ai or OpenAi()

backend/apps/owasp/models/managers/chapter.py

@@ -12,10 +12,8 @@ def get_queryset(self):
         return (
             super()
             .get_queryset()
-            .filter(is_active=True)
             .select_related("owasp_repository")
             .filter(
-                owasp_repository__is_archived=False,
                 owasp_repository__is_empty=False,
             )
         )

backend/apps/owasp/models/mixins/chapter.py

@@ -21,6 +21,11 @@ def idx_geo_location(self):
         """Return geographic location for indexing."""
         return self.latitude, self.longitude
 
+    @property
+    def idx_is_active(self):
+        """Return active status for indexing."""
+        return self.is_active
+
     @property
     def idx_key(self):
         """Return key for indexing."""

backend/apps/owasp/models/mixins/project.py

@@ -60,6 +60,11 @@ def idx_issues_count(self):
         """Return issues count for indexing."""
         return self.open_issues.count()
 
+    @property
+    def idx_is_active(self):
+        """Return active status for indexing."""
+        return self.is_active
+
     @property
     def idx_key(self):
         """Return key for indexing."""
@@ -160,4 +165,4 @@ def idx_type(self):
     @property
     def idx_updated_at(self):
         """Return updated at for indexing."""
-        return self.updated_at.timestamp()
+        return self.updated_at.timestamp() if self.updated_at else ""

backend/apps/owasp/models/project.py

@@ -142,7 +142,7 @@ def is_tool_type(self):
     @property
     def is_indexable(self):
         """Projects to index."""
-        return self.is_active and self.has_active_repositories
+        return self.has_active_repositories
 
     @property
     def nest_key(self):

backend/apps/slack/apps.py

@@ -9,7 +9,6 @@ class SlackConfig(AppConfig):
 
     app = (
         App(
-            request_verification_enabled=False,
             signing_secret=settings.SLACK_SIGNING_SECRET,
             token=settings.SLACK_BOT_TOKEN,
         )