Update environment to simplify setup process

OWASP · Sep 12, 2024 · d4f8759 · d4f8759
1 parent 56068c2
commit d4f8759
Show file tree

Hide file tree

Showing 99 changed files with 452,095 additions and 2,383 deletions.
diff --git a/.github/deploy/inventory.yaml → .github/ansible/inventory.yaml b/.github/deploy/inventory.yaml → .github/ansible/inventory.yaml
diff --git a/.github/deploy/staging.yaml → .github/ansible/staging/deploy.yaml b/.github/deploy/staging.yaml → .github/ansible/staging/deploy.yaml
@@ -3,30 +3,30 @@
   tasks:
     - name: Copy Nginx configuration file
       ansible.builtin.copy:
-        src: '{{ playbook_dir }}/../../nginx/staging.conf'
+        src: '{{ github_workspace }}/nginx/staging.conf'
         dest: ~/nginx/nginx.conf
         mode: '0644'
 
     - name: Copy docker-compose.yml
       ansible.builtin.copy:
-        src: '{{ playbook_dir }}/../../docker-compose-staging.yaml'
+        src: '{{ github_workspace }}/docker-compose-staging.yaml'
         dest: ~/docker-compose.yaml
         mode: '0644'
 
     - name: Copy Makefile
       ansible.builtin.copy:
-        src: '{{ playbook_dir }}/../../Makefile'
+        src: '{{ github_workspace }}/Makefile'
         dest: ~/Makefile
         mode: '0644'
 
     - name: Stop Services
       shell:
         cmd: 'docker compose rm --stop --force'
 
-    - name: Update Images
+    - name: Update Docker images
       shell:
         cmd: 'docker compose pull'
 
-    - name: Start Services
+    - name: Start services
       shell:
-        cmd: 'make run &'
+        cmd: 'docker compose up -d'
diff --git a/.github/workflows/ci-cd.yaml b/.github/workflows/ci-cd.yaml
@@ -30,6 +30,7 @@ jobs:
           cache: poetry
           cache-dependency-path: backend/poetry.lock
           python-version: '3.12'
+
       - name: Run pre-commit
         uses: pre-commit/[email protected]
 
@@ -73,21 +74,18 @@ jobs:
           cache-dependency-path: backend/poetry.lock
           python-version: '3.12'
 
-      - name: Install dependencies
-        run: |
-          cd backend
-          poetry install --no-root --with test
-
       - name: Run tests
         run: |
           make test
 
   build-docker-images:
-    environment: staging
     name: Build Docker Images
-    runs-on: ubuntu-latest
+    environment: staging
+    if: |
+      github.ref == 'refs/heads/main'
     needs:
       - run-tests
+    runs-on: ubuntu-latest
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -113,16 +111,18 @@ jobs:
           push: true
           tags: ${{ secrets.DOCKERHUB_USERNAME }}/owasp-nest-backend:latest
 
-  deploy:
-    environment: staging
+  deploy-staging:
     name: Deploy Nest Staging
     env:
       ANSIBLE_HOST_KEY_CHECKING: False
       STAGING_HOST_IP_ADDRESS: '${{ secrets.STAGING_HOST_IP_ADDRESS }}'
       STAGING_SSH_PRIVATE_KEY_PATH: '~/.ssh/nest_staging_private_key'
-    runs-on: ubuntu-latest
+    environment: staging
+    if: |
+      github.ref == 'refs/heads/main'
     needs:
       - build-docker-images
+    runs-on: ubuntu-latest
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -134,5 +134,5 @@ jobs:
           chmod 400 ${{ env.STAGING_SSH_PRIVATE_KEY_PATH }}
 
       - name: Run Nest deploy
-        working-directory: .github/deploy
-        run: ansible-playbook -i inventory.yaml staging.yaml
+        working-directory: .github/ansible
+        run: ansible-playbook -i inventory.yaml staging/deploy.yaml -e "github_workspace=$GITHUB_WORKSPACE"
diff --git a/Makefile b/Makefile
@@ -1,58 +1,67 @@
 build:
 	@docker compose build
-	@CMD="poetry install --no-root" $(MAKE) run-backend-command
-	@CMD="poetry run python manage.py migrate" $(MAKE) run-backend-command
 
 collect-static:
-	@CMD="poetry run python manage.py collectstatic --noinput" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py collectstatic --noinput" $(MAKE) exec-backend-command
+
+dump-data:
+	@CMD="poetry run python manage.py dumpdata github owasp --indent=2 --output=data/nest.json" $(MAKE) exec-backend-command
+
+exec-backend-command:
+	@docker exec -it nest-backend $(CMD) 2>/dev/null
 
 github-sync-owasp-organization:
-	@CMD="poetry run python manage.py github_sync_owasp_organization" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py github_sync_owasp_organization" $(MAKE) exec-backend-command
 
 github-sync-related-repositories:
-	@CMD="poetry run python manage.py github_sync_related_repositories" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py github_sync_related_repositories" $(MAKE) exec-backend-command
 
 index:
-	@CMD="poetry run python manage.py algolia_reindex" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py algolia_reindex" $(MAKE) exec-backend-command
+
+load-data:
+	@CMD="poetry run python manage.py load_data" $(MAKE) exec-backend-command
+
+merge-migrations:
+	@CMD="poetry run python manage.py makemigrations --merge" $(MAKE) exec-backend-command
 
 migrate:
-	@CMD="poetry run python manage.py migrate" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py migrate" $(MAKE) exec-backend-command
 
 migrations:
-	@CMD="poetry run python manage.py makemigrations" $(MAKE) run-backend-command
-
-migrations-merge:
-	@CMD="poetry run python manage.py makemigrations --merge" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py makemigrations" $(MAKE) exec-backend-command
 
 owasp-scrape-site-data:
-	@CMD="poetry run python manage.py owasp_scrape_site_data" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py owasp_scrape_site_data" $(MAKE) exec-backend-command
 
 owasp-update-projects:
-	@CMD="poetry run python manage.py owasp_update_projects" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py owasp_update_projects" $(MAKE) exec-backend-command
 
 pre-commit:
 	@pre-commit run -a
 
 purge-data:
-	@CMD="poetry run python manage.py purge_data" $(MAKE) run-backend-command
+	@CMD="poetry run python manage.py purge_data" $(MAKE) exec-backend-command
 
 run:
+	@$(MAKE) build
 	@docker compose up
 
-run-backend-command:
-	@docker compose run --rm backend $(CMD)
+setup:
+	@CMD="poetry run python manage.py createsuperuser" $(MAKE) exec-backend-command
 
 shell:
-	@CMD="/bin/bash" $(MAKE) run-backend-command
+	@CMD="/bin/bash" $(MAKE) exec-backend-command
 
 sync:
-	$(MAKE) github-sync-owasp-organization
-	$(MAKE) owasp-scrape-site-data
-	$(MAKE) github-sync-related-repositories
-	$(MAKE) owasp-update-projects
+	@$(MAKE) github-sync-owasp-organization
+	@$(MAKE) owasp-scrape-site-data
+	@$(MAKE) github-sync-related-repositories
+	@$(MAKE) owasp-update-projects
 
 test:
-	@cd backend && poetry run pytest; cd ..
+	@docker build -f backend/Dockerfile.test backend -t nest-backend-test 2>/dev/null
+	@docker run -e DJANGO_CONFIGURATION=Test nest-backend-test poetry run pytest 2>/dev/null
 
 update:
 	@$(MAKE) sync

diff --git a/.env/backend.test → backend/.env/template b/.env/backend.test → backend/.env/template
@@ -1,6 +1,6 @@
 DJANGO_ALGOLIA_API_KEY="None"
 DJANGO_ALGOLIA_APPLICATION_ID="None"
-DJANGO_ALLOWED_HOSTS="0.0.0.0,127.0.0.1,localhost"
+DJANGO_ALLOWED_HOSTS="localhost"
 DJANGO_AWS_ACCESS_KEY_ID="None"
 DJANGO_AWS_SECRET_ACCESS_KEY="None"
 DJANGO_CONFIGURATION="Test"

diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local
@@ -1,11 +1,12 @@
-FROM python:3.12
+FROM python:3.12-slim
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN groupadd owasp && \
     useradd --create-home --home-dir /home/owasp -g owasp owasp && \
-    python -m pip install --no-cache-dir poetry && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get update && apt-get upgrade && apt-get install gcc libpq-dev -y && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    python -m pip install --no-cache-dir poetry
 
 ENV PYTHONUNBUFFERED=1 \
     POETRY_NO_INTERACTION=1 \

diff --git a/backend/Dockerfile.staging b/backend/Dockerfile.staging
@@ -1,11 +1,12 @@
-FROM python:3.12
+FROM python:3.12-slim
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN groupadd owasp && \
     useradd --create-home --home-dir /home/owasp -g owasp owasp && \
-    python -m pip install --no-cache-dir poetry && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get update && apt-get upgrade && apt-get install gcc libpq-dev -y && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    python -m pip install --no-cache-dir poetry
 
 ENV PYTHONUNBUFFERED=1 \
     POETRY_NO_INTERACTION=1 \
@@ -14,13 +15,11 @@ ENV PYTHONUNBUFFERED=1 \
 WORKDIR /home/owasp
 
 COPY apps apps
+COPY manage.py poetry.lock pyproject.toml wsgi.py ./
 COPY settings settings
 COPY static static
 COPY templates templates
-COPY manage.py poetry.lock pyproject.toml wsgi.py ./
 
 RUN poetry install --no-root --without dev --without test
 
-EXPOSE 8000
-
 USER owasp
diff --git a/backend/Dockerfile.test b/backend/Dockerfile.test
@@ -0,0 +1,27 @@
+FROM python:3.12-slim
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN groupadd owasp && \
+    useradd --create-home --home-dir /home/owasp -g owasp owasp && \
+    apt-get update && apt-get upgrade && apt-get install gcc libpq-dev -y && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    python -m pip install --no-cache-dir poetry
+
+ENV PYTHONUNBUFFERED=1 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_CREATE=false
+
+WORKDIR /home/owasp
+
+COPY .env/template .env/template
+COPY apps apps
+COPY manage.py poetry.lock pyproject.toml wsgi.py ./
+COPY settings settings
+COPY static static
+COPY templates templates
+COPY tests tests
+
+RUN poetry install --no-root
+
+USER owasp
diff --git a/backend/apps/common/management/commands/load_data.py b/backend/apps/common/management/commands/load_data.py
@@ -0,0 +1,33 @@
+"""A command to load OWASP Nest data."""
+
+import contextlib
+
+from algoliasearch_django import register, unregister
+from algoliasearch_django.registration import RegistrationError
+from django.apps import apps
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+
+class Command(BaseCommand):
+    help = "Load OWASP Nest data."
+
+    def handle(self, *_args, **_options):
+        nest_apps = ("github", "owasp")
+
+        # Disable indexing
+        for nest_app in nest_apps:
+            for model in apps.get_app_config(nest_app).get_models():
+                with contextlib.suppress(RegistrationError):
+                    unregister(model)
+
+        # Run loaddata
+        with transaction.atomic():
+            call_command("loaddata", "data/nest.json", "-v", "3")
+
+        # Enable indexing
+        for nest_app in nest_apps:
+            for model in apps.get_app_config(nest_app).get_models():
+                with contextlib.suppress(RegistrationError):
+                    register(model)
diff --git a/backend/apps/common/management/commands/purge_data.py b/backend/apps/common/management/commands/purge_data.py
@@ -1,32 +1,18 @@
 """A command to purge OWASP Nest data."""
 
+from django.apps import apps
 from django.core.management.base import BaseCommand
 from django.db import connection
 
-from apps.github.models import Issue, Label, Organization, Release, Repository, User
-from apps.owasp.models import Chapter, Committee, Event, Project
-
-BATCH_SIZE = 10
-
 
 class Command(BaseCommand):
     help = "Purge OWASP Nest data."
 
     def handle(self, *_args, **options):
-        with connection.cursor() as cursor:
-            models = (
-                Chapter,
-                Committee,
-                Event,
-                Issue,
-                Label,
-                Organization,
-                Project,
-                Release,
-                Repository,
-                User,
-            )
+        nest_apps = ("github", "owasp")
 
-            for model in models:
-                cursor.execute(f"TRUNCATE TABLE {model._meta.db_table} CASCADE")  # noqa: SLF001
-                print(f"Purged GitHub {model._meta.verbose_name_plural}")  # noqa: SLF001
+        with connection.cursor() as cursor:
+            for nest_app in nest_apps:
+                for model in apps.get_app_config(nest_app).get_models():
+                    cursor.execute(f"TRUNCATE TABLE {model._meta.db_table} CASCADE")  # noqa: SLF001
+                    print(f"Purged GitHub {model._meta.verbose_name_plural}")  # noqa: SLF001
diff --git a/backend/apps/common/models.py b/backend/apps/common/models.py
@@ -2,6 +2,8 @@
 
 from django.db import models
 
+BATCH_SIZE = 1000
+
 
 class BulkSaveModel(models.Model):
     """Base model for bulk save action."""
@@ -12,10 +14,11 @@ class Meta:
     @staticmethod
     def bulk_save(model, objects):
         """Bulk save objects."""
-        model.objects.bulk_create(o for o in objects if not o.id)
+        model.objects.bulk_create((o for o in objects if not o.id), BATCH_SIZE)
         model.objects.bulk_update(
             (o for o in objects if o.id),
             fields=[field.name for field in model._meta.fields if not field.primary_key],  # noqa: SLF001
+            batch_size=BATCH_SIZE,
         )
         objects.clear()