Add staging deployment

Author: arkid15r

.dockerignore

@@ -1,2 +1,3 @@
 .env
 /data
+__pycache__

.github/deploy/inventory.yaml

@@ -0,0 +1,6 @@
+nest:
+  hosts:
+    nest_staging:
+      ansible_host: '{{ lookup("env", "STAGING_HOST_IP_ADDRESS") }}'
+      ansible_ssh_private_key_file: '{{ lookup("env", "STAGING_SSH_PRIVATE_KEY_PATH") }}'
+      ansible_user: staging

.github/deploy/staging.yaml

@@ -0,0 +1,26 @@
+- name: Deploy Nest to Staging
+  hosts: nest_staging
+  tasks:
+    - name: Copy Nginx configuration file
+      ansible.builtin.copy:
+        src: '{{ playbook_dir }}/../../nginx/nginx.conf'
+        dest: ~/nginx/nginx.conf
+        mode: '0644'
+
+    - name: Copy docker-compose.yml
+      ansible.builtin.copy:
+        src: '{{ playbook_dir }}/../../docker-compose-staging.yaml'
+        dest: ~/docker-compose.yaml
+        mode: '0644'
+
+    - name: Stop Services
+      shell:
+        cmd: 'docker-compose rm --stop --force'
+
+    - name: Update Images
+      shell:
+        cmd: 'docker-compose pull'
+
+    - name: Start Services
+      shell:
+        cmd: 'docker-compose up -d'

.github/workflows/ci-cd.yaml

@@ -41,7 +41,60 @@ jobs:
         with:
           languages: python
 
-      - name: Perform CodeQL Analysis
+      - name: Perform CodeQL analysis
         uses: github/codeql-action/analyze@v3
         with:
           category: '/language:python'
+
+  build-docker-images:
+    environment: staging
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    needs:
+      - pre-commit
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build backend image
+        uses: docker/build-push-action@v6
+        with:
+          context: backend
+          file: backend/Dockerfile.staging
+          platforms: linux/amd64
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/owasp-nest-backend:latest
+
+  deploy:
+    environment: staging
+    name: Deploy Nest Staging
+    env:
+      STAGING_SSH_PRIVATE_KEY_PATH: ~/.ssh/nest_staging_private_key
+    runs-on: ubuntu-latest
+    needs:
+      - build-docker-images
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Prepare SSH key
+        run: |
+          mkdir -m 700 ~/.ssh
+          echo "${{ secrets.STAGING_SSH_PRIVATE_KEY }}" > "${{ env.STAGING_SSH_PRIVATE_KEY_PATH}}"
+          chmod 400 "${{ env.STAGING_SSH_PRIVATE_KEY_PATH}}"
+
+      - name: Run Nest deploy
+        working-directory: .github/deploy
+        run: ansible-playbook -i inventory.yaml staging.yaml

.gitignore

@@ -8,4 +8,4 @@ __pycache__
 .venv
 .vscode
 *.code-workspace
-/data
+/volumes

Makefile

@@ -3,6 +3,9 @@ build:
 	@CMD="poetry install" $(MAKE) run-backend-command
 	@CMD="poetry run python manage.py migrate" $(MAKE) run-backend-command
 
+collect-static:
+	@CMD="poetry run python manage.py collectstatic --noinput" $(MAKE) run-backend-command
+
 migrate:
 	@CMD="poetry run python manage.py migrate" $(MAKE) run-backend-command
 

backend/Dockerfile.staging

@@ -0,0 +1,24 @@
+FROM python:3.12
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN groupadd owasp && \
+    useradd --create-home --home-dir /home/owasp -g owasp owasp && \
+    python -m pip install --no-cache-dir poetry && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /home/owasp
+
+COPY apps apps
+COPY manage.py .
+COPY poetry.lock .
+COPY pyproject.toml .
+COPY settings settings
+COPY templates templates
+COPY wsgi.py wsgi.py
+
+EXPOSE 8000
+
+USER owasp

backend/apps/github/management/commands/github_sync_owasp_organization.py

@@ -10,7 +10,7 @@
 from apps.owasp.constants import OWASP_ORGANIZATION_NAME
 from apps.owasp.models import Chapter, Committee, Event, Project
 
-BATCH_SIZE = 10
+BATCH_SIZE = 100
 
 
 class Command(BaseCommand):
@@ -75,11 +75,10 @@ def save_data():
                 (p for p in projects if p.id),
                 fields=[field.name for field in Project._meta.fields if not field.primary_key],  # noqa: SLF001
             )
-            print("Saved to DB.")
 
         gh = github.Github(os.getenv("GITHUB_TOKEN"), per_page=GITHUB_ITEMS_PER_PAGE)
         gh_owasp_organization = gh.get_organization(OWASP_ORGANIZATION_NAME)
-        remote_repositories_count = gh_owasp_organization.public_repos
+        remote_owasp_repositories_count = gh_owasp_organization.public_repos
 
         owasp_organization = None
         owasp_user = None
@@ -147,13 +146,17 @@ def save_data():
         # Save remaining data.
         save_data()
 
-        # Check repo counts.
-        local_repositories_count = Repository.objects.count()
-        result = "==" if remote_repositories_count == local_repositories_count else "!="
+        # Check repository counts.
+        local_owasp_repositories_count = Repository.objects.filter(
+            is_owasp_repository=True
+        ).count()
+        result = (
+            "==" if remote_owasp_repositories_count == local_owasp_repositories_count else "!="
+        )
         print(
             "\n"
             f"OWASP GitHub repositories count {result} synced repositories count: "
-            f"{remote_repositories_count} {result} {local_repositories_count}"
+            f"{remote_owasp_repositories_count} {result} {local_owasp_repositories_count}"
         )
 
         gh.close()

backend/apps/github/management/commands/github_sync_related_repositories.py

@@ -14,7 +14,7 @@
 
 logger = logging.getLogger(__name__)
 
-BATCH_SIZE = 10
+BATCH_SIZE = 100
 
 
 class Command(BaseCommand):