Update workflow and compose file

Author: dipu-bd

.github/workflows/deploy.yml

@@ -9,73 +9,62 @@ concurrency:
 
 jobs:
   deploy:
+    name: Deploy Server
     runs-on: ubuntu-latest
 
     steps:
       - name: 🛎️ Checkout Repository
         uses: actions/checkout@v4
 
-      - name: 🔐 SSH Key & .env setup
-        run: |
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "${{ secrets.SSH_SECRET }}" > ~/.ssh/bitanon.pem
-          chmod 600 ~/.ssh/bitanon.pem
-          echo "${{ secrets.ENV_FILE }}" > .env
-
-      # - name: 🐳 Build Docker Image
-      #   run: docker build -t lncrawl .
+      - name: 🔐 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: 🐳 Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: 💾 Docker Cache
-        uses: actions/cache@v4
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+      - name: 📝 Create .env file for Docker build
+        run: touch .env
 
-      - name: 🐳 Build Docker Image with Cache
+      - name: 🏗️ Build and push Container image
         uses: docker/build-push-action@v5
         with:
           context: .
-          tags: lncrawl:latest
+          platforms: linux/amd64
+          push: true
+          tags: ghcr.io/${{ github.repository }}:latest
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.revision=${{ github.sha }}
           outputs: type=docker,dest=/tmp/lncrawl.tar
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: 🛡️ Setup SSH Key
+        run: |
+          mkdir -p ~/.ssh
+          chmod 700 ~/.ssh
+          echo "${{ secrets.SSH_SECRET }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -H "${{ secrets.DEPLOY_SERVER }}" >> ~/.ssh/known_hosts
 
       - name: 📦 Transfer Docker Image
         run: |
-          bzip2 < /tmp/lncrawl.tar | \
-          ssh -i ~/.ssh/bitanon.pem -o StrictHostKeyChecking=no \
-              "${{ secrets.DEPLOY_SERVER }}" 'bunzip2 | docker load'
+          bzip2 < /tmp/lncrawl.tar | ssh "${{ secrets.DEPLOY_SERVER }}" 'bunzip2 | docker load'
 
-      # - name: 📦 Save and Transfer Docker Image
-      #   run: |
-      #     docker save lncrawl | bzip2 | \
-      #     ssh -i ~/.ssh/bitanon.pem -o StrictHostKeyChecking=no \
-      #         "${{ secrets.DEPLOY_SERVER }}" docker load
+      - name: 📤 Upload Compose File
+        run: |
+          scp ./scripts/server-compose.yml "${{ secrets.DEPLOY_SERVER }}:~/lncrawl/compose.yml"
 
-      - name: 🏗️ Prepare server environment
+      - name: 🚀 Deploy over SSH
         run: |
-          ssh -i ~/.ssh/bitanon.pem -o StrictHostKeyChecking=no \
-              "${{ secrets.DEPLOY_SERVER }}" << 'EOF'
+          ssh "${{ secrets.DEPLOY_SERVER }}" << 'EOF'
             mkdir -p ~/lncrawl/output_path
             cd ~/lncrawl
-            touch .env sqlite.db
-            chown 1000:1000 .env sqlite.db output_path
+            touch .env
+            chown 1000:1000 .env output_path
+            docker compose -f ~/lncrawl/compose.yml up -d
           EOF
-
-      - name: 📂 Upload Compose File
-        run: |
-          scp -i ~/.ssh/bitanon.pem -o StrictHostKeyChecking=no \
-              ./scripts/server-compose.yml \
-              "${{ secrets.DEPLOY_SERVER }}:~/lncrawl/compose.yml"
-
-      - name: 🚀 Run Container via Docker Compose
-        run: |
-          ssh -i ~/.ssh/bitanon.pem -o StrictHostKeyChecking=no \
-              "${{ secrets.DEPLOY_SERVER }}" \
-              "docker compose -f ~/lncrawl/compose.yml up -d"

.github/workflows/release.yml

@@ -164,43 +164,30 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Login to GitHub Container Registry
+      - name: 🔐 Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
+      - name: 🐳 Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Extract version
-        id: version
-        run: |
-          VERSION=${GITHUB_REF_NAME#v}
-          echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
-
       - name: 📝 Create .env file for Docker build
-        run: |
-          touch .env
+        run: touch .env
 
-      - name: 🏗️ Build and push Container image
+      - name: 🏗️ Build and Push Image
         uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64
           push: true
           tags: |
             ghcr.io/${{ github.repository }}:latest
-            ghcr.io/${{ github.repository }}:${{ steps.version.outputs.version }}
+            ghcr.io/${{ github.repository }}:${GITHUB_REF##*/}
           labels: |
             org.opencontainers.image.source=${{ github.event.repository.html_url }}
             org.opencontainers.image.revision=${{ github.sha }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-
-      - name: 🧪 Test Container
-        run: |
-          # Test listing sources
-          docker run --rm ghcr.io/${{ github.repository }}:${{ steps.version.outputs.version }} --list-sources

requirements.txt

@@ -38,6 +38,7 @@ sqlmodel
 passlib[argon2]
 python-jose[cryptography]
 python-dateutil
+psycopg[binary]
 
 # dev requirements
 wheel

scripts/server-compose.yml

@@ -1,13 +1,10 @@
 services:
   chrome:
     image: selenium/standalone-chrome:latest
-    cpus: 1.5
-    mem_limit: 1600m
     shm_size: 6gb
     restart: unless-stopped
     ports:
       - "7900:7900"
-      - "4444:4444"
     environment:
       SE_VNC_VIEW_ONLY: "1"
       SE_EVENT_BUS_PUBLISH_PORT: "4442"
@@ -18,11 +15,35 @@ services:
       SE_SCREEN_WIDTH: "1920"
       SE_SCREEN_HEIGHT: "1080"
       SE_NODE_GRID_URL: "false"
+    deploy:
+      resources:
+        limits:
+          cpus: '1.50'
+          memory: 2G
+        reservations:
+          cpus: '1.00'
+          memory: 1G
+
+  postgres:
+    image: postgres:alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: pguser
+      POSTGRES_PASSWORD: pgpass
+      POSTGRES_DB: lncrawl
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    deploy:
+      resources:
+        limits:
+          cpus: '0.50'
+          memory: 512M
+        reservations:
+          cpus: '0.25'
+          memory: 256M
 
   server:
     image: lncrawl
-    cpus: 1.8
-    mem_limit: 3200m
     restart: unless-stopped
     ports:
       - "23457:8000"
@@ -35,13 +56,23 @@ services:
     environment:
       - PYTHONUNBUFFERED=1
       - OUTPUT_PATH=/home/lncrawl/output
-      - DATABASE_URL=sqlite:////app/sqlite.db
+      - DATABASE_URL=postgresql+psycopg://pguser:pgpass@postgres:5432/lncrawl
     volumes:
       - ./output_path:/home/lncrawl/output
-      - ./sqlite.db:/app/sqlite.db
       - ./.env:/app/.env
     depends_on:
       - chrome
+      - postgres
     extra_hosts:
       - "host.docker.internal:host-gateway"
- 
+    deploy:
+      resources:
+        limits:
+          cpus: '1.50'
+          memory: 3G
+        reservations:
+          cpus: '1.00'
+          memory: 1G
+
+volumes:
+  pgdata: