CI workflow

.github/workflows/build-docker-images.yml

@@ -0,0 +1,48 @@
+name: Build Docker Images
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build-api:
+    name: Build API Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./apps/api/Dockerfile
+          push: false
+          load: true
+          tags: trieve/firecrawl:pr-${{ github.event.number }}
+          build-args: |
+            PORT=8080
+
+  build-puppeteer:
+    name: Build Puppeteer Service
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./apps/puppeteer-service-ts/Dockerfile
+          push: false
+          load: true
+          tags: trieve/puppeteer-service-ts:pr-${{ github.event.number }}
+          build-args: |
+            PORT=3000

apps/api/Dockerfile

@@ -1,39 +1,48 @@
 FROM node:20-slim AS base
-ENV PNPM_HOME="/pnpm"
-ENV PATH="$PNPM_HOME:$PATH"
-LABEL fly_launch_runtime="Node.js"
-RUN corepack enable
-COPY . /app
+
+# Create app directory
 WORKDIR /app
 
-FROM base AS prod-deps
-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
+# Install pnpm
+RUN npm install -g pnpm
+
+# Copy package files 
+COPY apps/api/package.json apps/api/pnpm-lock.yaml ./
 
-FROM base AS build
-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+# Install dependencies
+RUN pnpm install --frozen-lockfile
 
-RUN apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates
-RUN pnpm install
-RUN bash -c 'pnpm run build'
+# Install necessary build dependencies
+RUN apt-get update -qq && \
+    apt-get install -y \
+    ca-certificates \
+    git \
+    golang-go \
+    && update-ca-certificates
 
-# Install Go
-FROM golang:1.19 AS go-base
-COPY src/lib/go-html-to-md /app/src/lib/go-html-to-md
+# Copy the rest of the application
+COPY apps/api ./
 
-# Install Go dependencies and build parser lib
-RUN cd /app/src/lib/go-html-to-md && \
+# Build Go module
+COPY apps/api/src/lib/go-html-to-md ./src/lib/go-html-to-md
+RUN cd src/lib/go-html-to-md && \
     go mod tidy && \
     go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \
     chmod +x html-to-markdown.so
 
-FROM base
-RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y chromium chromium-sandbox && \
-    rm -rf /var/lib/apt/lists /var/cache/apt/archives
-COPY --from=prod-deps /app/node_modules /app/node_modules
-COPY --from=build /app /app
-COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/dist/src/lib/go-html-to-md/html-to-markdown.so
-
-# Start the server by default, this can be overwritten at runtime
-EXPOSE 8080
-ENV PUPPETEER_EXECUTABLE_PATH="/usr/bin/chromium"
+# Build the application
+RUN pnpm run build
+
+# Install runtime dependencies
+RUN apt-get install --no-install-recommends -y \
+    chromium \
+    chromium-sandbox \
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives
+
+# Environment setup
+ENV PUPPETEER_EXECUTABLE_PATH="/usr/bin/chromium"
+ARG PORT=8080
+ENV PORT=${PORT}
+EXPOSE ${PORT}
+
+CMD ["pnpm", "start"]

apps/puppeteer-service-ts/Dockerfile

@@ -19,3 +19,52 @@ ENV PORT=${PORT}
 EXPOSE ${PORT}
 
 CMD [ "pnpm", "start" ]
+FROM node:18-slim
+
+# Install system dependencies for Playwright
+RUN apt-get update && \
+    apt-get install -y wget gnupg && \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update && \
+    apt-get install -y \
+    google-chrome-stable \
+    fonts-ipafont-gothic \
+    fonts-wqy-zenhei \
+    fonts-thai-tlwg \
+    fonts-kacst \
+    fonts-symbola \
+    fonts-noto \
+    fonts-freefont-ttf \
+    --no-install-recommends && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install pnpm globally
+RUN npm install -g pnpm
+
+# Copy package files
+COPY apps/puppeteer-service-ts/package.json apps/puppeteer-service-ts/pnpm-lock.yaml ./
+
+# Install dependencies
+RUN pnpm install --frozen-lockfile
+
+# Copy the rest of the application
+COPY apps/puppeteer-service-ts .
+
+# Install Playwright with dependencies
+RUN npx playwright install --with-deps chromium
+
+# Build the application
+RUN pnpm run build
+
+# Set up environment
+ARG PORT=3000
+ENV PORT=${PORT}
+ENV PLAYWRIGHT_BROWSERS_PATH=/app/ms-playwright
+ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+
+EXPOSE ${PORT}
+
+CMD ["pnpm", "start"]