Test pwa

.github/workflows/build.yaml

@@ -8,17 +8,51 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v3
 
-      - name: Setup Node.js 16.x
-        uses: actions/setup-node@master
+      - uses: supabase/setup-cli@v1
         with:
-          node-version: 16.x
+          version: latest
+
+      - name: Setup Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+
+      # from https://github.com/supabase/supabase-action-example/
+      - run: supabase start -x studio,imgproxy,storage
+        working-directory: packages/server
+        env:
+          TYPECELL_GITHUB_OAUTH_SECRET: ${{ secrets.TYPECELL_GITHUB_OAUTH_SECRET }}
+          TYPECELL_GOOGLE_OAUTH_SECRET: ${{ secrets.TYPECELL_GOOGLE_OAUTH_SECRET }}
+      - run: supabase db lint
+        working-directory: packages/server
+        env:
+          TYPECELL_GITHUB_OAUTH_SECRET: ${{ secrets.TYPECELL_GITHUB_OAUTH_SECRET }}
+          TYPECELL_GOOGLE_OAUTH_SECRET: ${{ secrets.TYPECELL_GOOGLE_OAUTH_SECRET }}
+      - run: supabase test db
+        working-directory: packages/server
+        env:
+          TYPECELL_GITHUB_OAUTH_SECRET: ${{ secrets.TYPECELL_GITHUB_OAUTH_SECRET }}
+          TYPECELL_GOOGLE_OAUTH_SECRET: ${{ secrets.TYPECELL_GOOGLE_OAUTH_SECRET }}
+
+      - name: Verify generated supabase types are checked in
+        working-directory: packages/server
+        run: |
+          supabase gen types typescript --local --schema public > ../shared/src/schema.ts
+          if ! git diff --ignore-space-at-eol --exit-code --quiet ../shared/src/schema.ts; then
+            echo "Detected uncommitted changes after build. See status below:"
+            git diff
+            exit 1
+          fi
+        env:
+          TYPECELL_GITHUB_OAUTH_SECRET: ${{ secrets.TYPECELL_GITHUB_OAUTH_SECRET }}
+          TYPECELL_GOOGLE_OAUTH_SECRET: ${{ secrets.TYPECELL_GOOGLE_OAUTH_SECRET }}
 
       # doesn't seem to cause significant speed up, probably because it still needs to install deps
       - name: cache playwright
         id: playwright-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/ms-playwright
           key: pw3-${{ runner.os }}-${{ hashFiles('./package-lock.json') }}
@@ -33,7 +67,7 @@ jobs:
       #     key: lerna-${{ runner.os }}-${{ hashFiles('./package-lock.json') }}
 
       - name: Cache node modules
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-node-modules
         with:
@@ -45,84 +79,82 @@ jobs:
             ${{ runner.os }}-build-
             ${{ runner.os }}-
 
-      - name: Set correct access for docker containers (server/test/data)
-        run: chmod -R a+rw test-util/server/data
+      # - name: Set correct access for docker containers (server/test/data)
+      #   run: chmod -R a+rw test-util/server/data
 
-      - name: Build the docker-compose stack
-        run: docker-compose -f test-util/server/docker-compose.yml up -d
+      # - name: Build the docker-compose stack
+      #   run: docker-compose -f test-util/server/docker-compose.yml up -d
 
-      - name: Check running containers
-        run: docker ps -a
+      # - name: Check running containers
+      #   run: docker ps -a
 
-      - name: Check docker logs
-        run: docker logs synapse
+      # - name: Check docker logs
+      #   run: docker logs synapse
 
       - name: NPM CI
         run: npm ci
 
       # - name: Bootstrap packages
       #   run: npm run bootstrap
 
-      - name: Wait for Matrix
-        run: npx wait-on http://localhost:8888/_matrix/static/
+      # - name: Wait for Matrix
+      #   run: npx wait-on http://localhost:8888/_matrix/static/
 
       - name: Build packages
         run: npm run build
         env:
           CI: true
 
+      - name: Build packages (vite react)
+        run: npm run build:react
+        env:
+          CI: true
+          MODE: development
+
+      - name: Lint
+        run: npm run lint
+        env:
+          CI: true
+
       - name: Install Playwright
         run: npx playwright install --with-deps
         # if: steps.playwright-cache.outputs.cache-hit != 'true'
         # env:
         # PLAYWRIGHT_BROWSERS_PATH: 0 # https://github.com/microsoft/playwright/blob/main/docs/src/ci.md#caching-browsers
 
-      - name: Lint
-        run: npm run lint
+      - name: Start HocusPocus server
+        run: RUNNER_TRACKING_ID="" && npm run start:server &
         env:
           CI: true
 
-      # Option A: don't build in CI, but test against dev server
-      # (with Vite, building in CI is now fast enough so we can go for option B)
-
-      # - name: Run dev server
-      #   # Wait on config file needed for vite dev server: https://github.com/jeffbski/wait-on/issues/78
-      #   run: npm run start:local & npx wait-on http://localhost:5173 -c ./packages/editor/wait-on.conf.json
-      #   env:
-      #     CI: true
-
-      # - name: Run Playwright tests (against dev server)
-      #   run: npm run playwright:dev
+      - name: Run Tests
+        run: npm run test
+        env:
+          CI: true
 
-      # Option B: actually build and run react code and run tests against that
-      - name: Build and run preview server
+      # Actually build and run react code and run tests against that
+      - name: Build and run preview
         # Wait on config file needed for vite dev server: https://github.com/jeffbski/wait-on/issues/78
         run: npm run start:preview & npx wait-on http://localhost:4173 -c ./packages/editor/wait-on.conf.json
         env:
           CI: true
+          MODE: development
 
-      - name: Run Playwright tests (against preview server)
+      - name: Run Playwright tests (against preview)
         run: npm run playwright:preview
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-report-editor
           path: packages/editor/playwright-report/
           retention-days: 30
 
-      - uses: actions/upload-artifact@v2
-        if: always()
+      # Upload webpack-stats.json to use on relative-ci.yaml workflow
+      - name: Upload webpack stats artifact
+        uses: relative-ci/agent-upload-artifact-action@v1
         with:
-          name: playwright-report-engine
-          path: packages/engine/playwright-report/
-          retention-days: 30
-
-      - name: Run Tests
-        run: npm run test
-        env:
-          CI: true
-
+          webpackStatsFile: ./packages/editor/dist/webpack-stats.json
       # - name: Upload to coveralls
       #   uses: coverallsapp/github-action@master
       #   with:

.github/workflows/production.yaml

@@ -0,0 +1,30 @@
+name: Deploy production db
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    env:
+      SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
+      SUPABASE_DB_PASSWORD: ${{ secrets.PRODUCTION_DB_PASSWORD }}
+      PROJECT_ID: guzxrzrjknsekuefovon
+      TYPECELL_GITHUB_OAUTH_SECRET: NOT-USED # not used but needs to be set, otherwise supabase cli complains
+      TYPECELL_GOOGLE_OAUTH_SECRET: NOT-USED # not used but needs to be set, otherwise supabase cli complains
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: supabase/setup-cli@v1
+        with:
+          version: latest
+
+      - run: supabase link --project-ref $PROJECT_ID
+        working-directory: packages/server
+      - run: supabase db push
+        working-directory: packages/server

.github/workflows/relative-ci.yaml

@@ -0,0 +1,17 @@
+name: RelativeCI
+
+on:
+  workflow_run:
+    workflows: ["build"]
+    types:
+      - completed
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send bundle stats and build information to RelativeCI
+        uses: relative-ci/agent-action@v2
+        with:
+          key: ${{ secrets.RELATIVE_CI_KEY }}
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/staging.yaml

@@ -0,0 +1,30 @@
+name: Deploy staging db
+
+on:
+  push:
+    branches:
+      - staging
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    env:
+      SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
+      SUPABASE_DB_PASSWORD: ${{ secrets.STAGING_DB_PASSWORD }}
+      PROJECT_ID: glsqqdamehahvdqssxow
+      TYPECELL_GITHUB_OAUTH_SECRET: NOT-USED # not used but needs to be set, otherwise supabase cli complains
+      TYPECELL_GOOGLE_OAUTH_SECRET: NOT-USED # not used but needs to be set, otherwise supabase cli complains
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: supabase/setup-cli@v1
+        with:
+          version: latest
+
+      - run: supabase link --project-ref $PROJECT_ID
+        working-directory: packages/server
+      - run: supabase db push
+        working-directory: packages/server

.gitignore

@@ -27,3 +27,4 @@ yarn-error.log*
 
 .idea/
 .vercel
+tsconfig.tsbuildinfo

.npmrc

@@ -1 +1,2 @@
 @matrix-org:registry=https://gitlab.matrix.org/api/v4/packages/npm/
+install-links=true

.nvmrc

@@ -1 +1 @@
-v16
+v18.16.0

.vscode/settings.json

@@ -9,13 +9,18 @@
   "search.exclude": {
     "packages/editor/public/types": true,
     "**/coverage": true,
-    "**/dist": true
+    "**/dist": true,
+    "**/coverage/**/*": true,
+    "**/dist/**/*": true,
+    "**/node_modules/**/*": true,
+    "**/supabase": true
   },
   "vitest.exclude": [
     "**/node_modules/**",
     "**/dist/**",
     "**/cypress/**",
     "**/.{idea,git,cache,output,temp}/**",
     "**/end-to-end/**"
-  ]
+  ],
+  "typescript.preferences.autoImportFileExcludePatterns": ["**/*/dist"]
 }

CONTRIBUTING.md

@@ -11,7 +11,7 @@ blocknote
 │   ├── editor          - The main React application
 │   ├── engine          - The live-code execution engine
 │   ├── packager        - Tool to bundle TypeCell notebook apps (WIP)
-│   └── parsers         - Helpers to convert to / from TypeCell notebooks
+│   └── parsers         - Helpers to convert to / from TypeCell
 ├── patches             - patch-package patches
 └── test-util           - Server and data for unit tests
 ```

DEPLOYMENT.md

@@ -0,0 +1,22 @@
+# Deployment & CI
+
+TypeCell currently consists of
+
+|                                     | development / CI | staging                     | prod                     |
+| ----------------------------------- | ---------------- | --------------------------- | ------------------------ |
+| React app (packages/editor)         | Local            | Vercel (branch-specific)    | Vercel                   |
+| HocusPocus Server (packages/server) | Local            | Render (typecell-staging)   | Render (typecell-prod)   |
+| Supabase DB infra (packages/server) | Docker           | Supabase (typecell-staging) | Supabase (typecell-prod) |
+
+A few principles:
+
+- All Github Actions run against `local` (i.e.: no external dependencies)
+- All components should be easy to self-host, we don't want forced dependencies on any cloud provider (note that both supabase and the server are self-hostable).
+- Only www.typecell.org is considered `prod`. All Vercel branch-preview environments are considered `staging`, except the main branch which is `prod`.
+- Preview environments all share the same `staging` database. This means feature-branches must be compatible in terms of database schema.
+- Preview environments all share the same `HocusPocus backend server` (`packages/server`). This means feature-branches must be compatible to run against the same `packages/server`.
+- DB Migrations are deployed to `staging` / `prod` when they are pushed to the `staging` / `main` branch (see github workflows).
+
+## Self-host
+
+It should be fairly straightforward to host TypeCell yourself, at this moment we don't have a guide for this yet.

README.md

@@ -72,7 +72,7 @@ typecell
 │   ├── editor          - The main React application
 │   ├── engine          - The live-code execution engine
 │   ├── packager        - Tool to bundle TypeCell notebook apps (WIP)
-│   └── parsers         - Helpers to convert to / from TypeCell notebooks
+│   └── parsers         - Helpers to convert to / from TypeCell documents
 ├── patches             - patch-package patches
 └── test-util           - Server and data for unit tests
 ```