feat: add sitemap and update workflows (#27)

.github/workflows/ci.yml

@@ -50,6 +50,9 @@ jobs:
       - name: Build PDFs
         run: pnpm run build:pdfs
 
+      - name: Build Sitemap
+        run: pnpm run build:sitemap
+
       - name: Run build
         run: pnpm run build
 

.gitignore

@@ -30,3 +30,4 @@ coverage
 *.tsbuildinfo
 
 public/pdfs/
+public/sitemap.xml

package.json

@@ -14,6 +14,7 @@
     "format": "prettier --write src/",
     "format-check": "prettier --check src/",
     "build:pdfs": "vite-node scripts/build-pdfs.ts",
+    "build:sitemap": "vite-node scripts/build-sitemap.ts",
     "prepare": "husky"
   },
   "publishConfig": {
@@ -61,6 +62,7 @@
     "npm-run-all2": "^7.0.2",
     "prettier": "^3.4.2",
     "puppeteer": "^23.11.1",
+    "sitemap": "^8.0.0",
     "typescript": "~5.7.2",
     "vite": "^6.0.6",
     "vite-node": "^2.1.8",

public/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://how-to.lookscanned.io/sitemap.xml

scripts/build-sitemap.ts

@@ -0,0 +1,48 @@
+import { SitemapStream, streamToPromise } from 'sitemap'
+import { markdowns } from '../src/locales/how-to-use/markdowns'
+import { writeFile } from 'fs/promises'
+import { execSync } from 'child_process'
+
+const langs = Object.keys(markdowns) as (keyof typeof markdowns)[]
+const commitTimestamp = execSync('git log -1 --format=%at').toString().trim()
+const commitDate = new Date(parseInt(commitTimestamp) * 1000)
+const lastmod = commitDate.toISOString()
+
+const smStream = new SitemapStream({
+  hostname: 'https://how-to.lookscanned.io',
+  lastmodDateOnly: false,
+  xmlns: {
+    // trim the xml namespace
+    news: false, // flip to false to omit the xml namespace for news
+    xhtml: true,
+    image: false,
+    video: false,
+  },
+})
+
+const xmlBuffer = new Promise<Buffer>((resolve, reject) => {
+  streamToPromise(smStream).then(resolve).catch(reject)
+})
+
+smStream.write({
+  url: 'https://how-to.lookscanned.io',
+  lastmod,
+})
+
+for (const lang of langs) {
+  smStream.write({
+    url: `https://how-to.lookscanned.io/how-to-use/${lang}`,
+    lastmod,
+  })
+  smStream.write({
+    url: `https://how-to.lookscanned.io/pdfs/how-to-use/${lang}.pdf`,
+    lastmod,
+  })
+}
+
+smStream.end()
+
+const buffer = await xmlBuffer
+
+// write to ./public/sitemap.xml
+await writeFile('./public/sitemap.xml', buffer)