Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,9 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
sparse-checkout: website/wrangler.jsonc
sparse-checkout: |
website/wrangler.jsonc
website/scripts/indexnow.mjs
sparse-checkout-cone-mode: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
Expand All @@ -157,3 +159,10 @@ jobs:
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
run: npx --yes wrangler@4.145.0 deploy
# Tell Bing and the other IndexNow engines which pages this deploy changed, so
# they recrawl now rather than whenever. Never fails the deploy: a missed ping
# only means the next regular crawl picks the change up.
- name: Ping IndexNow
working-directory: website
continue-on-error: true
run: node scripts/indexnow.mjs build
59 changes: 59 additions & 0 deletions website/scripts/indexnow.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env node
// After a deploy, tell Bing (and every other IndexNow engine) which pages just changed,
// instead of waiting for its next crawl. ChatGPT search answers from Bing's index.
//
// Changed = <lastmod> within the last two days in any locale sitemap of the build. The
// sitemap's lastmod comes from `git log` per file (see docs.yml), so an untouched page
// is not resubmitted on every deploy. llms.txt is not in a sitemap, so it always goes.
//
// Usage: node scripts/indexnow.mjs <build dir> [--dry-run] (--dry-run lists the URLs, sends nothing)
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";

const HOST = "getopenscreen.com";
// The key is public by design: IndexNow checks it against /<key>.txt on the host.
const KEY = "233520eba1fb46fd9331a9921e77319d";
const WINDOW_MS = 2 * 24 * 60 * 60 * 1000;

function changedUrls(sitemapXml, now = Date.now()) {
const urls = [];
for (const [, block] of sitemapXml.matchAll(/<url>(.*?)<\/url>/gs)) {
const loc = block.match(/<loc>(.*?)<\/loc>/)?.[1];
const lastmod = block.match(/<lastmod>(.*?)<\/lastmod>/)?.[1];
if (loc && lastmod && now - Date.parse(lastmod) <= WINDOW_MS) urls.push(loc);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
return urls;
}

// The root sitemap, plus one per locale directory (fr/, zh-cn/, …).
function sitemaps(dir) {
const candidates = [join(dir, "sitemap.xml")];
for (const name of readdirSync(dir)) {
if (/^[a-z]{2}(-[a-z]{2})?$/.test(name)) candidates.push(join(dir, name, "sitemap.xml"));
}
return candidates.filter((path) => existsSync(path));
}

const build = process.argv[2] ?? "build";
const urls = new Set([`https://${HOST}/llms.txt`]);
for (const file of sitemaps(build)) {
for (const url of changedUrls(readFileSync(file, "utf8"))) urls.add(url);
}
if (process.argv.includes("--dry-run")) {
console.log([...urls].join("\n"));
} else {
const res = await fetch("https://api.indexnow.org/indexnow", {
method: "POST",
headers: { "Content-Type": "application/json; charset=utf-8" },
body: JSON.stringify({
host: HOST,
key: KEY,
keyLocation: `https://${HOST}/${KEY}.txt`,
urlList: [...urls],
}),
});
console.log(`IndexNow: ${urls.size} URL(s) submitted, HTTP ${res.status}`);
// 200 and 202 both mean accepted. Anything else is reported, never fatal: a failed
// ping only means Bing finds the change on its next crawl.
if (res.status !== 200 && res.status !== 202) console.log(await res.text());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
1 change: 1 addition & 0 deletions website/static/233520eba1fb46fd9331a9921e77319d.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
233520eba1fb46fd9331a9921e77319d
Loading