feat(config): Add FCrDNS checker

.github/actions/spelling/expect.txt

@@ -87,6 +87,11 @@ extldflags
 facebookgo
 Factset
 fastcgi
+FCr
+fcrdns
+fcrdnscheck
+fdc
+fdns
 fediverse
 finfos
 Firecrawl
@@ -136,6 +141,7 @@ impressum
 inp
 IPTo
 iptoasn
+isp
 iss
 isset
 ivh
@@ -227,6 +233,7 @@ redir
 redirectscheme
 refactors
 relayd
+remotehost
 reputational
 reqmeta
 risc

cmd/anubis/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/data"
 	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/fcrdns"
 	"github.com/TecharoHQ/anubis/internal/thoth"
 	libanubis "github.com/TecharoHQ/anubis/lib"
 	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
@@ -278,7 +279,8 @@ func main() {
 		log.Fatalf("you can't set COOKIE_DOMAIN and COOKIE_DYNAMIC_DOMAIN at the same time")
 	}
 
-	ctx := context.Background()
+	fdns := fcrdns.NewFCrDNS()
+	ctx := fcrdns.With(context.Background(), fdns)
 
 	// Thoth configuration
 	switch {
@@ -384,6 +386,7 @@ func main() {
 	}
 
 	s, err := libanubis.New(libanubis.Options{
+		FCrDNS:            fdns,
 		BasePrefix:        *basePrefix,
 		StripBasePrefix:   *stripBasePrefix,
 		Next:              rp,

data/crawlers/applebot.yaml

@@ -1,20 +1,16 @@
 # Indexing for search and Siri
 # https://support.apple.com/en-us/119829
 - name: applebot
-  user_agent_regex: Applebot
+  expression:
+    all:
+      - userAgent.matches("Applebot")
+      - fcrdns.check("\\.applebot\\.apple\\.com$")
   action: ALLOW
-  # https://search.developer.apple.com/applebot.json
-  remote_addresses: [
-    "17.241.208.160/27",
-    "17.241.193.160/27",
-    "17.241.200.160/27",
-    "17.22.237.0/24",
-    "17.22.245.0/24",
-    "17.22.253.0/24",
-    "17.241.75.0/24",
-    "17.241.219.0/24",
-    "17.241.227.0/24",
-    "17.246.15.0/24",
-    "17.246.19.0/24",
-    "17.246.23.0/24",
-  ]
+- name: not-applebot
+  expression:
+    all:
+      - userAgent.matches("Applebot")
+      - '!(fcrdns.check("\\.applebot\\.apple\\.com$"))'
+  action: WEIGH
+  weight:
+    adjust: 5

data/crawlers/bingbot.yaml

@@ -1,34 +1,15 @@
 - name: bingbot
-  user_agent_regex: \+http\://www\.bing\.com/bingbot\.htm
+  # https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26
+  expression:
+    all:
+      - userAgent.matches("\\+http\\://www\\.bing\\.com/bingbot\\.htm")
+      - fcrdns.check("\\.search\\.msn\\.com$")
   action: ALLOW
-  # https://www.bing.com/toolbox/bingbot.json
-  remote_addresses: [
-    "157.55.39.0/24",
-    "207.46.13.0/24",
-    "40.77.167.0/24",
-    "13.66.139.0/24",
-    "13.66.144.0/24",
-    "52.167.144.0/24",
-    "13.67.10.16/28",
-    "13.69.66.240/28",
-    "13.71.172.224/28",
-    "139.217.52.0/28",
-    "191.233.204.224/28",
-    "20.36.108.32/28",
-    "20.43.120.16/28",
-    "40.79.131.208/28",
-    "40.79.186.176/28",
-    "52.231.148.0/28",
-    "20.79.107.240/28",
-    "51.105.67.0/28",
-    "20.125.163.80/28",
-    "40.77.188.0/22",
-    "65.55.210.0/24",
-    "199.30.24.0/23",
-    "40.77.202.0/24",
-    "40.77.139.0/25",
-    "20.74.197.0/28",
-    "20.15.133.160/27",
-    "40.77.177.0/24",
-    "40.77.178.0/23"
-  ]
+- name: not-bingbot
+  expression:
+    all:
+      - userAgent.matches("\\+http\\://www\\.bing\\.com/bingbot\\.htm")
+      - '!(fcrdns.check("\\.search\\.msn\\.com$"))'
+  action: WEIGH
+  weight:
+    adjust: 5