Merge remote-tracking branch 'origin/main' into fcrdns

Author: Axelen123

.github/actions/spelling/allow.txt

@@ -2,4 +2,4 @@ github
 https
 ssh
 ubuntu
-workarounds
+workarounds

.github/actions/spelling/excludes.txt

@@ -83,7 +83,9 @@
 ^\Q.github/FUNDING.yml\E$
 ^\Q.github/workflows/spelling.yml\E$
 ^data/crawlers/
+^docs/blog/tags\.yml$
 ^docs/manifest/.*$
 ^docs/static/\.nojekyll$
+^lib/policy/config/testdata/bad/unparseable\.json$
 ignore$
-robots.txt
+robots.txt

.github/actions/spelling/expect.txt

@@ -44,7 +44,6 @@ chall
 challengemozilla
 checkpath
 checkresult
-chen
 chibi
 cidranger
 ckie
@@ -61,7 +60,6 @@ DDOS
 Debian
 debrpm
 decaymap
-decompiling
 Diffbot
 discordapp
 discordbot
@@ -188,6 +186,7 @@ ogtags
 omgili
 omgilibot
 openai
+opengraph
 openrc
 pag
 palemoon
@@ -218,6 +217,7 @@ rawler
 rcvar
 redir
 redirectscheme
+refactors
 relayd
 reputational
 reqmeta
@@ -261,6 +261,7 @@ techarohq
 templ
 templruntime
 testarea
+Thancred
 thoth
 thothmock
 Tik
@@ -300,6 +301,7 @@ xess
 xff
 XForwarded
 XNG
+XOB
 XReal
 yae
 YAMLTo

.github/actions/spelling/line_forbidden.patterns

@@ -273,14 +273,6 @@
 # Most people only have two hands. Reword.
 \b(?i)on the third hand\b
 
-# Should be `Open Graph`
-# unless talking about a specific Open Graph implementation:
-# - Java
-# - Node
-# - Py
-# - Ruby
-\bOpenGraph\b
-
 # Should be `OpenShift`
 \bOpenshift\b
 

.github/actions/spelling/patterns.txt

@@ -131,4 +131,4 @@ go install(?:\s+[a-z]+\.[-@\w/.]+)+
 
 # hit-count: 1 file-count: 1
 # microsoft
-\b(?:https?://|)(?:(?:(?:blogs|download\.visualstudio|docs|msdn2?|research)\.|)microsoft|blogs\.msdn)\.co(?:m|\.\w\w)/[-_a-zA-Z0-9()=./%]*
+\b(?:https?://|)(?:(?:(?:blogs|download\.visualstudio|docs|msdn2?|research)\.|)microsoft|blogs\.msdn)\.co(?:m|\.\w\w)/[-_a-zA-Z0-9()=./%]*

Makefile

@@ -18,6 +18,7 @@ assets: deps
 
 build: assets
 	$(GO) build -o ./var/anubis ./cmd/anubis
+	$(GO) build -o ./var/robots2policy ./cmd/robots2policy
 	@echo "Anubis is now built to ./var/anubis"
 
 lint: assets
@@ -27,6 +28,7 @@ lint: assets
 
 prebaked-build:
 	$(GO) build -o ./var/anubis -ldflags "-X 'github.com/TecharoHQ/anubis.Version=$(VERSION)'" ./cmd/anubis
+	$(GO) build -o ./var/robots2policy -ldflags "-X 'github.com/TecharoHQ/anubis.Version=$(VERSION)'" ./cmd/robots2policy
 
 test: assets
 	$(GO) test ./...

VERSION

@@ -1 +1 @@
-1.19.1
+1.20.0-pre1

cmd/anubis/main.go

@@ -333,24 +333,29 @@ func main() {
 		slog.Warn("REDIRECT_DOMAINS is not set, Anubis will only redirect to the same domain a request is coming from, see https://anubis.techaro.lol/docs/admin/configuration/redirect-domains")
 	}
 
-	s, err := libanubis.New(libanubis.Options{
-		BasePrefix:           *basePrefix,
-		StripBasePrefix:      *stripBasePrefix,
-		Next:                 rp,
-		Policy:               policy,
-		ServeRobotsTXT:       *robotsTxt,
-		PrivateKey:           priv,
-		CookieDomain:         *cookieDomain,
-		CookieExpiration:     *cookieExpiration,
-		CookiePartitioned:    *cookiePartitioned,
-		OGPassthrough:        *ogPassthrough,
-		OGTimeToLive:         *ogTimeToLive,
-		RedirectDomains:      redirectDomainsList,
-		Target:               *target,
-		WebmasterEmail:       *webmasterEmail,
-		OGCacheConsidersHost: *ogCacheConsiderHost,
-		FCrDNS:               fdns,
+	// If OpenGraph configuration values are not set in the config file, use the
+	// values from flags / envvars.
+	if !policy.OpenGraph.Enabled {
+		policy.OpenGraph.Enabled = *ogPassthrough
+		policy.OpenGraph.ConsiderHost = *ogCacheConsiderHost
+		policy.OpenGraph.TimeToLive = *ogTimeToLive
+		policy.OpenGraph.Override = map[string]string{}
+	}
 
+	s, err := libanubis.New(libanubis.Options{
+		FCrDNS:            fdns,
+		BasePrefix:        *basePrefix,
+		StripBasePrefix:   *stripBasePrefix,
+		Next:              rp,
+		Policy:            policy,
+		ServeRobotsTXT:    *robotsTxt,
+		PrivateKey:        priv,
+		CookieDomain:      *cookieDomain,
+		CookieExpiration:  *cookieExpiration,
+		CookiePartitioned: *cookiePartitioned,
+		RedirectDomains:   redirectDomainsList,
+		Target:            *target,
+		WebmasterEmail:    *webmasterEmail,
 	})
 	if err != nil {
 		log.Fatalf("can't construct libanubis.Server: %v", err)

data/botPolicies.yaml

@@ -56,7 +56,7 @@ bots:
   - name: countries-with-aggressive-scrapers
     action: WEIGH
     geoip:
-      counties:
+      countries:
         - BR
         - CN
     weight:
@@ -84,10 +84,88 @@ bots:
 
 dnsbl: false
 
+# Open Graph passthrough configuration, see here for more information:
+# https://anubis.techaro.lol/docs/admin/configuration/open-graph/
+openGraph:
+  # Enables Open Graph passthrough
+  enabled: false
+  # Enables the use of the HTTP host in the cache key, this enables
+  # caching metadata for multiple http hosts at once.
+  considerHost: false
+  # How long cached OpenGraph metadata should last in memory
+  ttl: 24h
+  # # If set, return these opengraph values instead of looking them up with
+  # # the target service.
+  # #
+  # # Correlates to properties in https://ogp.me/
+  # override:
+  #   # og:title is required, it is the title of the website
+  #   "og:title": "Techaro Anubis"
+  #   "og:description": >-
+  #     Anubis is a Web AI Firewall Utility that helps you fight the bots
+  #     away so that you can maintain uptime at work!
+  #   "description": >-
+  #     Anubis is a Web AI Firewall Utility that helps you fight the bots
+  #     away so that you can maintain uptime at work!
+
 # By default, send HTTP 200 back to clients that either get issued a challenge
 # or a denial. This seems weird, but this is load-bearing due to the fact that
 # the most aggressive scraper bots seem to really, really, want an HTTP 200 and
 # will stop sending requests once they get it.
 status_codes:
   CHALLENGE: 200
   DENY: 200
+
+# The weight thresholds for when to trigger individual challenges. Any
+# CHALLENGE will take precedence over this.
+#
+# A threshold has four configuration options:
+#
+#   - name: the name that is reported down the stack and used for metrics
+#   - expression: A CEL expression with the request weight in the variable
+#     weight
+#   - action: the Anubis action to apply, similar to in a bot policy
+#   - challenge: which challenge to send to the user, similar to in a bot policy
+#
+# See https://anubis.techaro.lol/docs/admin/configuration/thresholds for more
+# information.
+thresholds:
+  # By default Anubis ships with the following thresholds:
+  - name: minimal-suspicion # This client is likely fine, its soul is lighter than a feather
+    expression: weight <= 0 # a feather weighs zero units
+    action: ALLOW # Allow the traffic through
+  # For clients that had some weight reduced through custom rules, give them a
+  # lightweight challenge.
+  - name: mild-suspicion
+    expression:
+      all:
+        - weight > 0
+        - weight < 10
+    action: CHALLENGE
+    challenge:
+      # https://anubis.techaro.lol/docs/admin/configuration/challenges/metarefresh
+      algorithm: metarefresh
+      difficulty: 1
+      report_as: 1
+  # For clients that are browser-like but have either gained points from custom rules or
+  # report as a standard browser.
+  - name: moderate-suspicion
+    expression:
+      all:
+        - weight >= 10
+        - weight < 20
+    action: CHALLENGE
+    challenge:
+      # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
+      algorithm: fast
+      difficulty: 2 # two leading zeros, very fast for most clients
+      report_as: 2
+  # For clients that are browser like and have gained many points from custom rules
+  - name: extreme-suspicion
+    expression: weight >= 20
+    action: CHALLENGE
+    challenge:
+      # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
+      algorithm: fast
+      difficulty: 4
+      report_as: 4

data/bots/ai-robots-txt.yaml

@@ -2,5 +2,5 @@
 # Note: Blocks human-directed/non-training user agents
 - name: "ai-robots-txt"
   user_agent_regex: >-
-    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|Andibot|anthropic-ai|Applebot|Applebot-Extended|bedrockbot|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-SearchBot|Claude-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|MistralAI-User/1.0|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient.com|Perplexity-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|QuillBot|quillbot.com|SBIntuitionsBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot
+    AI2Bot|Ai2Bot-Dolma|aiHitBot|Amazonbot|Andibot|anthropic-ai|Applebot|Applebot-Extended|bedrockbot|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-SearchBot|Claude-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google-CloudVertexBot|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|meta-externalagent|Meta-ExternalAgent|meta-externalfetcher|Meta-ExternalFetcher|MistralAI-User/1.0|MyCentralAIScraperBot|NovaAct|OAI-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient.com|Perplexity-User|PerplexityBot|PetalBot|PhindBot|Poseidon Research Crawler|QualifiedBot|QuillBot|quillbot.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot-BA|SemrushBot-CT|SemrushBot-OCOB|SemrushBot-SI|SemrushBot-SWA|Sidetrade indexer bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot
   action: DENY