diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 145e53cff..c65bd5f33 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -106,6 +106,7 @@ externalfetcher extldflags facebookgo Factset +fahedouch fastcgi fediverse ffprobe @@ -194,6 +195,7 @@ lcj ldflags letsencrypt Lexentale +lfc lgbt licend licstart diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index ff13d9948..5b3ddb50f 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -22,6 +22,7 @@ jobs: - git-push - healthcheck - i18n + - log-file - palemoon/amd64 #- palemoon/i386 - robots_txt diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go index 6ad10274f..e21eeca1d 100644 --- a/cmd/anubis/main.go +++ b/cmd/anubis/main.go @@ -31,8 +31,8 @@ import ( "github.com/TecharoHQ/anubis/data" "github.com/TecharoHQ/anubis/internal" libanubis "github.com/TecharoHQ/anubis/lib" + "github.com/TecharoHQ/anubis/lib/config" botPolicy "github.com/TecharoHQ/anubis/lib/policy" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/lib/thoth" "github.com/TecharoHQ/anubis/web" "github.com/facebookgo/flagenv" @@ -273,9 +273,11 @@ func main() { return } - internal.InitSlog(*slogLevel) internal.SetHealth("anubis", healthv1.HealthCheckResponse_NOT_SERVING) + lg := internal.InitSlog(*slogLevel, os.Stderr) + lg.Info("starting up Anubis") + if *healthcheck { log.Println("running healthcheck") if err := doHealthCheck(); err != nil { @@ -303,7 +305,7 @@ func main() { if *metricsBind != "" { wg.Add(1) - go metricsServer(ctx, wg.Done) + go metricsServer(ctx, *lg.With("subsystem", "metrics"), wg.Done) } var rp http.Handler @@ -323,11 +325,11 @@ func main() { // Thoth configuration switch { case *thothURL != "" && *thothToken == "": - slog.Warn("THOTH_URL is set but no THOTH_TOKEN is set") + lg.Warn("THOTH_URL is set but no THOTH_TOKEN is set") case *thothURL == "" && *thothToken != "": - slog.Warn("THOTH_TOKEN is set but no THOTH_URL is set") + lg.Warn("THOTH_TOKEN is set but no THOTH_URL is set") case *thothURL != "" && *thothToken != "": - slog.Debug("connecting to Thoth") + lg.Debug("connecting to Thoth") thothClient, err := thoth.New(ctx, *thothURL, *thothToken, *thothInsecure) if err != nil { log.Fatalf("can't dial thoth at %s: %v", *thothURL, err) @@ -336,15 +338,19 @@ func main() { ctx = thoth.With(ctx, thothClient) } - policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty) + lg.Info("loading policy file", "fname", *policyFname) + policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel) if err != nil { log.Fatalf("can't parse policy file: %v", err) } + lg = policy.Logger + lg.Debug("swapped to new logger") + slog.SetDefault(lg) // Warn if persistent storage is used without a configured signing key if policy.Store.IsPersistent() { if *hs512Secret == "" && *ed25519PrivateKeyHex == "" && *ed25519PrivateKeyHexFile == "" { - slog.Warn("[misconfiguration] persistent storage backend is configured, but no private key is set. " + + lg.Warn("[misconfiguration] persistent storage backend is configured, but no private key is set. " + "Challenges will be invalidated when Anubis restarts. " + "Set HS512_SECRET, ED25519_PRIVATE_KEY_HEX, or ED25519_PRIVATE_KEY_HEX_FILE to ensure challenges survive service restarts. " + "See: https://anubis.techaro.lol/docs/admin/installation#key-generation") @@ -407,7 +413,7 @@ func main() { log.Fatalf("failed to generate ed25519 key: %v", err) } - slog.Warn("generating random key, Anubis will have strange behavior when multiple instances are behind the same load balancer target, for more information: see https://anubis.techaro.lol/docs/admin/installation#key-generation") + lg.Warn("generating random key, Anubis will have strange behavior when multiple instances are behind the same load balancer target, for more information: see https://anubis.techaro.lol/docs/admin/installation#key-generation") } var redirectDomainsList []string @@ -421,7 +427,7 @@ func main() { redirectDomainsList = append(redirectDomainsList, strings.TrimSpace(domain)) } } else { - slog.Warn("REDIRECT_DOMAINS is not set, Anubis will only redirect to the same domain a request is coming from, see https://anubis.techaro.lol/docs/admin/configuration/redirect-domains") + lg.Warn("REDIRECT_DOMAINS is not set, Anubis will only redirect to the same domain a request is coming from, see https://anubis.techaro.lol/docs/admin/configuration/redirect-domains") } anubis.CookieName = *cookiePrefix + "-auth" @@ -461,6 +467,7 @@ func main() { CookieSameSite: parseSameSite(*cookieSameSite), PublicUrl: *publicUrl, JWTRestrictionHeader: *jwtRestrictionHeader, + Logger: policy.Logger.With("subsystem", "anubis"), DifficultyInJWT: *difficultyInJWT, }) if err != nil { @@ -477,7 +484,7 @@ func main() { srv := http.Server{Handler: h, ErrorLog: internal.GetFilteredHTTPLogger()} listener, listenerUrl := setupListener(*bindNetwork, *bind) - slog.Info( + lg.Info( "listening", "url", listenerUrl, "difficulty", *challengeDifficulty, @@ -511,7 +518,7 @@ func main() { wg.Wait() } -func metricsServer(ctx context.Context, done func()) { +func metricsServer(ctx context.Context, lg slog.Logger, done func()) { defer done() mux := http.NewServeMux() @@ -537,7 +544,7 @@ func metricsServer(ctx context.Context, done func()) { srv := http.Server{Handler: mux, ErrorLog: internal.GetFilteredHTTPLogger()} listener, metricsUrl := setupListener(*metricsBindNetwork, *metricsBind) - slog.Debug("listening for metrics", "url", metricsUrl) + lg.Debug("listening for metrics", "url", metricsUrl) go func() { <-ctx.Done() diff --git a/cmd/containerbuild/main.go b/cmd/containerbuild/main.go index ce1995d72..a351f3471 100644 --- a/cmd/containerbuild/main.go +++ b/cmd/containerbuild/main.go @@ -28,7 +28,7 @@ func main() { flagenv.Parse() flag.Parse() - internal.InitSlog(*slogLevel) + slog.SetDefault(internal.InitSlog(*slogLevel, os.Stderr)) koDockerRepo := strings.TrimSuffix(*dockerRepo, "/"+filepath.Base(*dockerRepo)) diff --git a/cmd/robots2policy/main.go b/cmd/robots2policy/main.go index 3bb7219df..69fb2f969 100644 --- a/cmd/robots2policy/main.go +++ b/cmd/robots2policy/main.go @@ -12,7 +12,7 @@ import ( "regexp" "strings" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "sigs.k8s.io/yaml" ) diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index 66e09ea28..873f78d2f 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -21,9 +21,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Allow Renovate as an OCI registry client. - Properly handle 4in6 addresses so that IP matching works with those addresses. - Add support to simple Valkey/Redis cluster mode -- Open Graph passthrough now reuses the configured target Host/SNI/TLS settings, so metadata fetches succeed when the upstream certificate differs from the public domain. ([1283](https://github.com/TecharoHQ/anubis/pull/1283)) +- Open Graph passthrough now reuses the configured target Host/SNI/TLS settings, so metadata fetches succeed when the upstream certificate differs from the public domain. ([1283](https://github.com/TecharoHQ/anubis/pull/1283)) - Stabilize the CVE-2025-24369 regression test by always submitting an invalid proof instead of relying on random POW failures. +### Logging customization + +Anubis now supports the ability to log to multiple backends ("sinks"). This allows you to have Anubis [log to a file](./admin/policies.mdx#file-sink) instead of just logging to standard out. You can also customize the [logging level](./admin/policies.mdx#log-levels) in the policy file: + +```yaml +logging: + level: "warn" # much less verbose logging + sink: file # log to a file + parameters: + file: "./var/anubis.log" + maxBackups: 3 # keep at least 3 old copies + maxBytes: 67108864 # each file can have up to 64 Mi of logs + maxAge: 7 # rotate files out every n days + oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish + compress: true # gzip-compress old log files + useLocalTime: false # timezone for rotated files is UTC +``` + +Additionally, information about [how Anubis uses each logging level](./admin/policies.mdx#log-levels) has been added to the documentation. + ## v1.23.1: Lyse Hext - Echo 1 - Fix `SERVE_ROBOTS_TXT` setting after the double slash fix broke it. diff --git a/docs/docs/admin/policies.mdx b/docs/docs/admin/policies.mdx index 2065f49bc..a3a2e7d4a 100644 --- a/docs/docs/admin/policies.mdx +++ b/docs/docs/admin/policies.mdx @@ -328,6 +328,84 @@ If you are using [Redis™ Sentinel](https://redis.io/docs/latest/operate/oss_an | `username` | string | `azurediamond` | The username used to authenticate against the Redis™ Sentinel and Redis™ servers. | | `password` | string | `hunter2` | The password used to authenticate against the Redis™ Sentinel and Redis™ servers. | +## Logging management + +Anubis has very verbose logging out of the box. This is intentional and allows administrators to be sure that it is working merely by watching it work in real time. Some administrators may not appreciate this level of logging out of the box. As such, Anubis lets you customize details about how it logs data. + +Anubis uses a practice called [structured logging](https://stackify.com/what-is-structured-logging-and-why-developers-need-it/) to emit log messages with key-value pair context. In order to make analyzing large amounts of log messages easier, Anubis encodes all logs in JSON. This allows you to use any tool that can parse JSON to perform analytics or monitor for issues. + +Anubis exposes the following logging settings in the policy file: + +| Name | Type | Example | Description | +| :----------- | :----------------------- | :-------------- | :--------------------------------------------------------------------------------------------------------------------------------------- | +| `level` | [log level](#log-levels) | `info` | The logging level threshold. Any logs that are at or above this threshold will be drained to the sink. Any other logs will be discarded. | +| `sink` | string | `stdio`, `file` | The sink where the logs drain to as they are being recorded in Anubis. | +| `parameters` | object | | Parameters for the given logging sink. This will vary based on the logging sink of choice. See below for more information. | + +Anubis supports the following logging sinks: + +1. `file`: logs are emitted to a file that is rotated based on size and age. Old log files are compressed with gzip to save space. This allows for better integration with users that decide to use legacy service managers (OpenRC, FreeBSD's init, etc). +2. `stdio`: logs are emitted to the standard error stream of the Anubis process. This allows runtimes such as Docker, Podman, Systemd, and Kubernetes to capture logs with their native logging subsystems without any additional configuration. + +### Log levels + +Anubis uses Go's [standard library `log/slog` package](https://pkg.go.dev/log/slog) to emit structured logs. By default, Anubis logs at the [Info level](https://pkg.go.dev/log/slog#Level), which is fairly verbose out of the box. Here are the possible logging levels in Anubis: + +| Log level | Use in Anubis | +| :-------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DEBUG` | The raw unfiltered torrent of doom. Only use this if you are actively working on Anubis or have very good reasons to use it. | +| `INFO` | The default logging level, fairly verbose in order to make it easier for automation to parse. | +| `WARN` | A "more silent" logging level. Much less verbose. Some things that are now at the `info` level need to be moved up to the `warn` level in future patches. | +| `ERROR` | Only log error messages. | + +Additionally, you can set a "slightly higher" log level if you need to, such as: + +```yaml +logging: + sink: stdio + level: "INFO+1" +``` + +This isn't currently used by Anubis, but will be in the future for "slightly important" information. + +### `file` sink + +The `file` sink makes Anubis write its logs to the filesystem and rotate them out when the log file meets certain thresholds. This logging sink takes the following parameters: + +| Name | Type | Example | Description | +| :------------- | :-------------- | :-------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `file` | string | `/var/log/anubis.log` | The file where Anubis logs should be written to. Make sure the user Anubis is running as has write and file creation permissions to this directory. | +| `maxBackups` | number | `3` | The number of old log files that should be maintained when log files are rotated out. | +| `maxBytes` | number of bytes | `67108864` (64Mi) | The maximum size of each log file before it is rotated out. | +| `maxAge` | number of days | `7` | If a log file is more than this many days old, rotate it out. | +| `compress` | boolean | `true` | If true, compress old log files with gzip. This should be set to `true` and is only exposed as an option for dealing with legacy workflows where there is magical thinking about log files at play. | +| `useLocalTime` | boolean | `false` | If true, use the system local time zone to create log filenames instead of UTC. This should almost always be set to `false` and is only exposed for legacy workflows where there is magical thinking about time zones at play. | + +```yaml +logging: + sink: file + parameters: + file: "./var/anubis.log" + maxBackups: 3 # keep at least 3 old copies + maxBytes: 67108864 # each file can have up to 64 Mi of logs + maxAge: 7 # rotate files out every n days + compress: true # gzip-compress old log files + useLocalTime: false # timezone for rotated files is UTC +``` + +When files are rotated out, the old files will be named after the rotation timestamp in [RFC 3339 format](https://www.rfc-editor.org/rfc/rfc3339). + +### `stdio` sink + +By default, Anubis logs everything to the standard error stream of its process. This requires no configuration: + +```yaml +logging: + sink: stdio +``` + +If you use a service orchestration platform that does not capture the standard error stream of processes, you need to use a different logging sink. + ## Risk calculation for downstream services In case your service needs it for risk calculation reasons, Anubis exposes information about the rules that any requests match using a few headers: diff --git a/go.mod b/go.mod index a2beea57a..67ef39e5c 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 github.com/cespare/xxhash/v2 v2.3.0 github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 + github.com/fahedouch/go-logrotate v0.3.0 github.com/gaissmai/bart v0.26.0 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/cel-go v0.26.1 @@ -86,6 +87,7 @@ require ( github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/djherbis/times v1.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/docker v28.5.1+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect diff --git a/go.sum b/go.sum index 821409f52..9c809bad0 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= @@ -163,6 +165,8 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fahedouch/go-logrotate v0.3.0 h1:XP+dHIDgWZ1ckz43mG6gl5ASer3PZDVr755SVMyzaUQ= +github.com/fahedouch/go-logrotate v0.3.0/go.mod h1:X49m0bvPLkk71MHNCQ1yEfVEw8W/u+qvHa/hOnhCYf4= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -479,6 +483,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/log.go b/internal/log.go index fb2f2509c..4b8bd02fb 100644 --- a/internal/log.go +++ b/internal/log.go @@ -2,6 +2,7 @@ package internal import ( "fmt" + "io" "log" "log/slog" "net/http" @@ -9,7 +10,7 @@ import ( "strings" ) -func InitSlog(level string) { +func InitSlog(level string, sink io.Writer) *slog.Logger { var programLevel slog.Level if err := (&programLevel).UnmarshalText([]byte(level)); err != nil { fmt.Fprintf(os.Stderr, "invalid log level %s: %v, using info\n", level, err) @@ -19,11 +20,12 @@ func InitSlog(level string) { leveler := &slog.LevelVar{} leveler.Set(programLevel) - h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ + h := slog.NewJSONHandler(sink, &slog.HandlerOptions{ AddSource: true, Level: leveler, }) - slog.SetDefault(slog.New(h)) + result := slog.New(h) + return result } func GetRequestLogger(base *slog.Logger, r *http.Request) *slog.Logger { @@ -39,8 +41,7 @@ func GetRequestLogger(base *slog.Logger, r *http.Request) *slog.Logger { "user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), - "x-forwarded-for", - r.Header.Get("X-Forwarded-For"), + "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"), ) } diff --git a/internal/ogtags/cache_test.go b/internal/ogtags/cache_test.go index 89ba2299d..9f2c1243b 100644 --- a/internal/ogtags/cache_test.go +++ b/internal/ogtags/cache_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store" "github.com/TecharoHQ/anubis/lib/store/memory" ) diff --git a/internal/ogtags/fetch_test.go b/internal/ogtags/fetch_test.go index 864e8f2b3..402cb1843 100644 --- a/internal/ogtags/fetch_test.go +++ b/internal/ogtags/fetch_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/memory" "golang.org/x/net/html" ) diff --git a/internal/ogtags/integration_test.go b/internal/ogtags/integration_test.go index af56668b4..f13f75f42 100644 --- a/internal/ogtags/integration_test.go +++ b/internal/ogtags/integration_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/memory" ) diff --git a/internal/ogtags/mem_test.go b/internal/ogtags/mem_test.go index 7d2ac0cb6..3770a73ff 100644 --- a/internal/ogtags/mem_test.go +++ b/internal/ogtags/mem_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/memory" "golang.org/x/net/html" ) diff --git a/internal/ogtags/ogtags.go b/internal/ogtags/ogtags.go index f0c0adf31..62cd89c2f 100644 --- a/internal/ogtags/ogtags.go +++ b/internal/ogtags/ogtags.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store" ) diff --git a/internal/ogtags/ogtags_fuzz_test.go b/internal/ogtags/ogtags_fuzz_test.go index 499d9f525..6355eebf6 100644 --- a/internal/ogtags/ogtags_fuzz_test.go +++ b/internal/ogtags/ogtags_fuzz_test.go @@ -7,7 +7,7 @@ import ( "testing" "unicode/utf8" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/memory" "golang.org/x/net/html" ) diff --git a/internal/ogtags/ogtags_test.go b/internal/ogtags/ogtags_test.go index 7441119c1..2b931a814 100644 --- a/internal/ogtags/ogtags_test.go +++ b/internal/ogtags/ogtags_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/memory" ) diff --git a/internal/ogtags/parse_test.go b/internal/ogtags/parse_test.go index 55e536a32..2c92cbfb8 100644 --- a/internal/ogtags/parse_test.go +++ b/internal/ogtags/parse_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/memory" "golang.org/x/net/html" ) diff --git a/internal/test/playwright_test.go b/internal/test/playwright_test.go index 0cb72e98e..b1cab340a 100644 --- a/internal/test/playwright_test.go +++ b/internal/test/playwright_test.go @@ -595,7 +595,7 @@ func spawnAnubisWithOptions(t *testing.T, basePrefix string) string { fmt.Fprintf(w, "%d", time.Now().Unix()) }) - policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty) + policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty, "info") if err != nil { t.Fatal(err) } diff --git a/lib/anubis.go b/lib/anubis.go index 68dd9bc50..4fc154ede 100644 --- a/lib/anubis.go +++ b/lib/anubis.go @@ -27,10 +27,10 @@ import ( "github.com/TecharoHQ/anubis/internal/dnsbl" "github.com/TecharoHQ/anubis/internal/ogtags" "github.com/TecharoHQ/anubis/lib/challenge" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/localization" "github.com/TecharoHQ/anubis/lib/policy" "github.com/TecharoHQ/anubis/lib/policy/checker" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/lib/store" // challenge implementations diff --git a/lib/anubis_test.go b/lib/anubis_test.go index b05196fcf..78e602dff 100644 --- a/lib/anubis_test.go +++ b/lib/anubis_test.go @@ -20,8 +20,8 @@ import ( "github.com/TecharoHQ/anubis/data" "github.com/TecharoHQ/anubis/internal" "github.com/TecharoHQ/anubis/lib/challenge" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/lib/store" "github.com/TecharoHQ/anubis/lib/thoth/thothmock" ) @@ -58,7 +58,7 @@ func loadPolicies(t *testing.T, fname string, difficulty int) *policy.ParsedConf t.Logf("loading policy file: %s", fname) - anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty) + anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty, "info") if err != nil { t.Fatal(err) } @@ -250,7 +250,7 @@ func TestLoadPolicies(t *testing.T) { } defer fin.Close() - if _, err := policy.ParseConfig(t.Context(), fin, fname, 4); err != nil { + if _, err := policy.ParseConfig(t.Context(), fin, fname, 4, "info"); err != nil { t.Fatal(err) } }) diff --git a/lib/challenge/interface.go b/lib/challenge/interface.go index c7a19449c..4bef2e7f0 100644 --- a/lib/challenge/interface.go +++ b/lib/challenge/interface.go @@ -6,8 +6,8 @@ import ( "sort" "sync" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/lib/store" "github.com/a-h/templ" ) diff --git a/lib/challenge/proofofwork/proofofwork_test.go b/lib/challenge/proofofwork/proofofwork_test.go index c0611a531..6b12afeba 100644 --- a/lib/challenge/proofofwork/proofofwork_test.go +++ b/lib/challenge/proofofwork/proofofwork_test.go @@ -8,8 +8,8 @@ import ( "testing" "github.com/TecharoHQ/anubis/lib/challenge" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy" - "github.com/TecharoHQ/anubis/lib/policy/config" ) func mkRequest(t *testing.T, values map[string]string) *http.Request { diff --git a/lib/config.go b/lib/config.go index 1c4d7a007..2d8ac18b9 100644 --- a/lib/config.go +++ b/lib/config.go @@ -18,9 +18,9 @@ import ( "github.com/TecharoHQ/anubis/internal" "github.com/TecharoHQ/anubis/internal/ogtags" "github.com/TecharoHQ/anubis/lib/challenge" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/localization" "github.com/TecharoHQ/anubis/lib/policy" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/web" "github.com/TecharoHQ/anubis/xess" "github.com/a-h/templ" @@ -48,12 +48,13 @@ type Options struct { CookieSecure bool CookieSameSite http.SameSite Logger *slog.Logger + LogLevel string PublicUrl string JWTRestrictionHeader string DifficultyInJWT bool } -func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { +func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int, logLevel string) (*policy.ParsedConfig, error) { var fin io.ReadCloser var err error @@ -77,7 +78,7 @@ func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty } }(fin) - anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty) + anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty, logLevel) if err != nil { return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err) } diff --git a/lib/policy/config/asn.go b/lib/config/asn.go similarity index 100% rename from lib/policy/config/asn.go rename to lib/config/asn.go diff --git a/lib/policy/config/asn_test.go b/lib/config/asn_test.go similarity index 100% rename from lib/policy/config/asn_test.go rename to lib/config/asn_test.go diff --git a/lib/policy/config/check.go b/lib/config/check.go similarity index 100% rename from lib/policy/config/check.go rename to lib/config/check.go diff --git a/lib/policy/config/config.go b/lib/config/config.go similarity index 98% rename from lib/policy/config/config.go rename to lib/config/config.go index 577470a26..3e45b7af0 100644 --- a/lib/policy/config/config.go +++ b/lib/config/config.go @@ -332,6 +332,7 @@ type fileConfig struct { Thresholds []Threshold `json:"thresholds"` StatusCodes StatusCodes `json:"status_codes"` DNSBL bool `json:"dnsbl"` + Logging *Logging `json:"logging"` } func (c *fileConfig) Valid() error { @@ -363,6 +364,10 @@ func (c *fileConfig) Valid() error { } } + if err := c.Logging.Valid(); err != nil { + errs = append(errs, err) + } + if c.Store != nil { if err := c.Store.Valid(); err != nil { errs = append(errs, err) @@ -385,6 +390,7 @@ func Load(fin io.Reader, fname string) (*Config, error) { Store: &Store{ Backend: "memory", }, + Logging: (Logging{}).Default(), } if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil { @@ -404,6 +410,7 @@ func Load(fin io.Reader, fname string) (*Config, error) { }, StatusCodes: c.StatusCodes, Store: c.Store, + Logging: c.Logging, } if c.OpenGraph.TimeToLive != "" { @@ -469,6 +476,7 @@ type Config struct { Bots []BotConfig Thresholds []Threshold StatusCodes StatusCodes + Logging *Logging DNSBL bool } diff --git a/lib/policy/config/config_test.go b/lib/config/config_test.go similarity index 99% rename from lib/policy/config/config_test.go rename to lib/config/config_test.go index c35ceefb1..8702780b2 100644 --- a/lib/policy/config/config_test.go +++ b/lib/config/config_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/TecharoHQ/anubis/data" - . "github.com/TecharoHQ/anubis/lib/policy/config" + . "github.com/TecharoHQ/anubis/lib/config" ) func p[V any](v V) *V { return &v } diff --git a/lib/policy/config/expressionorlist.go b/lib/config/expressionorlist.go similarity index 100% rename from lib/policy/config/expressionorlist.go rename to lib/config/expressionorlist.go diff --git a/lib/policy/config/expressionorlist_test.go b/lib/config/expressionorlist_test.go similarity index 100% rename from lib/policy/config/expressionorlist_test.go rename to lib/config/expressionorlist_test.go diff --git a/lib/policy/config/geoip.go b/lib/config/geoip.go similarity index 100% rename from lib/policy/config/geoip.go rename to lib/config/geoip.go diff --git a/lib/policy/config/geoip_test.go b/lib/config/geoip_test.go similarity index 100% rename from lib/policy/config/geoip_test.go rename to lib/config/geoip_test.go diff --git a/lib/policy/config/impressum.go b/lib/config/impressum.go similarity index 100% rename from lib/policy/config/impressum.go rename to lib/config/impressum.go diff --git a/lib/policy/config/impressum_test.go b/lib/config/impressum_test.go similarity index 100% rename from lib/policy/config/impressum_test.go rename to lib/config/impressum_test.go diff --git a/lib/config/logging.go b/lib/config/logging.go new file mode 100644 index 000000000..b16a4ca56 --- /dev/null +++ b/lib/config/logging.go @@ -0,0 +1,124 @@ +package config + +import ( + "errors" + "fmt" + "log/slog" +) + +var ( + ErrMissingLoggingFileConfig = errors.New("config.Logging: missing value parameters in logging block") + ErrInvalidLoggingSink = errors.New("config.Logging: invalid sink") + ErrInvalidLoggingFileConfig = errors.New("config.LoggingFileConfig: invalid parameters") + ErrOutOfRange = errors.New("config: error out of range") +) + +type Logging struct { + Sink string `json:"sink"` // Logging sink, either "stdio" or "file" + Level *slog.Level `json:"level"` // Log level, if set supercedes the level in flags + Parameters *LoggingFileConfig `json:"parameters"` // Logging parameters, to be dynamic in the future +} + +const ( + LogSinkStdio = "stdio" + LogSinkFile = "file" +) + +func (l *Logging) Valid() error { + var errs []error + + switch l.Sink { + case LogSinkStdio: + // no validation needed + case LogSinkFile: + if l.Parameters == nil { + errs = append(errs, ErrMissingLoggingFileConfig) + } + + if err := l.Parameters.Valid(); err != nil { + errs = append(errs, err) + } + default: + errs = append(errs, fmt.Errorf("%w: sink %s is unknown to me", ErrInvalidLoggingSink, l.Sink)) + } + + if len(errs) != 0 { + return errors.Join(errs...) + } + + return nil +} + +func (Logging) Default() *Logging { + return &Logging{ + Sink: "stdio", + } +} + +type LoggingFileConfig struct { + Filename string `json:"file"` + MaxBackups int `json:"maxBackups"` + MaxBytes int64 `json:"maxBytes"` + MaxAge int `json:"maxAge"` + Compress bool `json:"compress"` + UseLocalTime bool `json:"useLocalTime"` +} + +func (lfc *LoggingFileConfig) Valid() error { + if lfc == nil { + return fmt.Errorf("logging file config is nil, why are you calling this?") + } + + var errs []error + + if lfc.Zero() { + errs = append(errs, ErrMissingValue) + } + + if lfc.Filename == "" { + errs = append(errs, fmt.Errorf("%w: filename", ErrMissingValue)) + } + + if lfc.MaxBackups < 0 { + errs = append(errs, fmt.Errorf("%w: max backup count %d is not greater than or equal to zero", ErrOutOfRange, lfc.MaxBackups)) + } + + if lfc.MaxAge < 0 { + errs = append(errs, fmt.Errorf("%w: max backup count %d is not greater than or equal to zero", ErrOutOfRange, lfc.MaxAge)) + } + + if len(errs) != 0 { + errs = append([]error{ErrInvalidLoggingFileConfig}, errs...) + return errors.Join(errs...) + } + + return nil +} + +func (lfc LoggingFileConfig) Zero() bool { + for _, cond := range []bool{ + lfc.Filename != "", + lfc.MaxBackups != 0, + lfc.MaxBytes != 0, + lfc.MaxAge != 0, + lfc.Compress, + lfc.UseLocalTime, + } { + if cond { + return false + } + } + + return true +} + +func (LoggingFileConfig) Default() *LoggingFileConfig { + return &LoggingFileConfig{ + Filename: "./var/anubis.log", + MaxBackups: 3, + MaxBytes: 104857600, // 100 Mi + MaxAge: 7, // 7 days + Compress: true, + UseLocalTime: false, + } +} diff --git a/lib/config/logging_test.go b/lib/config/logging_test.go new file mode 100644 index 000000000..c850cbd9b --- /dev/null +++ b/lib/config/logging_test.go @@ -0,0 +1,103 @@ +package config + +import ( + "errors" + "testing" +) + +func TestLoggingValid(t *testing.T) { + for _, tt := range []struct { + name string + input *Logging + want error + }{ + { + name: "simple happy", + input: (Logging{}).Default(), + }, + { + name: "default file config", + input: &Logging{ + Sink: LogSinkFile, + Parameters: (&LoggingFileConfig{}).Default(), + }, + }, + { + name: "invalid sink", + input: &Logging{ + Sink: "taco invalid", + }, + want: ErrInvalidLoggingSink, + }, + { + name: "missing parameters", + input: &Logging{ + Sink: LogSinkFile, + }, + want: ErrMissingLoggingFileConfig, + }, + { + name: "invalid parameters", + input: &Logging{ + Sink: LogSinkFile, + Parameters: &LoggingFileConfig{}, + }, + want: ErrInvalidLoggingFileConfig, + }, + { + name: "file sink with no filename", + input: &Logging{ + Sink: LogSinkFile, + Parameters: &LoggingFileConfig{ + Filename: "", + MaxBackups: 3, + MaxBytes: 104857600, // 100 Mi + MaxAge: 7, // 7 days + Compress: true, + UseLocalTime: false, + }, + }, + want: ErrMissingValue, + }, + { + name: "file sink with negative max backups", + input: &Logging{ + Sink: LogSinkFile, + Parameters: &LoggingFileConfig{ + Filename: "./var/anubis.log", + MaxBackups: -3, + MaxBytes: 104857600, // 100 Mi + MaxAge: 7, // 7 days + Compress: true, + UseLocalTime: false, + }, + }, + want: ErrOutOfRange, + }, + { + name: "file sink with negative max age", + input: &Logging{ + Sink: LogSinkFile, + Parameters: &LoggingFileConfig{ + Filename: "./var/anubis.log", + MaxBackups: 3, + MaxBytes: 104857600, // 100 Mi + MaxAge: -7, // 7 days + Compress: true, + UseLocalTime: false, + }, + }, + want: ErrOutOfRange, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := tt.input.Valid() + + if !errors.Is(err, tt.want) { + t.Logf("wanted error: %v", tt.want) + t.Logf(" got error: %v", err) + t.Fatal("got wrong error") + } + }) + } +} diff --git a/lib/policy/config/opengraph.go b/lib/config/opengraph.go similarity index 100% rename from lib/policy/config/opengraph.go rename to lib/config/opengraph.go diff --git a/lib/policy/config/opengraph_test.go b/lib/config/opengraph_test.go similarity index 100% rename from lib/policy/config/opengraph_test.go rename to lib/config/opengraph_test.go diff --git a/lib/policy/config/store.go b/lib/config/store.go similarity index 100% rename from lib/policy/config/store.go rename to lib/config/store.go diff --git a/lib/policy/config/store_test.go b/lib/config/store_test.go similarity index 97% rename from lib/policy/config/store_test.go rename to lib/config/store_test.go index 227b06bf5..407268543 100644 --- a/lib/policy/config/store_test.go +++ b/lib/config/store_test.go @@ -5,7 +5,7 @@ import ( "errors" "testing" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/store/bbolt" "github.com/TecharoHQ/anubis/lib/store/valkey" ) diff --git a/lib/policy/config/testdata/bad/badregexes.json b/lib/config/testdata/bad/badregexes.json similarity index 100% rename from lib/policy/config/testdata/bad/badregexes.json rename to lib/config/testdata/bad/badregexes.json diff --git a/lib/policy/config/testdata/bad/badregexes.yaml b/lib/config/testdata/bad/badregexes.yaml similarity index 100% rename from lib/policy/config/testdata/bad/badregexes.yaml rename to lib/config/testdata/bad/badregexes.yaml diff --git a/lib/policy/config/testdata/bad/import_and_bot.json b/lib/config/testdata/bad/import_and_bot.json similarity index 100% rename from lib/policy/config/testdata/bad/import_and_bot.json rename to lib/config/testdata/bad/import_and_bot.json diff --git a/lib/policy/config/testdata/bad/import_and_bot.yaml b/lib/config/testdata/bad/import_and_bot.yaml similarity index 100% rename from lib/policy/config/testdata/bad/import_and_bot.yaml rename to lib/config/testdata/bad/import_and_bot.yaml diff --git a/lib/policy/config/testdata/bad/import_invalid_file.json b/lib/config/testdata/bad/import_invalid_file.json similarity index 100% rename from lib/policy/config/testdata/bad/import_invalid_file.json rename to lib/config/testdata/bad/import_invalid_file.json diff --git a/lib/policy/config/testdata/bad/import_invalid_file.yaml b/lib/config/testdata/bad/import_invalid_file.yaml similarity index 100% rename from lib/policy/config/testdata/bad/import_invalid_file.yaml rename to lib/config/testdata/bad/import_invalid_file.yaml diff --git a/lib/policy/config/testdata/bad/impressum-no-footer.yaml b/lib/config/testdata/bad/impressum-no-footer.yaml similarity index 100% rename from lib/policy/config/testdata/bad/impressum-no-footer.yaml rename to lib/config/testdata/bad/impressum-no-footer.yaml diff --git a/lib/policy/config/testdata/bad/impressum-no-page-contents.yaml b/lib/config/testdata/bad/impressum-no-page-contents.yaml similarity index 100% rename from lib/policy/config/testdata/bad/impressum-no-page-contents.yaml rename to lib/config/testdata/bad/impressum-no-page-contents.yaml diff --git a/lib/policy/config/testdata/bad/invalid.json b/lib/config/testdata/bad/invalid.json similarity index 100% rename from lib/policy/config/testdata/bad/invalid.json rename to lib/config/testdata/bad/invalid.json diff --git a/lib/policy/config/testdata/bad/invalid.yaml b/lib/config/testdata/bad/invalid.yaml similarity index 100% rename from lib/policy/config/testdata/bad/invalid.yaml rename to lib/config/testdata/bad/invalid.yaml diff --git a/lib/config/testdata/bad/logging-invalid-sink.yaml b/lib/config/testdata/bad/logging-invalid-sink.yaml new file mode 100644 index 000000000..eadde4c36 --- /dev/null +++ b/lib/config/testdata/bad/logging-invalid-sink.yaml @@ -0,0 +1,2 @@ +logging: + sink: "nope" diff --git a/lib/config/testdata/bad/logging-no-parameters.yaml b/lib/config/testdata/bad/logging-no-parameters.yaml new file mode 100644 index 000000000..0cfea0314 --- /dev/null +++ b/lib/config/testdata/bad/logging-no-parameters.yaml @@ -0,0 +1,2 @@ +logging: + sink: "file" diff --git a/lib/policy/config/testdata/bad/multiple_expression_types.json b/lib/config/testdata/bad/multiple_expression_types.json similarity index 100% rename from lib/policy/config/testdata/bad/multiple_expression_types.json rename to lib/config/testdata/bad/multiple_expression_types.json diff --git a/lib/policy/config/testdata/bad/multiple_expression_types.yaml b/lib/config/testdata/bad/multiple_expression_types.yaml similarity index 100% rename from lib/policy/config/testdata/bad/multiple_expression_types.yaml rename to lib/config/testdata/bad/multiple_expression_types.yaml diff --git a/lib/policy/config/testdata/bad/nobots.json b/lib/config/testdata/bad/nobots.json similarity index 100% rename from lib/policy/config/testdata/bad/nobots.json rename to lib/config/testdata/bad/nobots.json diff --git a/lib/policy/config/testdata/bad/nobots.yaml b/lib/config/testdata/bad/nobots.yaml similarity index 100% rename from lib/policy/config/testdata/bad/nobots.yaml rename to lib/config/testdata/bad/nobots.yaml diff --git a/lib/policy/config/testdata/bad/opengraph_bad_ttl.yaml b/lib/config/testdata/bad/opengraph_bad_ttl.yaml similarity index 100% rename from lib/policy/config/testdata/bad/opengraph_bad_ttl.yaml rename to lib/config/testdata/bad/opengraph_bad_ttl.yaml diff --git a/lib/policy/config/testdata/bad/regex_ends_newline.json b/lib/config/testdata/bad/regex_ends_newline.json similarity index 100% rename from lib/policy/config/testdata/bad/regex_ends_newline.json rename to lib/config/testdata/bad/regex_ends_newline.json diff --git a/lib/policy/config/testdata/bad/regex_ends_newline.yaml b/lib/config/testdata/bad/regex_ends_newline.yaml similarity index 100% rename from lib/policy/config/testdata/bad/regex_ends_newline.yaml rename to lib/config/testdata/bad/regex_ends_newline.yaml diff --git a/lib/policy/config/testdata/bad/status-codes-0.json b/lib/config/testdata/bad/status-codes-0.json similarity index 100% rename from lib/policy/config/testdata/bad/status-codes-0.json rename to lib/config/testdata/bad/status-codes-0.json diff --git a/lib/policy/config/testdata/bad/status-codes-0.yaml b/lib/config/testdata/bad/status-codes-0.yaml similarity index 100% rename from lib/policy/config/testdata/bad/status-codes-0.yaml rename to lib/config/testdata/bad/status-codes-0.yaml diff --git a/lib/policy/config/testdata/bad/threshold-challenge-without-challenge.yaml b/lib/config/testdata/bad/threshold-challenge-without-challenge.yaml similarity index 100% rename from lib/policy/config/testdata/bad/threshold-challenge-without-challenge.yaml rename to lib/config/testdata/bad/threshold-challenge-without-challenge.yaml diff --git a/lib/policy/config/testdata/bad/thresholds.yaml b/lib/config/testdata/bad/thresholds.yaml similarity index 100% rename from lib/policy/config/testdata/bad/thresholds.yaml rename to lib/config/testdata/bad/thresholds.yaml diff --git a/lib/policy/config/testdata/bad/unparseable.json b/lib/config/testdata/bad/unparseable.json similarity index 100% rename from lib/policy/config/testdata/bad/unparseable.json rename to lib/config/testdata/bad/unparseable.json diff --git a/lib/policy/config/testdata/bad/unparseable.yaml b/lib/config/testdata/bad/unparseable.yaml similarity index 100% rename from lib/policy/config/testdata/bad/unparseable.yaml rename to lib/config/testdata/bad/unparseable.yaml diff --git a/lib/policy/config/testdata/good/allow_everyone.json b/lib/config/testdata/good/allow_everyone.json similarity index 100% rename from lib/policy/config/testdata/good/allow_everyone.json rename to lib/config/testdata/good/allow_everyone.json diff --git a/lib/policy/config/testdata/good/allow_everyone.yaml b/lib/config/testdata/good/allow_everyone.yaml similarity index 100% rename from lib/policy/config/testdata/good/allow_everyone.yaml rename to lib/config/testdata/good/allow_everyone.yaml diff --git a/lib/policy/config/testdata/good/block_cf_workers.json b/lib/config/testdata/good/block_cf_workers.json similarity index 100% rename from lib/policy/config/testdata/good/block_cf_workers.json rename to lib/config/testdata/good/block_cf_workers.json diff --git a/lib/policy/config/testdata/good/block_cf_workers.yaml b/lib/config/testdata/good/block_cf_workers.yaml similarity index 100% rename from lib/policy/config/testdata/good/block_cf_workers.yaml rename to lib/config/testdata/good/block_cf_workers.yaml diff --git a/lib/policy/config/testdata/good/challenge_cloudflare.yaml b/lib/config/testdata/good/challenge_cloudflare.yaml similarity index 100% rename from lib/policy/config/testdata/good/challenge_cloudflare.yaml rename to lib/config/testdata/good/challenge_cloudflare.yaml diff --git a/lib/policy/config/testdata/good/challengemozilla.json b/lib/config/testdata/good/challengemozilla.json similarity index 100% rename from lib/policy/config/testdata/good/challengemozilla.json rename to lib/config/testdata/good/challengemozilla.json diff --git a/lib/policy/config/testdata/good/challengemozilla.yaml b/lib/config/testdata/good/challengemozilla.yaml similarity index 100% rename from lib/policy/config/testdata/good/challengemozilla.yaml rename to lib/config/testdata/good/challengemozilla.yaml diff --git a/lib/policy/config/testdata/good/entropy.yaml b/lib/config/testdata/good/entropy.yaml similarity index 100% rename from lib/policy/config/testdata/good/entropy.yaml rename to lib/config/testdata/good/entropy.yaml diff --git a/lib/policy/config/testdata/good/everything_blocked.json b/lib/config/testdata/good/everything_blocked.json similarity index 100% rename from lib/policy/config/testdata/good/everything_blocked.json rename to lib/config/testdata/good/everything_blocked.json diff --git a/lib/policy/config/testdata/good/everything_blocked.yaml b/lib/config/testdata/good/everything_blocked.yaml similarity index 100% rename from lib/policy/config/testdata/good/everything_blocked.yaml rename to lib/config/testdata/good/everything_blocked.yaml diff --git a/lib/policy/config/testdata/good/geoip_us.yaml b/lib/config/testdata/good/geoip_us.yaml similarity index 100% rename from lib/policy/config/testdata/good/geoip_us.yaml rename to lib/config/testdata/good/geoip_us.yaml diff --git a/lib/policy/config/testdata/good/git_client.json b/lib/config/testdata/good/git_client.json similarity index 100% rename from lib/policy/config/testdata/good/git_client.json rename to lib/config/testdata/good/git_client.json diff --git a/lib/policy/config/testdata/good/git_client.yaml b/lib/config/testdata/good/git_client.yaml similarity index 100% rename from lib/policy/config/testdata/good/git_client.yaml rename to lib/config/testdata/good/git_client.yaml diff --git a/lib/policy/config/testdata/good/import_filesystem.json b/lib/config/testdata/good/import_filesystem.json similarity index 100% rename from lib/policy/config/testdata/good/import_filesystem.json rename to lib/config/testdata/good/import_filesystem.json diff --git a/lib/policy/config/testdata/good/import_filesystem.yaml b/lib/config/testdata/good/import_filesystem.yaml similarity index 100% rename from lib/policy/config/testdata/good/import_filesystem.yaml rename to lib/config/testdata/good/import_filesystem.yaml diff --git a/lib/policy/config/testdata/good/import_keep_internet_working.json b/lib/config/testdata/good/import_keep_internet_working.json similarity index 100% rename from lib/policy/config/testdata/good/import_keep_internet_working.json rename to lib/config/testdata/good/import_keep_internet_working.json diff --git a/lib/policy/config/testdata/good/import_keep_internet_working.yaml b/lib/config/testdata/good/import_keep_internet_working.yaml similarity index 100% rename from lib/policy/config/testdata/good/import_keep_internet_working.yaml rename to lib/config/testdata/good/import_keep_internet_working.yaml diff --git a/lib/policy/config/testdata/good/impressum.yaml b/lib/config/testdata/good/impressum.yaml similarity index 100% rename from lib/policy/config/testdata/good/impressum.yaml rename to lib/config/testdata/good/impressum.yaml diff --git a/lib/config/testdata/good/logging-file.yaml b/lib/config/testdata/good/logging-file.yaml new file mode 100644 index 000000000..c1f09b36f --- /dev/null +++ b/lib/config/testdata/good/logging-file.yaml @@ -0,0 +1,15 @@ +bots: + - name: simple + action: CHALLENGE + user_agent_regex: Mozilla + +logs: + sink: "file" + parameters: + file: "/var/log/botstopper/default.log" + maxBackups: 3 # keep at least 3 old copies + maxBytes: 67108864 # each file can have up to 64 MB of logs + maxAge: 7 # rotate files out every n days + oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish + compress: true + useLocalTime: false # timezone for rotated files is UTC diff --git a/lib/config/testdata/good/logging-stdio.yaml b/lib/config/testdata/good/logging-stdio.yaml new file mode 100644 index 000000000..d0fcf54b8 --- /dev/null +++ b/lib/config/testdata/good/logging-stdio.yaml @@ -0,0 +1,7 @@ +bots: + - name: simple + action: CHALLENGE + user_agent_regex: Mozilla + +logging: + sink: "stdio" diff --git a/lib/policy/config/testdata/good/no-thresholds.yaml b/lib/config/testdata/good/no-thresholds.yaml similarity index 100% rename from lib/policy/config/testdata/good/no-thresholds.yaml rename to lib/config/testdata/good/no-thresholds.yaml diff --git a/lib/policy/config/testdata/good/old_xesite.json b/lib/config/testdata/good/old_xesite.json similarity index 100% rename from lib/policy/config/testdata/good/old_xesite.json rename to lib/config/testdata/good/old_xesite.json diff --git a/lib/policy/config/testdata/good/opengraph_all_good.yaml b/lib/config/testdata/good/opengraph_all_good.yaml similarity index 100% rename from lib/policy/config/testdata/good/opengraph_all_good.yaml rename to lib/config/testdata/good/opengraph_all_good.yaml diff --git a/lib/policy/config/testdata/good/simple-weight.yaml b/lib/config/testdata/good/simple-weight.yaml similarity index 100% rename from lib/policy/config/testdata/good/simple-weight.yaml rename to lib/config/testdata/good/simple-weight.yaml diff --git a/lib/policy/config/testdata/good/status-codes-paranoid.json b/lib/config/testdata/good/status-codes-paranoid.json similarity index 100% rename from lib/policy/config/testdata/good/status-codes-paranoid.json rename to lib/config/testdata/good/status-codes-paranoid.json diff --git a/lib/policy/config/testdata/good/status-codes-paranoid.yaml b/lib/config/testdata/good/status-codes-paranoid.yaml similarity index 100% rename from lib/policy/config/testdata/good/status-codes-paranoid.yaml rename to lib/config/testdata/good/status-codes-paranoid.yaml diff --git a/lib/policy/config/testdata/good/status-codes-rfc.json b/lib/config/testdata/good/status-codes-rfc.json similarity index 100% rename from lib/policy/config/testdata/good/status-codes-rfc.json rename to lib/config/testdata/good/status-codes-rfc.json diff --git a/lib/policy/config/testdata/good/status-codes-rfc.yaml b/lib/config/testdata/good/status-codes-rfc.yaml similarity index 100% rename from lib/policy/config/testdata/good/status-codes-rfc.yaml rename to lib/config/testdata/good/status-codes-rfc.yaml diff --git a/lib/policy/config/testdata/good/thresholds.yaml b/lib/config/testdata/good/thresholds.yaml similarity index 100% rename from lib/policy/config/testdata/good/thresholds.yaml rename to lib/config/testdata/good/thresholds.yaml diff --git a/lib/policy/config/testdata/good/weight-no-weight.yaml b/lib/config/testdata/good/weight-no-weight.yaml similarity index 100% rename from lib/policy/config/testdata/good/weight-no-weight.yaml rename to lib/config/testdata/good/weight-no-weight.yaml diff --git a/lib/policy/config/testdata/hack-test.json b/lib/config/testdata/hack-test.json similarity index 100% rename from lib/policy/config/testdata/hack-test.json rename to lib/config/testdata/hack-test.json diff --git a/lib/policy/config/testdata/hack-test.yaml b/lib/config/testdata/hack-test.yaml similarity index 100% rename from lib/policy/config/testdata/hack-test.yaml rename to lib/config/testdata/hack-test.yaml diff --git a/lib/policy/config/threshold.go b/lib/config/threshold.go similarity index 100% rename from lib/policy/config/threshold.go rename to lib/config/threshold.go diff --git a/lib/policy/config/threshold_test.go b/lib/config/threshold_test.go similarity index 100% rename from lib/policy/config/threshold_test.go rename to lib/config/threshold_test.go diff --git a/lib/policy/config/weight.go b/lib/config/weight.go similarity index 100% rename from lib/policy/config/weight.go rename to lib/config/weight.go diff --git a/lib/config_test.go b/lib/config_test.go index af31ca6b2..5d392dbb3 100644 --- a/lib/config_test.go +++ b/lib/config_test.go @@ -12,13 +12,13 @@ import ( ) func TestInvalidChallengeMethod(t *testing.T) { - if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) { + if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4, "info"); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) { t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err) } } func TestBadConfigs(t *testing.T) { - finfos, err := os.ReadDir("policy/config/testdata/bad") + finfos, err := os.ReadDir("config/testdata/bad") if err != nil { t.Fatal(err) } @@ -26,7 +26,7 @@ func TestBadConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty); err == nil { + if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty, "info"); err == nil { t.Fatal(err) } else { t.Log(err) @@ -36,7 +36,7 @@ func TestBadConfigs(t *testing.T) { } func TestGoodConfigs(t *testing.T) { - finfos, err := os.ReadDir("policy/config/testdata/good") + finfos, err := os.ReadDir("config/testdata/good") if err != nil { t.Fatal(err) } @@ -46,13 +46,13 @@ func TestGoodConfigs(t *testing.T) { t.Run(st.Name(), func(t *testing.T) { t.Run("with-thoth", func(t *testing.T) { ctx := thothmock.WithMockThoth(t) - if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { + if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil { t.Fatal(err) } }) t.Run("without-thoth", func(t *testing.T) { - if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { + if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil { t.Fatal(err) } }) diff --git a/lib/policy/bot.go b/lib/policy/bot.go index 479bccc3a..25c04870f 100644 --- a/lib/policy/bot.go +++ b/lib/policy/bot.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy/checker" - "github.com/TecharoHQ/anubis/lib/policy/config" ) type Bot struct { diff --git a/lib/policy/celchecker.go b/lib/policy/celchecker.go index 2b06392a9..033bcfeaf 100644 --- a/lib/policy/celchecker.go +++ b/lib/policy/celchecker.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/TecharoHQ/anubis/internal" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy/expressions" "github.com/google/cel-go/cel" "github.com/google/cel-go/common/types" diff --git a/lib/policy/checkresult.go b/lib/policy/checkresult.go index 31737dda5..c75ccb36a 100644 --- a/lib/policy/checkresult.go +++ b/lib/policy/checkresult.go @@ -3,7 +3,7 @@ package policy import ( "log/slog" - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" ) type CheckResult struct { diff --git a/lib/policy/policy.go b/lib/policy/policy.go index 27bd66617..8f6dfaa35 100644 --- a/lib/policy/policy.go +++ b/lib/policy/policy.go @@ -6,12 +6,16 @@ import ( "fmt" "io" "log/slog" + "os" "sync/atomic" + "time" + "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy/checker" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/lib/store" "github.com/TecharoHQ/anubis/lib/thoth" + "github.com/fahedouch/go-logrotate" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -38,6 +42,7 @@ type ParsedConfig struct { StatusCodes config.StatusCodes DefaultDifficulty int DNSBL bool + Logger *slog.Logger } func newParsedConfig(orig *config.Config) *ParsedConfig { @@ -48,7 +53,7 @@ func newParsedConfig(orig *config.Config) *ParsedConfig { } } -func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { +func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int, logLevel string) (*ParsedConfig, error) { c, err := config.Load(fin, fname) if err != nil { return nil, err @@ -202,6 +207,27 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic validationErrs = append(validationErrs, config.ErrUnknownStoreBackend) } + if c.Logging.Level != nil { + logLevel = c.Logging.Level.String() + } + + switch c.Logging.Sink { + case config.LogSinkStdio: + result.Logger = internal.InitSlog(logLevel, os.Stderr) + case config.LogSinkFile: + out := &logrotate.Logger{ + Filename: c.Logging.Parameters.Filename, + FilenameTimeFormat: time.RFC3339, + MaxBytes: c.Logging.Parameters.MaxBytes, + MaxAge: c.Logging.Parameters.MaxAge, + MaxBackups: c.Logging.Parameters.MaxBackups, + LocalTime: c.Logging.Parameters.UseLocalTime, + Compress: c.Logging.Parameters.Compress, + } + + result.Logger = internal.InitSlog(logLevel, out) + } + if len(validationErrs) > 0 { return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, errors.Join(validationErrs...)) } diff --git a/lib/policy/policy_test.go b/lib/policy/policy_test.go index 103728ac0..56ce37315 100644 --- a/lib/policy/policy_test.go +++ b/lib/policy/policy_test.go @@ -19,14 +19,14 @@ func TestDefaultPolicyMustParse(t *testing.T) { } defer fin.Close() - if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty); err != nil { + if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty, "info"); err != nil { t.Fatalf("can't parse config: %v", err) } } func TestGoodConfigs(t *testing.T) { - finfos, err := os.ReadDir("config/testdata/good") + finfos, err := os.ReadDir("../config/testdata/good") if err != nil { t.Fatal(err) } @@ -35,26 +35,26 @@ func TestGoodConfigs(t *testing.T) { st := st t.Run(st.Name(), func(t *testing.T) { t.Run("with-thoth", func(t *testing.T) { - fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) + fin, err := os.Open(filepath.Join("..", "config", "testdata", "good", st.Name())) if err != nil { t.Fatal(err) } defer fin.Close() ctx := thothmock.WithMockThoth(t) - if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err != nil { + if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil { t.Fatal(err) } }) t.Run("without-thoth", func(t *testing.T) { - fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) + fin, err := os.Open(filepath.Join("..", "config", "testdata", "good", st.Name())) if err != nil { t.Fatal(err) } defer fin.Close() - if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty); err != nil { + if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil { t.Fatal(err) } }) @@ -65,7 +65,7 @@ func TestGoodConfigs(t *testing.T) { func TestBadConfigs(t *testing.T) { ctx := thothmock.WithMockThoth(t) - finfos, err := os.ReadDir("config/testdata/bad") + finfos, err := os.ReadDir("../config/testdata/bad") if err != nil { t.Fatal(err) } @@ -73,13 +73,13 @@ func TestBadConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - fin, err := os.Open(filepath.Join("config", "testdata", "bad", st.Name())) + fin, err := os.Open(filepath.Join("..", "config", "testdata", "bad", st.Name())) if err != nil { t.Fatal(err) } defer fin.Close() - if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err == nil { + if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err == nil { t.Fatal(err) } else { t.Log(err) diff --git a/lib/policy/thresholds.go b/lib/policy/thresholds.go index 1f77f6311..7e79ac51f 100644 --- a/lib/policy/thresholds.go +++ b/lib/policy/thresholds.go @@ -1,7 +1,7 @@ package policy import ( - "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/policy/expressions" "github.com/google/cel-go/cel" ) diff --git a/test/go.mod b/test/go.mod index ec4f37784..b10d48d59 100644 --- a/test/go.mod +++ b/test/go.mod @@ -42,11 +42,13 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/djherbis/times v1.6.0 // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.9.1 // indirect github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect + github.com/fahedouch/go-logrotate v0.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gaissmai/bart v0.26.0 // indirect github.com/go-logr/logr v1.4.3 // indirect diff --git a/test/go.sum b/test/go.sum index 0c97034b7..8cbcdbc06 100644 --- a/test/go.sum +++ b/test/go.sum @@ -82,6 +82,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= @@ -98,6 +100,8 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fahedouch/go-logrotate v0.3.0 h1:XP+dHIDgWZ1ckz43mG6gl5ASer3PZDVr755SVMyzaUQ= +github.com/fahedouch/go-logrotate v0.3.0/go.mod h1:X49m0bvPLkk71MHNCQ1yEfVEw8W/u+qvHa/hOnhCYf4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0= @@ -251,6 +255,7 @@ golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= @@ -271,6 +276,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/log-file/anubis.yaml b/test/log-file/anubis.yaml new file mode 100644 index 000000000..e8b6ca772 --- /dev/null +++ b/test/log-file/anubis.yaml @@ -0,0 +1,18 @@ +bots: + - name: challenge + user_agent_regex: CHALLENGE + action: CHALLENGE + +status_codes: + CHALLENGE: 200 + DENY: 403 + +logging: + sink: file + parameters: + file: "./var/anubis.log" + maxBackups: 3 # keep at least 3 old copies + maxBytes: 67108864 # each file can have up to 64 Mi of logs + maxAge: 7 # rotate files out every n days + compress: true + useLocalTime: false # timezone for rotated files is UTC diff --git a/test/log-file/input.txt b/test/log-file/input.txt new file mode 100644 index 000000000..639f5842b --- /dev/null +++ b/test/log-file/input.txt @@ -0,0 +1,178 @@ +/wiki//bin +/wiki//boot +/wiki//dev +/wiki//dev/de +/wiki//dev/en +/wiki//dev/en-ca +/wiki//dev/es +/wiki//dev/fr +/wiki//dev/hr +/wiki//dev/hu +/wiki//dev/it +/wiki//dev/ja +/wiki//dev/ko +/wiki//dev/pl +/wiki//dev/pt-br +/wiki//dev/ro +/wiki//dev/ru +/wiki//dev/sv +/wiki//dev/uk +/wiki//dev/zh-cn +/wiki//etc +/wiki//etc/conf.d +/wiki//etc/env.d +/wiki//etc/fstab +/wiki//etc/fstab/de +/wiki//etc/fstab/en +/wiki//etc/fstab/es +/wiki//etc/fstab/fr +/wiki//etc/fstab/hu +/wiki//etc/fstab/it +/wiki//etc/fstab/ja +/wiki//etc/fstab/ko +/wiki//etc/fstab/ru +/wiki//etc/fstab/sv +/wiki//etc/fstab/uk +/wiki//etc/fstab/zh-cn +/wiki//etc/hosts +/wiki//etc/local.d +/wiki//etc/make.conf +/wiki//etc/portage +/wiki//etc/portage/bashrc +/wiki//etc/portage/Bashrc +/wiki//etc/portage/binrepos.conf +/wiki//etc/portage/binrepos.conf/en +/wiki//etc/portage/binrepos.conf/hu +/wiki//etc/portage/binrepos.conf/ja +/wiki//etc/portage/binrepos.conf/ru +/wiki//etc/portage/categories +/wiki//etc/portage/color.map +/wiki//etc/portage/env +/wiki//etc/portage/img/ico.png +/wiki//etc/portage/license_groups +/wiki//etc/portage/make.conf +/wiki//etc/portage/make.conf/de +/wiki//etc/portage/make.conf/de/etc/portage/make.conf +/wiki//etc/portage/make.conf/en +/wiki//etc/portage/make.conf/es +/wiki//etc/portage/make.conf/fr +/wiki//etc/portage/make.conf/hu +/wiki//etc/portage/make.conf/it +/wiki//etc/portage/make.conf/it/var/db/repos/gentoo/licenses +/wiki//etc/portage/make.conf/ja +/wiki//etc/portage/make.conf/pl +/wiki//etc/portage/make.conf/ru +/wiki//etc/portage/make.conf/uk +/wiki//etc/portage/make.conf/zh-cn +/wiki//etc/portage/make.profile +/wiki//etc/portage/mirrors +/wiki//etc/portage/modules +/wiki//etc/portage/package.accept_keywords +/wiki//etc/portage/package.env +/wiki//etc/portage/package.license +/wiki//etc/portage/package.license/en +/wiki//etc/portage/package.license/es +/wiki//etc/portage/package.license/hu +/wiki//etc/portage/package.license/ja +/wiki//etc/portage/package.mask +/wiki//etc/portage/package.mask/en +/wiki//etc/portage/package.mask/hu +/wiki//etc/portage/package.mask/ja +/wiki//etc/portage/package.properties +/wiki//etc/portage/package.unmask +/wiki//etc/portage/package.use +/wiki//etc/portage/package.use/de +/wiki//etc/portage/package.use/en +/wiki//etc/portage/package.use/es +/wiki//etc/portage/package.use/fr +/wiki//etc/portage/package.use/hu +/wiki//etc/portage/package.use/it +/wiki//etc/portage/package.use/ja +/wiki//etc/portage/package.use/ru +/wiki//etc/portage/package.use/uk +/wiki//etc/portage/package.use/zh-cn +/wiki//etc/portage/patches +/wiki//etc/portage/profile/make.defaults +/wiki//etc/portage/profile/package.provided +/wiki//etc/portage/profile/package.provided/etc/portage/profile/package.provided +/wiki//etc/portage/profile/package.provided/etc/portage/profiles/package.provided +/wiki//etc/portage/profile/package.use.mask +/wiki//etc/portage/profiles/package.provided +/wiki//etc/portage/profiles/package.use.mask +/wiki//etc/portage/profiles/package.use.mask/etc/portage/profile/package.use.mask +/wiki//etc/portage/profiles/package.use.mask/etc/portage/profiles/package.use.mask +/wiki//etc/portage/profiles/use.mask +/wiki//etc/portage/profile/use.mask +/wiki//etc/portage/repos.conf +/wiki//etc/portage/repos.conf/brother-overlay.conf +/wiki//etc/portage/repos.conf/de +/wiki//etc/portage/repos.conf/en +/wiki//etc/portage/repos.conf/es +/wiki//etc/portage/repos.conf/etc/portage/repos.conf/gentoo.conf +/wiki//etc/portage/repos.conf/fr +/wiki//etc/portage/repos.conf/fr/etc/portage/repos.conf/gentoo.conf +/wiki//etc/portage/repos.conf/gentoo.conf +/wiki//etc/portage/repos.conf/gentoo.conf/etc/portage/repos.conf/gentoo.conf +/wiki//etc/portage/repos.conf/hr +/wiki//etc/portage/repos.conf/hu +/wiki//etc/portage/repos.conf/it +/wiki//etc/portage/repos.conf/ja +/wiki//etc/portage/repos.conf/ko +/wiki//etc/portage/repos.conf/pl +/wiki//etc/portage/repos.conf/pt-br +/wiki//etc/portage/repos.conf/ru +/wiki//etc/portage/repos.conf/uk +/wiki//etc/portage/repos.conf/zh-cn +/wiki//etc/portage/savedconfig +/wiki//etc/portage/sets +/wiki//etc/profile +/wiki//etc/profile.env +/wiki//etc/sandbox.conf +/wiki//home +/wiki//lib +/wiki//lib64 +/wiki//media +/wiki//mnt +/wiki//opt +/wiki//proc +/wiki//proc/config.gz +/wiki//run +/wiki//sbin +/wiki//srv +/wiki//sys +/wiki//tmp +/wiki//usr +/wiki//usr/bin +/wiki//usr_move +/wiki//usr/portage +/wiki//usr/portage/distfiles +/wiki//usr/portage/licenses +/wiki//usr/portage/metadata +/wiki//usr/portage/metadata/md5-cache +/wiki//usr/portage/metadata/md5-cache/usr/portage/metadata/md5-cache +/wiki//usr/portage/metadata/md5-cache/var/db/repos/gentoo//metadata/md5-cache +/wiki//usr/portage/packages +/wiki//usr/portage/profiles +/wiki//usr/portage/profiles/license_groups +/wiki//usr/portage/profiles/license_groups/usr/portage/profiles/license_groups +/wiki//usr/portage/profiles/license_groups/var/db/repos/gentoo//profiles/license_groups +/wiki//usr/share/doc/ +/wiki//var/cache/binpkgs +/wiki//var/cache/distfiles +/wiki//var/db/pkg +/wiki//var/db/pkg%22 +/wiki//var/db/repos/gentoo +/wiki//var/db/repos/gentoo/licenses +/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo//licenses +/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo/licenses +/wiki//var/db/repos/gentoo/metadata +/wiki//var/db/repos/gentoo/metadata/md5-cache +/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo//metadata +/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo/metadata +/wiki//var/db/repos/gentoo/profiles +/wiki//var/db/repos/gentoo/profiles/license_groups +/wiki//var/db/repos/gentoo/profiles/package.mask +/wiki//var/lib/portage +/wiki//var/lib/portage/world +/wiki//var/run +/gcc-bugs/bug-122002-4@http.gcc.gnu.org%2Fbugzilla%2F/T/ \ No newline at end of file diff --git a/test/log-file/test.mjs b/test/log-file/test.mjs new file mode 100644 index 000000000..8b036cd3f --- /dev/null +++ b/test/log-file/test.mjs @@ -0,0 +1,88 @@ +import { statSync } from "fs"; + +async function getPage(path) { + return fetch(`http://localhost:8923${path}`, { + headers: { + 'User-Agent': 'CHALLENGE' + } + }) + .then(resp => { + if (resp.status !== 200) { + throw new Error(`wanted status 200, got status: ${resp.status}`); + } + return resp; + }) + .then(resp => resp.text()); +} + +async function getFileSize(filePath) { + try { + return statSync(filePath).size; + } catch (error) { + return 0; + } +} + +(async () => { + const logFilePath = "./var/anubis.log"; + + // Get initial log file size + const initialSize = await getFileSize(logFilePath); + console.log(`Initial log file size: ${initialSize} bytes`); + + // Make 35 requests with different paths + const requests = []; + for (let i = 0; i < 35; i++) { + requests.push(`/test${i}`); + } + + const resultSheet = {}; + let failed = false; + + for (const path of requests) { + try { + const resp = await getPage(path); + resultSheet[path] = { + success: true, + line: resp.split("\n")[0], + }; + } catch (error) { + resultSheet[path] = { + success: false, + error: error.message, + }; + console.log(`✗ Request to ${path} failed: ${error.message}`); + failed = true; + } + } + + // Check final log file size + const finalSize = await getFileSize(logFilePath); + console.log(`Final log file size: ${finalSize} bytes`); + console.log(`Size increase: ${finalSize - initialSize} bytes`); + + // Verify that log file size increased + if (finalSize <= initialSize) { + console.error("ERROR: Log file size did not increase after making requests!"); + failed = true; + } + + let successCount = 0; + for (let [k, v] of Object.entries(resultSheet)) { + if (!v.success) { + console.error({ path: k, error: v.error }); + } else { + successCount++; + } + } + + console.log(`Successful requests: ${successCount}/${requests.length}`); + + if (failed) { + console.error("Test failed: Some requests failed or log file size did not increase"); + process.exit(1); + } else { + console.log("Test passed: All requests succeeded and log file size increased"); + process.exit(0); + } +})(); \ No newline at end of file diff --git a/test/log-file/test.sh b/test/log-file/test.sh new file mode 100755 index 000000000..c830a0bca --- /dev/null +++ b/test/log-file/test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +function cleanup() { + pkill -P $$ +} + +trap cleanup EXIT SIGINT + +# Build static assets +(cd ../.. && npm ci && npm run assets) + +go tool anubis --help 2>/dev/null || : + +go run ../cmd/httpdebug & + +go tool anubis \ + --policy-fname ./anubis.yaml \ + --use-remote-address \ + --target=http://localhost:3923 & + +sleep 2 + +backoff-retry node ./test.mjs diff --git a/test/log-file/var/.gitignore b/test/log-file/var/.gitignore new file mode 100644 index 000000000..c96a04f00 --- /dev/null +++ b/test/log-file/var/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/web/index.go b/web/index.go index 7b171ea14..6ff90527f 100644 --- a/web/index.go +++ b/web/index.go @@ -4,8 +4,8 @@ import ( "github.com/a-h/templ" "github.com/TecharoHQ/anubis/lib/challenge" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/localization" - "github.com/TecharoHQ/anubis/lib/policy/config" ) func Base(title string, body templ.Component, impressum *config.Impressum, localizer *localization.SimpleLocalizer) templ.Component { diff --git a/web/index.templ b/web/index.templ index 8e34ea4b1..a6752fab2 100644 --- a/web/index.templ +++ b/web/index.templ @@ -3,8 +3,8 @@ package web import ( "fmt" "github.com/TecharoHQ/anubis" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/localization" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/xess" ) diff --git a/web/index_templ.go b/web/index_templ.go index 0442aa04b..71a70064a 100644 --- a/web/index_templ.go +++ b/web/index_templ.go @@ -11,8 +11,8 @@ import templruntime "github.com/a-h/templ/runtime" import ( "fmt" "github.com/TecharoHQ/anubis" + "github.com/TecharoHQ/anubis/lib/config" "github.com/TecharoHQ/anubis/lib/localization" - "github.com/TecharoHQ/anubis/lib/policy/config" "github.com/TecharoHQ/anubis/xess" )