testdata: Initial action/challenges testing

.drone.jsonnet

@@ -12,6 +12,7 @@ local Build(mirror, go, alpine, os, arch) = {
         CGO_ENABLED: "0",
         GOOS: os,
         GOARCH: arch,
+        GORACE: "halt_on_error=1"
     },
     steps: [
         {
@@ -26,6 +27,16 @@ local Build(mirror, go, alpine, os, arch) = {
                 "go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime",
             ],
         },
+        {
+            name: "test",
+            image: "golang:" + go +"-alpine" + alpine,
+            mirror: mirror,
+            commands: [
+                "apk update",
+                "apk add --no-cache git",
+                "go test -p 1 -timeout 20m -v  ./tests/"
+            ],
+        },
         {
             name: "check-policy-forgejo",
             image: "alpine:" + alpine,
@@ -44,6 +55,15 @@ local Build(mirror, go, alpine, os, arch) = {
                 "./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/generic.yml --policy-snippets examples/snippets/"
             ],
         },
+        {
+            name: "check-policy-spa",
+            image: "alpine:" + alpine,
+            mirror: mirror,
+            depends_on: ["build"],
+            commands: [
+                "./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/spa.yml --policy-snippets examples/snippets/"
+            ],
+        },
         {
             name: "test-wasm-success",
             image: "alpine:" + alpine,
@@ -83,6 +103,16 @@ local Publish(mirror, registry, repo, secret, go, alpine, os, arch, trigger, pla
     },
     trigger: trigger,
     steps: [
+        {
+            name: "test",
+            image: "golang:" + go +"-alpine" + alpine,
+            mirror: mirror,
+            commands: [
+                "apk update",
+                "apk add --no-cache git",
+                "go test -p 1 -timeout 20m -v  ./tests/"
+            ],
+        },
         {
             name: "setup-buildkitd",
             image: "alpine:" + alpine,

.drone.yml

@@ -3,6 +3,7 @@ environment:
   CGO_ENABLED: "0"
   GOARCH: amd64
   GOOS: linux
+  GORACE: halt_on_error=1
   GOTOOLCHAIN: local
 kind: pipeline
 name: build-1.24-alpine3.21-amd64
@@ -19,6 +20,13 @@ steps:
   image: golang:1.24-alpine3.21
   mirror: https://mirror.gcr.io
   name: build
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
     --policy examples/forgejo.yml --policy-snippets examples/snippets/
@@ -35,6 +43,14 @@ steps:
   image: alpine:3.21
   mirror: https://mirror.gcr.io
   name: check-policy-generic
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/spa.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-spa
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
     -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
@@ -63,6 +79,7 @@ environment:
   CGO_ENABLED: "0"
   GOARCH: arm64
   GOOS: linux
+  GORACE: halt_on_error=1
   GOTOOLCHAIN: local
 kind: pipeline
 name: build-1.24-alpine3.21-arm64
@@ -79,6 +96,13 @@ steps:
   image: golang:1.24-alpine3.21
   mirror: https://mirror.gcr.io
   name: build
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
     --policy examples/forgejo.yml --policy-snippets examples/snippets/
@@ -95,6 +119,14 @@ steps:
   image: alpine:3.21
   mirror: https://mirror.gcr.io
   name: check-policy-generic
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/spa.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-spa
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
     -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
@@ -125,6 +157,13 @@ platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - echo '[registry."docker.io"]' > buildkitd.toml
   - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
@@ -173,6 +212,13 @@ platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - echo '[registry."docker.io"]' > buildkitd.toml
   - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
@@ -221,6 +267,13 @@ platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - echo '[registry."docker.io"]' > buildkitd.toml
   - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
@@ -269,6 +322,13 @@ platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - echo '[registry."docker.io"]' > buildkitd.toml
   - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
@@ -317,6 +377,13 @@ platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - echo '[registry."docker.io"]' > buildkitd.toml
   - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
@@ -365,6 +432,13 @@ platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
 - commands:
   - echo '[registry."docker.io"]' > buildkitd.toml
   - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
@@ -408,6 +482,6 @@ trigger:
 type: docker
 ---
 kind: signature
-hmac: 7d15ec708707d96b5741471555875d0001b84da74a7688baf0bae6fea0dbf138
+hmac: 07ac33f9298a9910aacb29ef18931cb999841f76be8a95ca210f9f3704c347f9
 
 ...

Dockerfile

@@ -33,6 +33,7 @@ FROM --platform=$TARGETPLATFORM ${from}
 
 COPY --from=build /go/bin/go-away /bin/go-away
 COPY examples/snippets/ /snippets/
+COPY docker-entrypoint.sh /
 
 ENV TZ UTC
 
@@ -63,13 +64,4 @@ EXPOSE 6060/tcp
 
 ENV JWT_PRIVATE_KEY_SEED="${GOAWAY_JWT_PRIVATE_KEY_SEED}"
 
-ENTRYPOINT  /bin/go-away --bind "${GOAWAY_BIND}" --bind-network "${GOAWAY_BIND_NETWORK}" --socket-mode "${GOAWAY_SOCKET_MODE}" \
+ENTRYPOINT ["/docker-entrypoint.sh"]
-            --metrics-bind "${GOAWAY_METRICS_BIND}" --debug-bind "${GOAWAY_DEBUG_BIND}" \
-            --config "${GOAWAY_CONFIG}" \
-            --policy "${GOAWAY_POLICY}" --policy-snippets "/snippets" --policy-snippets "${GOAWAY_POLICY_SNIPPETS}" \
-            --client-ip-header "${GOAWAY_CLIENT_IP_HEADER}" --backend-ip-header "${GOAWAY_BACKEND_IP_HEADER}" \
-            --cache "${GOAWAY_CACHE}" \
-            --challenge-template "${GOAWAY_CHALLENGE_TEMPLATE}" --challenge-template-theme "${GOAWAY_CHALLENGE_TEMPLATE_THEME}" \
-            --slog-level "${GOAWAY_SLOG_LEVEL}" \
-            --acme-autocert "${GOAWAY_ACME_AUTOCERT}" \
-            --backend "${GOAWAY_BACKEND}"

README.md

@@ -1,7 +1,7 @@
 ### <a id=why></a>
 # go-away
 
-Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots. 
+Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots. Uses conventional non-nuclear options.
 
 [![Build Status](https://ci.gammaspectra.live/api/badges/git/go-away/status.svg)](https://ci.gammaspectra.live/git/go-away)
 [![Go Reference](https://pkg.go.dev/badge/git.gammaspectra.live/git/go-away.svg)](https://pkg.go.dev/git.gammaspectra.live/git/go-away)
@@ -88,6 +88,7 @@ In addition to the common PASS / CHALLENGE / DENY rules, go-away offers more act
 
 |  Action   | Behavior                                                                | Terminating |
 |:---------:|:------------------------------------------------------------------------|:-----------:|
+|   NONE    | Do nothing, continue. Useful for specifying on checks or challenges.    |     No      |
 |   PASS    | Passes the request to the backend immediately                           |     Yes     |
 |   DENY    | Denies the request with a descriptive page                              |     Yes     |
 |   BLOCK   | Denies the request with a response code                                 |     Yes     |
@@ -95,6 +96,7 @@ In addition to the common PASS / CHALLENGE / DENY rules, go-away offers more act
 | CHALLENGE | Issues a challenge that when passed, acts like PASS                     |     Yes     |
 |   CHECK   | Issues a challenge that when passed, continues executing rules          |     No      |
 |   PROXY   | Proxies request to a different backend, with optional path replacements |     Yes     |
+|  CONTEXT  | Modify the request context and apply different options                  |     No      |
 
 
 CHECK allows the client to be challenged but continue matching rules after these, for example, chaining a list of challenges that must be passed. 
@@ -250,31 +252,32 @@ See [examples/snippets/](examples/snippets/) for some defaults including indexer
 In the past few years this small git instance has been hit by waves and waves of scraping.
 This was usually fought back by random useragent blocks for bots that did not follow [robots.txt](/robots.txt), until the past half year, where low-effort mass scraping was used more prominently.
 
-Recently these networks go from using residential IP blocks to sending requests at several hundred rps.
+Recently these networks go from using residential IP blocks to sending requests at several hundred requests per second.
 
 If the server gets sluggish, more requests pile up. Even when denied they scrape for weeks later. Effectively spray and pray scraping, process later.
 
 At some point about 300Mbit/s of incoming requests (not including the responses) was hitting the server. And all of them nonsense URLs, or hitting archive/bundle downloads per commit.
 
-If AI is so smart, why not just git clone the repositories?
+**If AI is so smart, why not just git clone the repositories?**
 
+* Wikimedia has posted about [How crawlers impact the operations of the Wikimedia projects](https://diff.wikimedia.org/2025/04/01/how-crawlers-impact-the-operations-of-the-wikimedia-projects/) [01/04/2025]
 
-Xe (anubis creator) has written about similar frustrations in several blogposts:
+* Xe (Anubis creator) has written about similar frustrations in several blogposts:
+  * [Amazon's AI crawler is making my git server unstable](https://xeiaso.net/notes/2025/amazon-crawler/) [01/17/2025]
+  * [Anubis works](https://xeiaso.net/notes/2025/anubis-works/) [04/12/2025]
 
-* [Amazon's AI crawler is making my git server unstable](https://xeiaso.net/notes/2025/amazon-crawler/) [01/17/2025]
+* Drew DeVault (sourcehut) has posted several articles and outages regarding the same issues:
-* [Anubis works](https://xeiaso.net/notes/2025/anubis-works/) [04/12/2025]
+  * [Drew Blog: Please stop externalizing your costs directly into my face](https://drewdevault.com/2025/03/17/2025-03-17-Stop-externalizing-your-costs-on-me.html) [17/03/2025]
+    * (fun tidbit: I'm the one quoted as having the feedback discussion interrupted to deal with bots!)
+   * [sourcehut status: LLM crawlers continue to DDoS SourceHut](https://status.sr.ht/issues/2025-03-17-git.sr.ht-llms/) [17/03/2025]
+  * [sourcehut Blog: You cannot have our user's data](https://sourcehut.org/blog/2025-04-15-you-cannot-have-our-users-data/) [15/04/2025]
 
-Drew DeVault (sourcehut) has posted several articles regarding the same issues:
+* Others were also suffering at the same time [[1]](https://donotsta.re/notice/AreSNZlRlJv73AW7tI) [[2]](https://community.ipfire.org/t/suricata-ruleset-to-prevent-ai-scraping/11974) [[3]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[4]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[5]](https://blog.nytsoi.net/2025/03/01/obliterated-by-ai).
-* [Please stop externalizing your costs directly into my face](https://drewdevault.com/2025/03/17/2025-03-17-Stop-externalizing-your-costs-on-me.html) [17/03/2025]
-  * (fun tidbit: I'm the one quoted as having the feedback discussion interrupted to deal with bots!)
-* [sourcehut Blog: You cannot have our user's data](https://sourcehut.org/blog/2025-04-15-you-cannot-have-our-users-data/)
-
-Others were also suffering at the same time [[1]](https://donotsta.re/notice/AreSNZlRlJv73AW7tI) [[2]](https://community.ipfire.org/t/suricata-ruleset-to-prevent-ai-scraping/11974) [[3]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[4]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[5]](https://blog.nytsoi.net/2025/03/01/obliterated-by-ai).
 
 ---
 Initially I deployed Anubis, and yeah, it does work!
 
-This tool started as a way to replace [Anubis](https://anubis.techaro.lol/) as it was not found as featureful as desired.
+This tool started as a way to replace [Anubis](https://anubis.techaro.lol/) as it was not found as featureful as desired, and the impact was too high.
 
 go-away may not be as straight to configure as Anubis but this was chosen to reduce impact on legitimate users, and offers many more options to dynamically target new waves.
 

cmd/go-away/main.go

@@ -288,6 +288,8 @@ func main() {
 		fatal(fmt.Errorf("failed to create server: %w", err))
 	}
 
+	server.ErrorLog = slog.NewLogLogger(slog.With("server", "http").Handler(), slog.LevelError)
+
 	go func() {
 		handler, err := loadPolicyState()
 		if err != nil {
@@ -325,8 +327,9 @@ func main() {
 			mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 			debugServer := http.Server{
-				Addr:    opt.BindDebug,
+				Addr:     opt.BindDebug,
-				Handler: mux,
+				Handler:  mux,
+				ErrorLog: slog.NewLogLogger(slog.With("server", "debug").Handler(), slog.LevelError),
 			}
 
 			slog.Warn(
@@ -344,8 +347,9 @@ func main() {
 			mux := http.NewServeMux()
 			mux.Handle("/metrics", promhttp.Handler())
 			metricsServer := http.Server{
-				Addr:    opt.BindMetrics,
+				Addr:     opt.BindMetrics,
-				Handler: mux,
+				Handler:  mux,
+				ErrorLog: slog.NewLogLogger(slog.With("server", "metrics").Handler(), slog.LevelError),
 			}
 
 			slog.Warn(

docker-entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+set -e
+
+if [ "${1#-}" != "$1" ]; then
+        set -- /bin/go-away \
+                --bind "${GOAWAY_BIND}" --bind-network "${GOAWAY_BIND_NETWORK}" --socket-mode "${GOAWAY_SOCKET_MODE}" \
+                --metrics-bind "${GOAWAY_METRICS_BIND}" --debug-bind "${GOAWAY_DEBUG_BIND}" \
+                --config "${GOAWAY_CONFIG}" \
+                --policy "${GOAWAY_POLICY}" --policy-snippets "/snippets" --policy-snippets "${GOAWAY_POLICY_SNIPPETS}" \
+                --client-ip-header "${GOAWAY_CLIENT_IP_HEADER}" --backend-ip-header "${GOAWAY_BACKEND_IP_HEADER}" \
+                --cache "${GOAWAY_CACHE}" \
+                --challenge-template "${GOAWAY_CHALLENGE_TEMPLATE}" --challenge-template-theme "${GOAWAY_CHALLENGE_TEMPLATE_THEME}" \
+                --slog-level "${GOAWAY_SLOG_LEVEL}" \
+                --acme-autocert "${GOAWAY_ACME_AUTOCERT}" \
+                --backend "${GOAWAY_BACKEND}" \
+                "$@"
+fi
+
+if [ "$1" = "go-away" ]; then
+        shift
+        set -- /bin/go-away "$@"
+fi
+
+exec "$@"

embed/templates/challenge-anubis.gohtml

@@ -5,15 +5,11 @@
     <link rel="stylesheet" href="{{ .Path }}/assets/static/anubis/style.css?cacheBust={{ .Random }}"/>
     <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
     <meta name="referrer" content="origin"/>
-    {{ range $key, $value := .Meta }}
+    {{ range .Meta }}
-        {{ if eq $key "refresh"}}
+    <meta {{ range $key, $value := . }}{{ $key | attr }}="{{ $value }}" {{end}}/>
-        <meta http-equiv="{{ $key }}" content="{{ $value }}"/>
-        {{else}}
-        <meta name="{{ $key }}" content="{{ $value }}"/>
-        {{end}}
     {{ end }}
     {{ range .HeaderTags }}
-        {{ . }}
+    {{ . }}
     {{ end }}
 </head>
 <body id="top">

embed/templates/challenge-forgejo.gohtml

@@ -1,24 +1,16 @@
 <!DOCTYPE html>
-{{$theme := "forgejo-auto"}}
+{{$theme := "forgejo-auto"}}{{ if .Theme }}{{$theme = .Theme}}{{ end }}
-{{ if .Theme }}
-    {{$theme = .Theme}}
-{{ end }}
 <html lang="en-US" data-theme="{{ $theme }}">
 <head>
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
     <title>{{ .Title }}</title>
     <meta name="referrer" content="origin">
-
+    {{ range .Meta }}
-    {{ range $key, $value := .Meta }}
+    <meta {{ range $key, $value := . }}{{ $key | attr }}="{{ $value }}" {{end}}/>
-        {{ if eq $key "refresh"}}
-            <meta http-equiv="{{ $key }}" content="{{ $value }}"/>
-        {{else}}
-            <meta name="{{ $key }}" content="{{ $value }}"/>
-        {{end}}
     {{ end }}
     {{ range .HeaderTags }}
-        {{ . }}
+    {{ . }}
     {{ end }}
 
 
@@ -80,9 +72,11 @@
                         </div>
                     {{end}}
 
-                    <noscript>
+                    {{if .EndTags }}
-                        {{ .Strings.Get "noscript" }}
+                        <noscript>
-                    </noscript>
+                            {{ .Strings.Get "noscript_warning" }}
+                        </noscript>
+                    {{end}}
 
                     <p><small>{{ .Strings.Get "details_contact_admin_with_request_id" }}: <em>{{ .Id }}</em></small></p>
                 </div>

examples/forgejo.yml

@@ -81,6 +81,7 @@ conditions:
     - 'path.matches("^/[^/]+$") && "tab" in query && query.tab == "activity"'
 
 
+# Rules are checked sequentially in order, from top to bottom
 rules:
   - name: allow-well-known-resources
     conditions:
@@ -286,6 +287,21 @@ rules:
     conditions:
       - '!(method == "HEAD" || method == "GET")'
 
+  # Enable fetching OpenGraph and other tags from backend on these paths
+  - name: enable-meta-tags
+    action: context
+    settings:
+      context-set:
+        # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
+        proxy-meta-tags: "true"
+        
+      # Set additional response headers
+      #response-headers:
+      # X-Clacks-Overhead:
+      #  - GNU Terry Pratchett
+
+
+
   - name: plaintext-browser
     action: challenge
     settings:
@@ -293,6 +309,7 @@ rules:
     conditions:
       - 'userAgent.startsWith("Lynx/")'
 
+  # Comment this rule out to not challenge tool-like user agents
   - name: standard-tools
     action: challenge
     settings:
@@ -307,3 +324,5 @@ rules:
       challenges: [http-cookie-check, preload-link, meta-refresh, resource-load, js-pow-sha256]
     conditions:
       - '($is-generic-browser)'
+
+# If end of rules is reached, default is PASS

examples/generic.yml

@@ -38,7 +38,7 @@ conditions:
     - 'userAgent.matches("^Mozilla/[1-4]")'
 
 
-
+# Rules are checked sequentially in order, from top to bottom
 rules:
   - name: allow-well-known-resources
     conditions:
@@ -137,6 +137,19 @@ rules:
     conditions:
       - '!(method == "HEAD" || method == "GET")'
 
+  # Enable fetching OpenGraph and other tags from backend on these paths
+  - name: enable-meta-tags
+    action: context
+    settings:
+      context-set:
+        # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
+        proxy-meta-tags: "true"
+
+      # Set additional response headers
+      #response-headers:
+      # X-Clacks-Overhead:
+      #  - GNU Terry Pratchett
+
   - name: plaintext-browser
     action: challenge
     settings:
@@ -144,14 +157,15 @@ rules:
     conditions:
       - 'userAgent.startsWith("Lynx/")'
 
-  - name: standard-tools
+  # Uncomment this rule out to challenge tool-like user agents
-    action: challenge
+  #- name: standard-tools
-    settings:
+  #  action: challenge
-      challenges: [cookie]
+  #  settings:
-    conditions:
+  #    challenges: [cookie]
-      - '($is-generic-robot-ua)'
+  #  conditions:
-      - '($is-tool-ua)'
+  #    - '($is-generic-robot-ua)'
-      - '!($is-generic-browser)'
+  #    - '($is-tool-ua)'
+  #    - '!($is-generic-browser)'
 
   - name: standard-browser
     action: challenge
@@ -159,3 +173,5 @@ rules:
       challenges: [preload-link, meta-refresh, resource-load, js-pow-sha256]
     conditions:
       - '($is-generic-browser)'
+
+# If end of rules is reached, default is PASS

examples/spa.yml

@@ -0,0 +1,87 @@
+# Example cmdline (forward requests from upstream to port :8080)
+# $ go-away --bind :8080 --backend site.example.com=http://site:3000 --policy examples/spa.yml --policy-snippets example/snippets/ --challenge-template anubis
+
+
+
+# Define networks to be used later below
+networks:
+  # Networks will get included from snippets
+
+
+challenges:
+  # Challenges will get included from snippets
+  
+conditions:
+  # Conditions will get replaced on rules AST when found as ($condition-name)
+
+
+  is-static-asset:
+    - 'path == "/apple-touch-icon.png"'
+    - 'path == "/apple-touch-icon-precomposed.png"'
+    - 'path.matches("\\.(manifest|ttf|woff|woff2|jpg|jpeg|gif|png|webp|avif|svg|mp4|webm|css|js|mjs|wasm)$")'
+    # Add other paths where you have static assets
+    # - 'path.startsWith("/static/") || path.startsWith("/assets/")'
+
+
+# Rules are checked sequentially in order, from top to bottom
+rules:
+  - name: allow-well-known-resources
+    conditions:
+      - '($is-well-known-asset)'
+    action: pass
+
+  - name: allow-static-resources
+    conditions:
+      - '($is-static-asset)'
+    action: pass
+
+  - name: unknown-crawlers
+    conditions:
+      # No user agent set
+      - 'userAgent == ""'
+    action: deny
+
+  # Enable fetching OpenGraph and other tags from backend on index
+  - name: enable-meta-tags
+    action: context
+    conditions:
+      - 'path == "/" || path == "/index.html"'
+    settings:
+      context-set:
+        # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
+        proxy-meta-tags: "true"
+
+  # Challenge incoming visitors so challenge is remembered on api endpoints
+  # API requests will have this challenge stored
+  - name: index
+    conditions:
+      - 'path == "/" || path == "/index.html"'
+    settings:
+      challenges: [ preload-link, header-refresh ]
+    action: challenge
+
+  # Allow PUT/DELETE/PATCH/POST requests in general
+  - name: non-get-request
+    action: pass
+    conditions:
+      - '!(method == "HEAD" || method == "GET")'
+
+  # Challenge rest of endpoints (SPA API etc.)
+  # Above rule on index ensures clients have passed a challenge beforehand
+  - name: standard-browser
+    action: challenge
+    settings:
+      challenges: [ preload-link, header-refresh ]
+      # Fallback on cookie challenge
+      fail: challenge
+      fail-settings:
+        challenges: [ cookie ]
+    conditions:
+      - '($is-generic-browser)'
+
+  - name: other-fetchers
+    action: challenge
+    settings:
+      challenges: [ cookie ]
+    conditions:
+      - '!($is-generic-browser)'

lib/action/challenge.go

@@ -142,8 +142,10 @@ func (a Challenge) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Re
 
 		expiry := data.Expiration(reg.Duration)
 		key := challenge.GetChallengeKeyForRequest(data.State, reg, expiry, r)
-		data.State.ChallengeIssued(r, reg, r.URL.String(), logger)
 		result = reg.IssueChallenge(w, r, key, expiry)
+		if result != challenge.VerifyResultSkip {
+			data.State.ChallengeIssued(r, reg, r.URL.String(), logger)
+		}
 		data.ChallengeVerify[reg.Id()] = result
 		data.ChallengeState[reg.Id()] = challenge.VerifyStatePass
 		switch result {

lib/action/context.go

@@ -0,0 +1,55 @@
+package action
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionCONTEXT] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		params := ContextDefaultSettings
+
+		if settings != nil {
+			ymlData, err := settings.MarshalYAML()
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.Unmarshal(ymlData, &params)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return Context{
+			opts: params,
+		}, nil
+	}
+}
+
+var ContextDefaultSettings = ContextSettings{}
+
+type ContextSettings struct {
+	ContextSet      map[string]string `yaml:"context-set"`
+	ResponseHeaders map[string]string `yaml:"response-headers"`
+}
+
+type Context struct {
+	opts ContextSettings
+}
+
+func (a Context) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	data := challenge.RequestDataFromContext(r.Context())
+	for k, v := range a.opts.ContextSet {
+		data.SetOpt(k, v)
+	}
+
+	for k, v := range a.opts.ResponseHeaders {
+		w.Header().Set(k, v)
+	}
+
+	return true, nil
+}

lib/challenge/data.go

@@ -46,6 +46,8 @@ type RequestData struct {
 	fp     map[string]string
 	header traits.Mapper
 	query  traits.Mapper
+
+	opts map[string]string
 }
 
 func CreateRequestData(r *http.Request, state StateInterface) (*http.Request, *RequestData) {
@@ -84,13 +86,16 @@ func CreateRequestData(r *http.Request, state StateInterface) (*http.Request, *R
 
 	data.query = http_cel.NewValuesMap(q)
 	data.header = http_cel.NewMIMEMap(textproto.MIMEHeader(r.Header))
+	data.opts = make(map[string]string)
 
 	sum := sha256.New()
 	sum.Write([]byte(r.Host))
 	sum.Write([]byte{0})
+	sum.Write(data.NetworkPrefix().AsSlice())
+	sum.Write([]byte{0})
 	sum.Write(state.PublicKey())
 	sum.Write([]byte{0})
-	data.CookiePrefix = utils.CookiePrefix + hex.EncodeToString(sum.Sum(nil)[:4]) + "-"
+	data.CookiePrefix = utils.CookiePrefix + hex.EncodeToString(sum.Sum(nil)[:6]) + "-"
 
 	r = r.WithContext(context.WithValue(r.Context(), requestDataContextKey{}, &data))
 	r = utils.SetRemoteAddress(r, data.RemoteAddress)
@@ -126,6 +131,61 @@ func (d *RequestData) Parent() cel.Activation {
 	return nil
 }
 
+func (d *RequestData) NetworkPrefix() netip.Addr {
+	address := d.RemoteAddress.Addr().Unmap()
+	if address.Is4() {
+		// Take a /24 for IPv4
+		prefix, _ := address.Prefix(24)
+		return prefix.Addr()
+	} else {
+		// Take a /64 for IPv6
+		prefix, _ := address.Prefix(64)
+		return prefix.Addr()
+	}
+}
+
+const (
+	RequestOptBackendHost   = "backend-host"
+	RequestOptCacheMetaTags = "proxy-meta-tags"
+)
+
+func (d *RequestData) SetOpt(n, v string) {
+	d.opts[n] = v
+}
+
+func (d *RequestData) GetOpt(n, def string) string {
+	v, ok := d.opts[n]
+	if !ok {
+		return def
+	}
+	return v
+}
+
+func (d *RequestData) GetOptBool(n string, def bool) bool {
+	v, ok := d.opts[n]
+	if !ok {
+		return def
+	}
+	switch v {
+	case "true", "t", "1", "yes", "yep", "y", "ok":
+		return true
+	case "false", "f", "0", "no", "nope", "n", "err":
+		return false
+	default:
+		return def
+	}
+}
+
+func (d *RequestData) BackendHost() (http.Handler, string) {
+	host := d.r.Host
+
+	if opt := d.GetOpt(RequestOptBackendHost, ""); opt != "" && opt != host {
+		host = d.r.Host
+	}
+
+	return d.State.GetBackend(host), host
+}
+
 func (d *RequestData) EvaluateChallenges(w http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
 	var issuedChallenge string
@@ -183,7 +243,7 @@ func (d *RequestData) HasValidChallenge(id Id) bool {
 	return d.ChallengeVerify[id].Ok()
 }
 
-func (d *RequestData) Headers(headers http.Header) {
+func (d *RequestData) RequestHeaders(headers http.Header) {
 	headers.Set("X-Away-Id", d.Id.String())
 
 	for id, result := range d.ChallengeVerify {

lib/challenge/helper.go

@@ -8,18 +8,33 @@ import (
 	"git.gammaspectra.live/git/go-away/utils"
 	"net/http"
 	"net/url"
+	"strings"
 )
 
+var ErrInvalidToken = errors.New("invalid token")
+var ErrMismatchedToken = errors.New("mismatched token")
+var ErrMismatchedTokenHappyEyeballs = errors.New("mismatched token: IPv4 to IPv6 upgrade detected, retrying")
+
 func NewKeyVerifier() (verify VerifyFunc, issue func(key Key) string) {
 	return func(key Key, token []byte, r *http.Request) (VerifyResult, error) {
 			expectedKey, err := hex.DecodeString(string(token))
 			if err != nil {
 				return VerifyResultFail, err
 			}
+			if len(expectedKey) != KeySize {
+				return VerifyResultFail, ErrInvalidToken
+			}
 			if subtle.ConstantTimeCompare(key[:], expectedKey) == 1 {
 				return VerifyResultOK, nil
 			}
-			return VerifyResultFail, errors.New("invalid token")
+
+			kk := Key(expectedKey)
+			// IPv4 -> IPv6 Happy Eyeballs
+			if key.Get(KeyFlagIsIPv4) == 0 && kk.Get(KeyFlagIsIPv4) > 0 {
+				return VerifyResultOK, ErrMismatchedTokenHappyEyeballs
+			}
+
+			return VerifyResultFail, ErrMismatchedToken
 		}, func(key Key) string {
 			return hex.EncodeToString(key[:])
 		}
@@ -95,7 +110,9 @@ func RedirectUrl(r *http.Request, reg *Registration) (*url.URL, error) {
 	data := RequestDataFromContext(r.Context())
 	values := uri.Query()
 	values.Set(QueryArgRequestId, data.Id.String())
-	values.Set(QueryArgReferer, r.Referer())
+	if ref := r.Referer(); ref != "" {
+		values.Set(QueryArgReferer, r.Referer())
+	}
 	values.Set(QueryArgChallenge, reg.Name)
 	uri.RawQuery = values.Encode()
 
@@ -104,6 +121,26 @@ func RedirectUrl(r *http.Request, reg *Registration) (*url.URL, error) {
 
 func VerifyHandlerChallengeResponseFunc(state StateInterface, data *RequestData, w http.ResponseWriter, r *http.Request, verifyResult VerifyResult, err error, redirect string) {
 	if err != nil {
+		// Happy Eyeballs! auto retry
+		if errors.Is(err, ErrMismatchedTokenHappyEyeballs) {
+			reqUri := *r.URL
+			q := reqUri.Query()
+
+			ref := q.Get(QueryArgReferer)
+			// delete query parameters that were set by go-away
+			for k := range q {
+				if strings.HasPrefix(k, QueryArgPrefix) {
+					q.Del(k)
+				}
+			}
+			if ref != "" {
+				q.Set(QueryArgReferer, ref)
+			}
+			reqUri.RawQuery = q.Encode()
+
+			http.Redirect(w, r, reqUri.String(), http.StatusTemporaryRedirect)
+			return
+		}
 		state.ErrorPage(w, r, http.StatusBadRequest, err, redirect)
 		return
 	} else if !verifyResult.Ok() {

lib/challenge/key.go

@@ -42,13 +42,13 @@ func KeyFromString(s string) (Key, error) {
 
 func GetChallengeKeyForRequest(state StateInterface, reg *Registration, until time.Time, r *http.Request) Key {
 	data := RequestDataFromContext(r.Context())
-	address := data.RemoteAddress
+
 	hasher := sha256.New()
 	hasher.Write([]byte("challenge\x00"))
 	hasher.Write([]byte(reg.Name))
 	hasher.Write([]byte{0})
-	ipBuf := address.Addr().Unmap().As16()
+	keyAddr := data.NetworkPrefix().As16()
-	hasher.Write(ipBuf[:])
+	hasher.Write(keyAddr[:])
 	hasher.Write([]byte{0})
 
 	// specific headers
@@ -73,7 +73,7 @@ func GetChallengeKeyForRequest(state StateInterface, reg *Registration, until ti
 
 	sum[0] = 0
 
-	if address.Addr().Unmap().Is4() {
+	if data.RemoteAddress.Addr().Unmap().Is4() {
 		// Is IPv4, mark
 		sum.Set(KeyFlagIsIPv4)
 	}

lib/challenge/refresh/refresh.go

@@ -13,7 +13,7 @@ func init() {
 }
 
 type Parameters struct {
-	Mode string `yaml:"refresh-mode"`
+	Mode string `yaml:"refresh-via"`
 }
 
 var DefaultParameters = Parameters{
@@ -47,8 +47,11 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 
 		if params.Mode == "meta" {
 			state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
-				"Meta": map[string]string{
+				"Meta": []map[string]string{
-					"refresh": "0; url=" + uri.String(),
+					{
+						"http-equiv": "refresh",
+						"content":    "0; url=" + uri.String(),
+					},
 				},
 			})
 		} else {

lib/http.go

@@ -8,9 +8,12 @@ import (
 	"git.gammaspectra.live/git/go-away/lib/challenge"
 	"git.gammaspectra.live/git/go-away/lib/policy"
 	"git.gammaspectra.live/git/go-away/utils"
+	"golang.org/x/net/html"
 	"log/slog"
 	"net/http"
+	"slices"
 	"strings"
+	"time"
 )
 
 func GetLoggerForRequest(r *http.Request) *slog.Logger {
@@ -35,6 +38,98 @@ func GetLoggerForRequest(r *http.Request) *slog.Logger {
 	return slog.With(args...)
 }
 
+func (state *State) fetchMetaTags(host string, backend http.Handler, r *http.Request) []html.Node {
+	uri := *r.URL
+	q := uri.Query()
+	for k := range q {
+		if strings.HasPrefix(k, challenge.QueryArgPrefix) {
+			q.Del(k)
+		}
+	}
+	uri.RawQuery = q.Encode()
+
+	key := fmt.Sprintf("%s:%s", host, uri.String())
+
+	if v, ok := state.tagCache.Get(key); ok {
+		return v
+	}
+
+	result := utils.FetchTags(backend, &uri, "meta")
+	if result == nil {
+		return nil
+	}
+
+	entries := make([]html.Node, 0, len(result))
+
+	safeAttributes := []string{"name", "property", "content"}
+	for _, n := range result {
+		if n.Namespace != "" {
+			continue
+		}
+
+		var name string
+		for _, attr := range n.Attr {
+			if attr.Namespace != "" {
+				continue
+			}
+			if attr.Key == "name" {
+				name = attr.Val
+				break
+			}
+			if attr.Key == "property" && name == "" {
+				name = attr.Val
+			}
+		}
+
+		// prevent unwanted keys like CSRF and other internal entries to pass through as much as possible
+
+		var keep bool
+		if strings.HasPrefix("og:", name) || strings.HasPrefix("fb:", name) || strings.HasPrefix("twitter:", name) || strings.HasPrefix("profile:", name) {
+			// social / OpenGraph tags
+			keep = true
+		} else if name == "vcs" || strings.HasPrefix("vcs:", name) {
+			// source tags
+			keep = true
+		} else if name == "forge" || strings.HasPrefix("forge:", name) {
+			// forge tags
+			keep = true
+		} else {
+			switch name {
+			// standard content tags
+			case "application-name", "author", "description", "keywords", "robots", "thumbnail":
+				keep = true
+			case "go-import", "go-source":
+				// golang tags
+				keep = true
+			case "apple-itunes-app":
+			}
+		}
+
+		// prevent other arbitrary arguments
+		if keep {
+			newNode := html.Node{
+				Type: html.ElementNode,
+				Data: n.Data,
+			}
+			for _, attr := range n.Attr {
+				if attr.Namespace != "" {
+					continue
+				}
+				if slices.Contains(safeAttributes, attr.Key) {
+					newNode.Attr = append(newNode.Attr, attr)
+				}
+			}
+			if len(newNode.Attr) == 0 {
+				continue
+			}
+			entries = append(entries, newNode)
+		}
+	}
+
+	state.tagCache.Set(key, entries, time.Hour*6)
+	return entries
+}
+
 func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 	host := r.Host
 
@@ -46,6 +141,19 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	getBackend := func() http.Handler {
+		if opt := data.GetOpt(challenge.RequestOptBackendHost, ""); opt != "" && opt != host {
+			b := state.GetBackend(host)
+			if b == nil {
+				http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+				// return empty backend
+				return http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
+			}
+			return b
+		}
+		return backend
+	}
+
 	lg := state.Logger(r)
 
 	cleanupRequest := func(r *http.Request, fromChallenge bool) {
@@ -66,7 +174,7 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 		}
 		r.URL.RawQuery = q.Encode()
 
-		data.Headers(r.Header)
+		data.RequestHeaders(r.Header)
 
 		// delete cookies set by go-away to prevent user tracking that way
 		cookies := r.Cookies()
@@ -81,7 +189,7 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 	for _, rule := range state.rules {
 		next, err := rule.Evaluate(lg, w, r, func() http.Handler {
 			cleanupRequest(r, true)
-			return backend
+			return getBackend()
 		})
 		if err != nil {
 			state.ErrorPage(w, r, http.StatusInternalServerError, err, "")
@@ -103,7 +211,7 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 		r.Header.Set("X-Away-Action", "PASS")
 
 		cleanupRequest(r, false)
-		return backend
+		return getBackend()
 	})
 }
 

lib/policy/rule.go

@@ -26,6 +26,9 @@ const (
 
 	// RuleActionPROXY Proxies request to a backend, with optional path replacements
 	RuleActionPROXY RuleAction = "PROXY"
+
+	// RuleActionCONTEXT Changes Request Context information or properties
+	RuleActionCONTEXT RuleAction = "CONTEXT"
 )
 
 type Rule struct {

lib/state.go

@@ -4,7 +4,10 @@ import (
 	http_cel "codeberg.org/gone/http-cel"
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"git.gammaspectra.live/git/go-away/lib/challenge"
 	"git.gammaspectra.live/git/go-away/lib/policy"
@@ -12,12 +15,14 @@ import (
 	"git.gammaspectra.live/git/go-away/utils"
 	"github.com/google/cel-go/cel"
 	"github.com/yl2chen/cidranger"
+	"golang.org/x/net/html"
 	"log/slog"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -43,11 +48,13 @@ type State struct {
 
 	close chan struct{}
 
+	tagCache *utils.DecayMap[string, []html.Node]
+
 	Mux *http.ServeMux
 }
 
-func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSettings) (handler http.Handler, err error) {
+func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSettings) (state *State, err error) {
-	state := new(State)
+	state = new(State)
 	state.close = make(chan struct{})
 	state.settings = settings
 	state.opt = opt
@@ -117,15 +124,19 @@ func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSetti
 		for i, e := range network {
 			prefixes, err := func() ([]net.IPNet, error) {
 				var useCache bool
+
+				cacheKey := fmt.Sprintf("%s-%d-", k, i)
 				if e.Url != nil {
 					slog.Debug("loading network url list", "network", k, "url", *e.Url)
 					useCache = true
+					sum := sha256.Sum256([]byte(*e.Url))
+					cacheKey += hex.EncodeToString(sum[:4])
 				} else if e.ASN != nil {
 					slog.Debug("loading ASN", "network", k, "asn", *e.ASN)
 					useCache = true
+					cacheKey += strconv.FormatInt(int64(*e.ASN), 10)
 				}
 
-				cacheKey := fmt.Sprintf("%s-%d", k, i)
 				var cached []net.IPNet
 				if useCache && networkCache != nil {
 					//TODO: add randomness
@@ -165,16 +176,22 @@ func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSetti
 				}
 				return prefixes, nil
 			}()
+			if err != nil {
+				if e.Url != nil {
+					slog.Error("error loading network list", "network", k, "url", *e.Url, "error", err)
+				} else if e.ASN != nil {
+					slog.Error("error loading ASN", "network", k, "asn", *e.ASN, "error", err)
+				} else {
+					slog.Error("error loading list", "network", k, "error", err)
+				}
+				continue
+			}
 			for _, prefix := range prefixes {
 				err = ranger.Insert(cidranger.NewBasicRangerEntry(prefix))
 				if err != nil {
 					return nil, fmt.Errorf("networks %s: error inserting prefix %s: %v", k, prefix.String(), err)
 				}
 			}
-			if err != nil {
-				slog.Error("error loading network list", "network", k, "url", *e.Url, "error", err)
-				continue
-			}
 		}
 
 		slog.Warn("loaded network prefixes", "network", k, "count", ranger.Len())
@@ -231,5 +248,30 @@ func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSetti
 		return nil, err
 	}
 
+	state.tagCache = utils.NewDecayMap[string, []html.Node]()
+
+	go func() {
+		ticker := time.NewTicker(time.Minute * 37)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				state.tagCache.Decay()
+			case <-state.close:
+				return
+			}
+		}
+	}()
+
 	return state, nil
 }
+
+func (state *State) Close() error {
+	select {
+	case <-state.close:
+		return errors.New("already closed")
+	default:
+		close(state.close)
+	}
+	return nil
+}

lib/template.go

@@ -36,7 +36,14 @@ func init() {
 }
 
 func initTemplate(name, data string) error {
-	tpl := template.New(name)
+	tpl := template.New(name).Funcs(template.FuncMap{
+		"attr": func(s string) template.HTMLAttr {
+			return template.HTMLAttr(s)
+		},
+		"safe": func(s string) template.HTML {
+			return template.HTML(s)
+		},
+	})
 	_, err := tpl.Parse(data)
 	if err != nil {
 		return err
@@ -68,6 +75,22 @@ func (state *State) ChallengePage(w http.ResponseWriter, r *http.Request, status
 		input["Title"] = state.Options().Strings.Get("title_challenge")
 	}
 
+	if data.GetOptBool(challenge.RequestOptCacheMetaTags, false) {
+		backend, host := data.BackendHost()
+		if tags := state.fetchMetaTags(host, backend, r); len(tags) > 0 {
+			tagMap, _ := input["Meta"].([]map[string]string)
+
+			for _, tag := range tags {
+				tagAttrs := make(map[string]string, len(tag.Attr))
+				for _, v := range tag.Attr {
+					tagAttrs[v.Key] = v.Val
+				}
+				tagMap = append(tagMap, tagAttrs)
+			}
+			input["Meta"] = tagMap
+		}
+	}
+
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 
 	buf := bytes.NewBuffer(make([]byte, 0, 8192))
@@ -103,6 +126,22 @@ func (state *State) ErrorPage(w http.ResponseWriter, r *http.Request, status int
 		input[k] = v
 	}
 
+	if data.GetOptBool(challenge.RequestOptCacheMetaTags, false) {
+		backend, host := data.BackendHost()
+		if tags := state.fetchMetaTags(host, backend, r); len(tags) > 0 {
+			tagMap, _ := input["Meta"].([]map[string]string)
+
+			for _, tag := range tags {
+				tagAttrs := make(map[string]string, len(tag.Attr))
+				for _, v := range tag.Attr {
+					tagAttrs[v.Key] = v.Val
+				}
+				tagMap = append(tagMap, tagAttrs)
+			}
+			input["Meta"] = tagMap
+		}
+	}
+
 	err2 := templates["challenge-"+state.Options().ChallengeTemplate+".gohtml"].Execute(buf, input)
 	if err2 != nil {
 		// nested errors!

tests/action_test.go

@@ -0,0 +1,280 @@
+package tests
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/utils"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func testAction(t *testing.T, pol policy.Policy, expected int, url string) (*http.Response, error) {
+	settings := setupDefaultSettings(t)
+	var r *http.Response
+	err := MakeGoAwayState(pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		request, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return err
+		}
+		request.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		response, err := do(request)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != expected {
+			return fmt.Errorf("expected status code %d, got %d", expected, response.StatusCode)
+		}
+		r = response
+
+		return nil
+	})
+	return r, err
+}
+
+func TestActionPass(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: pass
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	_, err = testAction(t, *pol, http.StatusOK, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestActionNone(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: none
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	_, err = testAction(t, *pol, http.StatusOK, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestActionDrop(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: drop
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusForbidden, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) != 0 {
+		t.Fatal(fmt.Errorf("expected empty response, got %s", string(data)))
+	}
+}
+
+func TestActionDeny(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: deny
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusForbidden, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) == 0 {
+		t.Fatal(errors.New("expected non-empty response, got none"))
+	}
+}
+
+func TestActionBlock(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: block
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusForbidden, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) == 0 {
+		t.Fatal(errors.New("expected non-empty response, got none"))
+	}
+}
+
+func TestActionCode(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: code
+    settings:
+      http-code: 418
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	_, err = testAction(t, *pol, http.StatusTeapot, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestActionContextResponseHeaders(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: context
+    settings:
+      response-headers:
+        X-World-Domination: yes
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusOK, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response.Header.Get("X-World-Domination") != "yes" {
+		t.Fatal(fmt.Errorf("expected header set, got %s", response.Header.Get("X-World-Domination")))
+	}
+}
+
+func TestActionContextSetMetaTags(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test-context
+    conditions: ["true"]
+    action: context
+    settings:
+      context-set:
+        proxy-meta-tags: yes
+
+  - name: test
+    conditions: ["true"]
+    action: deny
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	uri, err := url.Parse("/test")
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+	q := uri.Query()
+	q.Set("mime-type", "text/html")
+	q.Set("content", base64.RawURLEncoding.EncodeToString([]byte(`
+<!DOCTYPE html>
+<html>
+<head>
+	<meta name="description" content="test">
+</head>
+</html>
+`)))
+
+	uri.RawQuery = q.Encode()
+
+	response, err := testAction(t, *pol, http.StatusForbidden, uri.String())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tags := utils.FetchTagsFromReader(response.Body, "meta")
+
+	if str := func() string {
+		for _, t := range tags {
+			var is bool
+			var val string
+			for _, a := range t.Attr {
+				if a.Key == "name" && a.Val == "description" {
+					is = true
+				}
+				if a.Key == "content" {
+					val = a.Val
+				}
+			}
+			if is {
+				return val
+			}
+		}
+		return "NONE"
+	}(); str != "test" {
+		t.Fatal(fmt.Errorf("expected meta tag with 'test', got %s", str))
+	}
+}

tests/away.go

@@ -0,0 +1,34 @@
+package tests
+
+import (
+	"git.gammaspectra.live/git/go-away/lib"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/lib/settings"
+	"net/http"
+	"net/http/httptest"
+)
+
+var DefaultSettings = policy.StateSettings{
+	Cache: nil,
+	Backends: map[string]http.Handler{
+		"*": MakeTestBackend(),
+	},
+	MainName:              "go-away/tests",
+	MainVersion:           "testing",
+	BasePath:              "/.go-away",
+	ChallengeResponseCode: http.StatusTeapot,
+	ClientIpHeader:        "X-Forwarded-For",
+}
+
+func MakeGoAwayState(pol policy.Policy, stateSettings policy.StateSettings, f func(do func(r *http.Request, errs ...error) (*http.Response, error)) error) error {
+	state, err := lib.NewState(pol, settings.DefaultSettings, stateSettings)
+	if err != nil {
+		return err
+	}
+
+	return f(func(r *http.Request, errs ...error) (*http.Response, error) {
+		recorder := httptest.NewRecorder()
+		state.ServeHTTP(recorder, r)
+		return recorder.Result(), nil
+	})
+}

tests/backend.go

@@ -0,0 +1,57 @@
+package tests
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"strconv"
+)
+
+func MakeTestBackend() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		responseCode := http.StatusOK
+		var err error
+		if opt := q.Get("http-code"); opt != "" {
+			rc, err := strconv.ParseInt(opt, 10, 64)
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			responseCode = int(rc)
+		}
+		type ResponseJson struct {
+			Method string `json:"method"`
+			Path   string `json:"path"`
+			Query  string `json:"query"`
+		}
+
+		if opt := q.Get("mime-type"); opt != "" {
+			w.Header().Set("Content-Type", opt)
+		} else {
+			w.Header().Set("Content-Type", "application/json")
+		}
+
+		var data []byte
+		if opt := q.Get("content"); opt != "" {
+			data, err = base64.RawURLEncoding.DecodeString(opt)
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+		} else {
+			data, err = json.Marshal(ResponseJson{
+				Method: r.Method,
+				Path:   r.URL.Path,
+				Query:  r.URL.RawQuery,
+			})
+			if err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+		}
+
+		w.WriteHeader(responseCode)
+		_, _ = w.Write(data)
+	})
+}

tests/challenge_test.go

@@ -0,0 +1,362 @@
+package tests
+
+import (
+	"encoding/hex"
+	"fmt"
+	challenge2 "git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"golang.org/x/net/html"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func setupDefaultSettings(t *testing.T) policy.StateSettings {
+	settings := DefaultSettings
+	slog.SetDefault(slog.New(initLogger(t)))
+
+	return settings
+}
+
+func TestChallengeCookie(t *testing.T) {
+	settings := setupDefaultSettings(t)
+
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+challenges:
+  "challenge-cookie":
+    runtime: "cookie"
+
+rules:
+  - name: catch-all
+    conditions: ["true"]
+    action: challenge
+    settings:
+      challenges: ["challenge-cookie"]
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	var expectedCode = http.StatusTemporaryRedirect
+
+	err = MakeGoAwayState(*pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		challenge, err := http.NewRequest(http.MethodGet, "/test", nil)
+		challenge.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		challengeResponse, err := do(challenge)
+		if err != nil {
+			return err
+		}
+		defer challengeResponse.Body.Close()
+		if challengeResponse.StatusCode != expectedCode {
+			return fmt.Errorf("expected challenge status code %d, got %d", expectedCode, challengeResponse.StatusCode)
+		} else if cookies := challengeResponse.Cookies(); len(cookies) == 0 {
+			return fmt.Errorf("expected set cookies to be non-empty, got none")
+		} else if challengeResponse.Header.Get("Location") == "" {
+			return fmt.Errorf("expected header 'Location' to be non-empty, got none")
+		}
+
+		solveLocation := challengeResponse.Header.Get("Location")
+
+		if !strings.HasPrefix(solveLocation, "/test") {
+			return fmt.Errorf("expected next location to start with '/test', got %s", solveLocation)
+		}
+
+		// test pass
+		pass, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		pass.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+		for _, c := range challengeResponse.Cookies() {
+			pass.AddCookie(c)
+		}
+
+		response, err := do(pass)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected pass status code %d, got %d", http.StatusOK, response.StatusCode)
+		}
+
+		// test failure
+		fail, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		fail.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err = do(fail)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusForbidden {
+			return fmt.Errorf("expected fail status code %d, got %d", http.StatusForbidden, response.StatusCode)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestChallengeHeaderRefresh(t *testing.T) {
+	settings := setupDefaultSettings(t)
+
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+challenges:
+  "challenge-header-refresh":
+    runtime: "refresh"
+    parameters:
+      refresh-via: "header"
+
+rules:
+  - name: catch-all
+    conditions: ["true"]
+    action: challenge
+    settings:
+      challenges: ["challenge-header-refresh"]
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	var expectedCode = settings.ChallengeResponseCode
+
+	err = MakeGoAwayState(*pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		challenge, err := http.NewRequest(http.MethodGet, "/test", nil)
+		challenge.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		challengeResponse, err := do(challenge)
+		if err != nil {
+			return err
+		}
+		defer challengeResponse.Body.Close()
+		if challengeResponse.StatusCode != expectedCode {
+			return fmt.Errorf("expected challenge status code %d, got %d", expectedCode, challengeResponse.StatusCode)
+		} else if challengeResponse.Header.Get("Refresh") == "" {
+			return fmt.Errorf("expected header 'Refresh' to be non-empty, got none")
+		}
+
+		solveLocation, err := url.QueryUnescape(strings.Split(challengeResponse.Header.Get("Refresh"), "; url=")[1])
+		if err != nil {
+			return err
+		}
+
+		// test solve
+		solve, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		solve.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err := do(solve)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusTemporaryRedirect {
+			return fmt.Errorf("expected solve status code %d, got %d", http.StatusTemporaryRedirect, response.StatusCode)
+		} else if cookies := response.Cookies(); len(cookies) == 0 {
+			return fmt.Errorf("expected set cookies to be non-empty, got none")
+		} else if response.Header.Get("Location") == "" {
+			return fmt.Errorf("expected header 'Location' to be non-empty, got none")
+		} else if !strings.HasPrefix(response.Header.Get("Location"), "/test") {
+			return fmt.Errorf("expected next location to start with '/test', got %s", response.Header.Get("Location"))
+		}
+
+		// test pass
+		pass, err := http.NewRequest(http.MethodGet, response.Header.Get("Location"), nil)
+		pass.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+		for _, c := range response.Cookies() {
+			pass.AddCookie(c)
+		}
+
+		response, err = do(pass)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected pass status code %d, got %d", http.StatusOK, response.StatusCode)
+		}
+
+		// test failure
+		uri, err := url.Parse(solveLocation)
+		q := uri.Query()
+		q.Set(challenge2.QueryArgToken, hex.EncodeToString(make([]byte, challenge2.KeySize)))
+		uri.RawQuery = q.Encode()
+
+		fail, err := http.NewRequest(http.MethodGet, uri.String(), nil)
+		fail.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err = do(fail)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusBadRequest {
+			return fmt.Errorf("expected fail status code %d, got %d", http.StatusBadRequest, response.StatusCode)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestChallengeMetaRefresh(t *testing.T) {
+	settings := setupDefaultSettings(t)
+
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+challenges:
+  "challenge-meta-refresh":
+    runtime: "refresh"
+    parameters:
+      refresh-via: "meta"
+
+rules:
+  - name: catch-all
+    conditions: ["true"]
+    action: challenge
+    settings:
+      challenges: ["challenge-meta-refresh"]
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	var expectedCode = settings.ChallengeResponseCode
+
+	err = MakeGoAwayState(*pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		challenge, err := http.NewRequest(http.MethodGet, "/test", nil)
+		challenge.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		challengeResponse, err := do(challenge)
+		if err != nil {
+			return err
+		}
+		defer challengeResponse.Body.Close()
+		if challengeResponse.StatusCode != expectedCode {
+			return fmt.Errorf("expected challenge status code %d, got %d", expectedCode, challengeResponse.StatusCode)
+		} else if challengeResponse.Header.Get("Refresh") != "" {
+			return fmt.Errorf("expected header 'Refresh' to be empty, got \"%s\"", challengeResponse.Header.Get("Refresh"))
+		}
+
+		node, err := html.ParseWithOptions(challengeResponse.Body, html.ParseOptionEnableScripting(false))
+		if err != nil {
+			return nil
+		}
+
+		var refresh string
+		for n := range node.Descendants() {
+			if n.Type == html.ElementNode && n.Data == "meta" {
+				var is bool
+				var val string
+				for _, a := range n.Attr {
+					if a.Key == "http-equiv" && a.Val == "refresh" {
+						is = true
+					}
+					if a.Key == "content" {
+						val = a.Val
+					}
+				}
+				if is {
+					refresh = val
+					break
+				}
+			}
+		}
+
+		solveLocation, err := url.QueryUnescape(strings.Split(refresh, "; url=")[1])
+		if err != nil {
+			return err
+		}
+
+		// test solve
+		solve, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		solve.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err := do(solve)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusTemporaryRedirect {
+			return fmt.Errorf("expected solve status code %d, got %d", http.StatusTemporaryRedirect, response.StatusCode)
+		} else if cookies := response.Cookies(); len(cookies) == 0 {
+			return fmt.Errorf("expected set cookies to be non-empty, got none")
+		} else if response.Header.Get("Location") == "" {
+			return fmt.Errorf("expected header 'Location' to be non-empty, got none")
+		} else if !strings.HasPrefix(response.Header.Get("Location"), "/test") {
+			return fmt.Errorf("expected next location to start with '/test', got %s", response.Header.Get("Location"))
+		}
+
+		// test pass
+		pass, err := http.NewRequest(http.MethodGet, response.Header.Get("Location"), nil)
+		pass.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+		for _, c := range response.Cookies() {
+			pass.AddCookie(c)
+		}
+
+		response, err = do(pass)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected pass status code %d, got %d", http.StatusOK, response.StatusCode)
+		}
+
+		// test failure
+		uri, err := url.Parse(solveLocation)
+		q := uri.Query()
+		q.Set(challenge2.QueryArgToken, hex.EncodeToString(make([]byte, challenge2.KeySize)))
+		uri.RawQuery = q.Encode()
+
+		fail, err := http.NewRequest(http.MethodGet, uri.String(), nil)
+		fail.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err = do(fail)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusBadRequest {
+			return fmt.Errorf("expected fail status code %d, got %d", http.StatusBadRequest, response.StatusCode)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

tests/logger_test.go

@@ -0,0 +1,57 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"testing"
+)
+
+type logger struct {
+	t     *testing.T
+	attrs []slog.Attr
+}
+
+func (l logger) Enabled(ctx context.Context, level slog.Level) bool {
+	return true
+}
+
+func (l logger) Handle(ctx context.Context, record slog.Record) error {
+	str := fmt.Sprintf("[%s] %s", record.Level, record.Message)
+
+	if record.NumAttrs() > 0 || len(l.attrs) > 0 {
+		str += ": "
+	}
+	for _, attr := range l.attrs {
+		str += fmt.Sprintf("%s=%s ", attr.Key, attr.Value.String())
+	}
+	record.Attrs(func(attr slog.Attr) bool {
+		str += fmt.Sprintf("%s=%s ", attr.Key, attr.Value.String())
+		return true
+	})
+
+	if record.Level == slog.LevelError {
+		l.t.Error(str)
+	} else {
+		l.t.Log(str)
+	}
+	return nil
+}
+
+func (l logger) WithAttrs(attrs []slog.Attr) slog.Handler {
+	newAttrs := make([]slog.Attr, 0, len(attrs)+len(l.attrs))
+	newAttrs = append(newAttrs, l.attrs...)
+	newAttrs = append(newAttrs, attrs...)
+	return logger{
+		t:     l.t,
+		attrs: newAttrs,
+	}
+}
+
+func (l logger) WithGroup(name string) slog.Handler {
+	return l
+}
+
+func initLogger(t *testing.T) slog.Handler {
+	return logger{t: t}
+}

utils/decaymap.go

@@ -10,17 +10,17 @@ func zilch[T any]() T {
 	return zero
 }
 
-type DecayMap[K, V comparable] struct {
+type DecayMap[K comparable, V any] struct {
 	data map[K]DecayMapEntry[V]
 	lock sync.RWMutex
 }
 
-type DecayMapEntry[V comparable] struct {
+type DecayMapEntry[V any] struct {
 	Value  V
 	expiry time.Time
 }
 
-func NewDecayMap[K, V comparable]() *DecayMap[K, V] {
+func NewDecayMap[K comparable, V any]() *DecayMap[K, V] {
 	return &DecayMap[K, V]{
 		data: make(map[K]DecayMapEntry[V]),
 	}

utils/radb.go

@@ -53,6 +53,9 @@ func (db *RADb) query(fn func(n int, record []byte) error, queries ...string) er
 
 	scanner := bufio.NewScanner(conn)
 	scanner.Split(bufio.ScanLines)
+	// 16 MiB lines
+	const bufferSize = 1024 * 1024 * 16
+	scanner.Buffer(make([]byte, 0, bufferSize), bufferSize)
 
 	for _, q := range queries {
 
@@ -76,6 +79,10 @@ func (db *RADb) query(fn func(n int, record []byte) error, queries ...string) er
 			}
 			n++
 		}
+
+		if scanner.Err() != nil {
+			return scanner.Err()
+		}
 	}
 
 	if len(queries) > 1 {
@@ -90,11 +97,6 @@ func (db *RADb) query(fn func(n int, record []byte) error, queries ...string) er
 	return nil
 }
 
-func init() {
-	db, _ := NewRADb()
-	db.FetchIPInfo(net.ParseIP("162.158.62.1"))
-}
-
 func (db *RADb) FetchIPInfo(ip net.IP) (result []string, err error) {
 	var ipNet net.IPNet
 	if ip4 := ip.To4(); ip4 != nil {

utils/tagfetcher.go

@@ -0,0 +1,59 @@
+package utils
+
+import (
+	"golang.org/x/net/html"
+	"io"
+	"mime"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+)
+
+func FetchTags(backend http.Handler, uri *url.URL, kind string) (result []html.Node) {
+	writer := httptest.NewRecorder()
+	backend.ServeHTTP(writer, &http.Request{
+		Method: http.MethodGet,
+		URL:    uri,
+		Header: http.Header{
+			"User-Agent": []string{"Mozilla 5.0 (compatible; go-away/1.0 fetch-tags) TwitterBot/1.0"},
+			"Accept":     []string{"text/html,application/xhtml+xml"},
+		},
+		Close: true,
+	})
+	response := writer.Result()
+	if response == nil {
+		return nil
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return nil
+	}
+
+	if contentType, _, _ := mime.ParseMediaType(response.Header.Get("Content-Type")); contentType != "text/html" && contentType != "application/xhtml+xml" {
+		return nil
+	}
+
+	return FetchTagsFromReader(response.Body, kind)
+}
+
+func FetchTagsFromReader(r io.Reader, kind string) (result []html.Node) {
+	//TODO: handle non UTF-8 documents
+	node, err := html.ParseWithOptions(r, html.ParseOptionEnableScripting(false))
+	if err != nil {
+		return nil
+	}
+
+	for n := range node.Descendants() {
+		if n.Type == html.ElementNode && n.Data == kind {
+			result = append(result, html.Node{
+				Type:      n.Type,
+				DataAtom:  n.DataAtom,
+				Data:      n.Data,
+				Namespace: n.Namespace,
+				Attr:      n.Attr,
+			})
+		}
+	}
+
+	return result
+}