testdata: Initial action/challenges testing

Requests using this user agent are coming from the same Amazon net-
works as Bytespider.

.drone.jsonnet

@@ -1,5 +1,5 @@
 // yaml_stream.jsonnet
-local Build(go, alpine, os, arch) = {
+local Build(mirror, go, alpine, os, arch) = {
     kind: "pipeline",
     type: "docker",
     name: "build-" + go + "-alpine" + alpine + "-" + arch,
@@ -8,25 +8,66 @@ local Build(go, alpine, os, arch) = {
         arch: arch
     },
     environment: {
+        GOTOOLCHAIN: "local",
         CGO_ENABLED: "0",
         GOOS: os,
         GOARCH: arch,
+        GORACE: "halt_on_error=1"
     },
     steps: [
         {
             name: "build",
             image: "golang:" + go +"-alpine" + alpine,
+            mirror: mirror,
             commands: [
                 "apk update",
                 "apk add --no-cache git",
                 "mkdir .bin",
-                "go build -v -o ./.bin/go-away ./cmd/go-away",
+                "go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away",
                 "go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime",
             ],
         },
+        {
+            name: "test",
+            image: "golang:" + go +"-alpine" + alpine,
+            mirror: mirror,
+            commands: [
+                "apk update",
+                "apk add --no-cache git",
+                "go test -p 1 -timeout 20m -v  ./tests/"
+            ],
+        },
+        {
+            name: "check-policy-forgejo",
+            image: "alpine:" + alpine,
+            mirror: mirror,
+            depends_on: ["build"],
+            commands: [
+                "./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/forgejo.yml --policy-snippets examples/snippets/"
+            ],
+        },
+        {
+            name: "check-policy-generic",
+            image: "alpine:" + alpine,
+            mirror: mirror,
+            depends_on: ["build"],
+            commands: [
+                "./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/generic.yml --policy-snippets examples/snippets/"
+            ],
+        },
+        {
+            name: "check-policy-spa",
+            image: "alpine:" + alpine,
+            mirror: mirror,
+            depends_on: ["build"],
+            commands: [
+                "./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/spa.yml --policy-snippets examples/snippets/"
+            ],
+        },
         {
             name: "test-wasm-success",
             image: "alpine:" + alpine,
+            mirror: mirror,
             depends_on: ["build"],
             commands: [
                 "./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm " +
@@ -39,6 +80,7 @@ local Build(go, alpine, os, arch) = {
         {
             name: "test-wasm-fail",
             image: "alpine:" + alpine,
+            mirror: mirror,
             depends_on: ["build"],
             commands: [
                 "./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm " +
@@ -51,39 +93,63 @@ local Build(go, alpine, os, arch) = {
     ]
 };
 
-local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
+local Publish(mirror, registry, repo, secret, go, alpine, os, arch, trigger, platforms, extra) = {
     kind: "pipeline",
     type: "docker",
-    name: "publish-" + go + "-alpine" + alpine,
+    name: "publish-" + go + "-alpine" + alpine + "-" + secret,
     platform: {
         os: os,
         arch: arch,
     },
     trigger: trigger,
     steps: [
+        {
+            name: "test",
+            image: "golang:" + go +"-alpine" + alpine,
+            mirror: mirror,
+            commands: [
+                "apk update",
+                "apk add --no-cache git",
+                "go test -p 1 -timeout 20m -v  ./tests/"
+            ],
+        },
+        {
+            name: "setup-buildkitd",
+            image: "alpine:" + alpine,
+            mirror: mirror,
+            commands: [
+                "echo '[registry.\"docker.io\"]' > buildkitd.toml",
+                "echo '  mirrors = [\"mirror.gcr.io\"]' >> buildkitd.toml"
+            ],
+        },
         {
             name: "docker",
             image: "plugins/buildx",
             privileged: true,
             environment: {
-                DOCKER_BUILDKIT: "1"
+                DOCKER_BUILDKIT: "1",
+                SOURCE_DATE_EPOCH: 0,
+                TZ: "UTC",
+                LC_ALL: "C",
+                PLUGIN_BUILDER_CONFIG: "buildkitd.toml",
+                PLUGIN_BUILDER_DRIVER: "docker-container",
             },
             settings: {
-                  registry: "git.gammaspectra.live",
-                  repo: "git.gammaspectra.live/git/go-away",
+                  registry: registry,
+                  repo: repo,
+                  mirror: mirror,
                   compress: true,
                   platform: platforms,
-                  builder_driver: "docker-container",
                   build_args: {
                     from_builder: "golang:" + go +"-alpine" + alpine,
                     from: "alpine:" + alpine,
                   },
                   auto_tag_suffix: "alpine" + alpine,
                   username: {
-                    from_secret: "git_username",
+                    from_secret: secret + "_username",
                   },
                   password: {
-                    from_secret: "git_password",
+                    from_secret: secret + "_password",
                   },
             } + extra,
         },
@@ -91,16 +157,24 @@ local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
 };
 
 #
+local containerArchitectures = ["linux/amd64", "linux/arm64", "linux/riscv64"];
+
+local alpineVersion = "3.21";
+local goVersion = "1.24";
+
+local mirror = "https://mirror.gcr.io";
 
 [
-    Build("1.22", "3.20", "linux", "amd64"),
-    Build("1.22", "3.20", "linux", "arm64"),
-    Build("1.24", "3.21", "linux", "amd64"),
-    Build("1.24", "3.21", "linux", "arm64"),
+    Build(mirror, goVersion, alpineVersion, "linux", "amd64"),
+    Build(mirror, goVersion, alpineVersion, "linux", "arm64"),
 
     # latest
-    Publish("1.24", "3.21", "linux", "amd64", {event: ["push"], branch: ["master"], }, ["linux/amd64", "linux/arm64"], {tags: ["latest"],}) + {name: "publish-latest"},
-    Publish("1.24", "3.21", "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, ["linux/amd64", "linux/arm64"], {auto_tag: true,}),
-    # legacy
-    Publish("1.22", "3.20", "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, ["linux/amd64", "linux/arm64"], {auto_tag: true,}),
+    Publish(mirror, "git.gammaspectra.live", "git.gammaspectra.live/git/go-away", "git", goVersion, alpineVersion, "linux", "amd64", {event: ["push"], branch: ["master"], }, containerArchitectures, {tags: ["latest"],}) + {name: "publish-latest-git"},
+    Publish(mirror, "codeberg.org", "codeberg.org/gone/go-away", "codeberg", goVersion, alpineVersion, "linux", "amd64", {event: ["push"], branch: ["master"], }, containerArchitectures, {tags: ["latest"],}) + {name: "publish-latest-codeberg"},
+    Publish(mirror, "ghcr.io", "ghcr.io/weebdatahoarder/go-away", "github", goVersion, alpineVersion, "linux", "amd64", {event: ["push"], branch: ["master"], }, containerArchitectures, {tags: ["latest"],}) + {name: "publish-latest-github"},
+
+    # modern
+    Publish(mirror, "git.gammaspectra.live", "git.gammaspectra.live/git/go-away", "git", goVersion, alpineVersion, "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, containerArchitectures, {auto_tag: true,}),
+    Publish(mirror, "codeberg.org", "codeberg.org/gone/go-away", "codeberg", goVersion, alpineVersion, "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, containerArchitectures, {auto_tag: true,}),
+    Publish(mirror, "ghcr.io", "ghcr.io/weebdatahoarder/go-away", "github", goVersion, alpineVersion, "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, containerArchitectures, {auto_tag: true,}),
 ]

.drone.yml

@@ -3,86 +3,8 @@ environment:
   CGO_ENABLED: "0"
   GOARCH: amd64
   GOOS: linux
-kind: pipeline
-name: build-1.22-alpine3.20-amd64
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - apk update
-  - apk add --no-cache git
-  - mkdir .bin
-  - go build -v -o ./.bin/go-away ./cmd/go-away
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
-  image: golang:1.22-alpine3.20
-  name: build
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge.json -verify-challenge-out
-    0
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-success
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge-fail.json -verify-challenge-out
-    1
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-fail
-type: docker
----
-environment:
-  CGO_ENABLED: "0"
-  GOARCH: arm64
-  GOOS: linux
-kind: pipeline
-name: build-1.22-alpine3.20-arm64
-platform:
-  arch: arm64
-  os: linux
-steps:
-- commands:
-  - apk update
-  - apk add --no-cache git
-  - mkdir .bin
-  - go build -v -o ./.bin/go-away ./cmd/go-away
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
-  image: golang:1.22-alpine3.20
-  name: build
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge.json -verify-challenge-out
-    0
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-success
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge-fail.json -verify-challenge-out
-    1
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-fail
-type: docker
----
-environment:
-  CGO_ENABLED: "0"
-  GOARCH: amd64
-  GOOS: linux
+  GORACE: halt_on_error=1
+  GOTOOLCHAIN: local
 kind: pipeline
 name: build-1.24-alpine3.21-amd64
 platform:
@@ -93,10 +15,42 @@ steps:
   - apk update
   - apk add --no-cache git
   - mkdir .bin
-  - go build -v -o ./.bin/go-away ./cmd/go-away
+  - go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away
   - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
   image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
   name: build
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/forgejo.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-forgejo
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/generic.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-generic
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/spa.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-spa
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
     -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
@@ -106,6 +60,7 @@ steps:
   depends_on:
   - build
   image: alpine:3.21
+  mirror: https://mirror.gcr.io
   name: test-wasm-success
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
@@ -116,6 +71,7 @@ steps:
   depends_on:
   - build
   image: alpine:3.21
+  mirror: https://mirror.gcr.io
   name: test-wasm-fail
 type: docker
 ---
@@ -123,6 +79,8 @@ environment:
   CGO_ENABLED: "0"
   GOARCH: arm64
   GOOS: linux
+  GORACE: halt_on_error=1
+  GOTOOLCHAIN: local
 kind: pipeline
 name: build-1.24-alpine3.21-arm64
 platform:
@@ -133,10 +91,42 @@ steps:
   - apk update
   - apk add --no-cache git
   - mkdir .bin
-  - go build -v -o ./.bin/go-away ./cmd/go-away
+  - go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away
   - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
   image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
   name: build
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/forgejo.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-forgejo
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/generic.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-generic
+- commands:
+  - ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80
+    --policy examples/spa.yml --policy-snippets examples/snippets/
+  depends_on:
+  - build
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: check-policy-spa
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
     -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
@@ -146,6 +136,7 @@ steps:
   depends_on:
   - build
   image: alpine:3.21
+  mirror: https://mirror.gcr.io
   name: test-wasm-success
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
@@ -156,17 +147,36 @@ steps:
   depends_on:
   - build
   image: alpine:3.21
+  mirror: https://mirror.gcr.io
   name: test-wasm-fail
 type: docker
 ---
 kind: pipeline
-name: publish-latest
+name: publish-latest-git
 platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - echo '[registry."docker.io"]' > buildkitd.toml
+  - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: setup-buildkitd
 - environment:
     DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    PLUGIN_BUILDER_CONFIG: buildkitd.toml
+    PLUGIN_BUILDER_DRIVER: docker-container
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
   image: plugins/buildx
   name: docker
   privileged: true
@@ -175,13 +185,14 @@ steps:
     build_args:
       from: alpine:3.21
       from_builder: golang:1.24-alpine3.21
-    builder_driver: docker-container
     compress: true
+    mirror: https://mirror.gcr.io
     password:
       from_secret: git_password
     platform:
     - linux/amd64
     - linux/arm64
+    - linux/riscv64
     registry: git.gammaspectra.live
     repo: git.gammaspectra.live/git/go-away
     tags:
@@ -196,13 +207,141 @@ trigger:
 type: docker
 ---
 kind: pipeline
-name: publish-1.24-alpine3.21
+name: publish-latest-codeberg
 platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - echo '[registry."docker.io"]' > buildkitd.toml
+  - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: setup-buildkitd
 - environment:
     DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    PLUGIN_BUILDER_CONFIG: buildkitd.toml
+    PLUGIN_BUILDER_DRIVER: docker-container
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    compress: true
+    mirror: https://mirror.gcr.io
+    password:
+      from_secret: codeberg_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    - linux/riscv64
+    registry: codeberg.org
+    repo: codeberg.org/gone/go-away
+    tags:
+    - latest
+    username:
+      from_secret: codeberg_username
+trigger:
+  branch:
+  - master
+  event:
+  - push
+type: docker
+---
+kind: pipeline
+name: publish-latest-github
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - echo '[registry."docker.io"]' > buildkitd.toml
+  - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: setup-buildkitd
+- environment:
+    DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    PLUGIN_BUILDER_CONFIG: buildkitd.toml
+    PLUGIN_BUILDER_DRIVER: docker-container
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    compress: true
+    mirror: https://mirror.gcr.io
+    password:
+      from_secret: github_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    - linux/riscv64
+    registry: ghcr.io
+    repo: ghcr.io/weebdatahoarder/go-away
+    tags:
+    - latest
+    username:
+      from_secret: github_username
+trigger:
+  branch:
+  - master
+  event:
+  - push
+type: docker
+---
+kind: pipeline
+name: publish-1.24-alpine3.21-git
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - echo '[registry."docker.io"]' > buildkitd.toml
+  - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: setup-buildkitd
+- environment:
+    DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    PLUGIN_BUILDER_CONFIG: buildkitd.toml
+    PLUGIN_BUILDER_DRIVER: docker-container
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
   image: plugins/buildx
   name: docker
   privileged: true
@@ -212,13 +351,14 @@ steps:
     build_args:
       from: alpine:3.21
       from_builder: golang:1.24-alpine3.21
-    builder_driver: docker-container
     compress: true
+    mirror: https://mirror.gcr.io
     password:
       from_secret: git_password
     platform:
     - linux/amd64
     - linux/arm64
+    - linux/riscv64
     registry: git.gammaspectra.live
     repo: git.gammaspectra.live/git/go-away
     username:
@@ -232,33 +372,107 @@ trigger:
 type: docker
 ---
 kind: pipeline
-name: publish-1.22-alpine3.20
+name: publish-1.24-alpine3.21-codeberg
 platform:
   arch: amd64
   os: linux
 steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - echo '[registry."docker.io"]' > buildkitd.toml
+  - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: setup-buildkitd
 - environment:
     DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    PLUGIN_BUILDER_CONFIG: buildkitd.toml
+    PLUGIN_BUILDER_DRIVER: docker-container
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
   image: plugins/buildx
   name: docker
   privileged: true
   settings:
     auto_tag: true
-    auto_tag_suffix: alpine3.20
+    auto_tag_suffix: alpine3.21
     build_args:
-      from: alpine:3.20
-      from_builder: golang:1.22-alpine3.20
-    builder_driver: docker-container
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
     compress: true
+    mirror: https://mirror.gcr.io
     password:
-      from_secret: git_password
+      from_secret: codeberg_password
     platform:
     - linux/amd64
     - linux/arm64
-    registry: git.gammaspectra.live
-    repo: git.gammaspectra.live/git/go-away
+    - linux/riscv64
+    registry: codeberg.org
+    repo: codeberg.org/gone/go-away
     username:
-      from_secret: git_username
+      from_secret: codeberg_username
+trigger:
+  event:
+  - promote
+  - tag
+  target:
+  - production
+type: docker
+---
+kind: pipeline
+name: publish-1.24-alpine3.21-github
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk update
+  - apk add --no-cache git
+  - go test -p 1 -timeout 20m -v  ./tests/
+  image: golang:1.24-alpine3.21
+  mirror: https://mirror.gcr.io
+  name: test
+- commands:
+  - echo '[registry."docker.io"]' > buildkitd.toml
+  - echo '  mirrors = ["mirror.gcr.io"]' >> buildkitd.toml
+  image: alpine:3.21
+  mirror: https://mirror.gcr.io
+  name: setup-buildkitd
+- environment:
+    DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    PLUGIN_BUILDER_CONFIG: buildkitd.toml
+    PLUGIN_BUILDER_DRIVER: docker-container
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag: true
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    compress: true
+    mirror: https://mirror.gcr.io
+    password:
+      from_secret: github_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    - linux/riscv64
+    registry: ghcr.io
+    repo: ghcr.io/weebdatahoarder/go-away
+    username:
+      from_secret: github_username
 trigger:
   event:
   - promote
@@ -268,6 +482,6 @@ trigger:
 type: docker
 ---
 kind: signature
-hmac: 3cbd114d368c7bd348105921d85c703db1c1bc46de79f00daabbca23ffac6050
+hmac: 07ac33f9298a9910aacb29ef18931cb999841f76be8a95ca210f9f3704c347f9
 
 ...

CHALLENGES.md

@@ -0,0 +1,132 @@
+# Challenges
+
+Challenges can be [transparent](#transparent) (not shown to user, depends on backend or other logic), [non-JavaScript](#non-javascript) (challenges common browser properties), or [custom JavaScript](README.md#custom-javascript) (from Proof of Work to fingerprinting or Captcha is supported)
+
+## Transparent
+
+### http
+
+Verify incoming requests against a specified backend to allow the user through. Cookies and some other headers are passed.
+
+For example, this allows verifying the user cookies against the backend to have the user skip all other challenges.
+
+Example on Forgejo, checks that current user is authenticated:
+```yaml
+  http-cookie-check:
+    runtime: http
+    parameters:
+      http-url: http://forgejo:3000/user/stopwatches
+      # http-url: http://forgejo:3000/repo/search
+      # http-url: http://forgejo:3000/notifications/new
+      http-method: GET
+      http-cookie: i_like_gitea
+      http-code: 200
+      verify-probability: 0.1
+```
+
+### preload-link
+
+Requires HTTP/2+ response parsing and logic, silent challenge (does not display a challenge page).
+
+Browsers that support [103 Early Hints](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/103) are indicated to fetch a CSS resource via [Link](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link) preload that solves the challenge.
+
+The server waits until solved or defined timeout, then continues on other challenges if failed.
+
+Example:
+```yaml
+  preload-link:
+    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
+    runtime: "preload-link"
+    parameters:
+      preload-early-hint-deadline: 3s
+```
+
+### dnsbl
+
+You can configure a [DNSBL (Domain Name System blocklist)](https://en.wikipedia.org/wiki/Domain_Name_System_blocklist) to be queried.
+
+This allows you to serve harder or different challenges to higher risk clients, or block them from specific sections.
+
+Only rules that match a DNSBL challenge will cause a query to be sent, meaning the bulk of requests will not be sent to this service upstream.
+
+Results will be temporarily cached.
+
+By default, [DroneBL](https://dronebl.org/) is used.
+
+Example challenge definition and rule:
+```yaml
+challenges:
+  dnsbl:
+  runtime: dnsbl
+  parameters:
+    # dnsbl-host: "dnsbl.dronebl.org"
+    dnsbl-decay: 1h
+    dnsbl-timeout: 1s
+    
+rules:
+  # check DNSBL and serve harder challenges
+  - name: undesired-dnsbl
+    action: check
+    settings:
+      challenges: [dnsbl]
+      # if DNSBL fails, check additional challenges
+      fail: check
+      fail-settings:
+        challenges: [js-pow-sha256]
+```
+
+## Non-JavaScript
+
+### cookie
+
+Requires HTTP parsing and a Cookie Jar, silent challenge (does not display a challenge page unless failed).
+
+Serves the client with a Set-Cookie that solves the challenge, and redirects it back to the same page. Browser must present the cookie to load.
+
+Several tools implement this, but usually not mass scrapers.
+
+### header-refresh
+
+Requires HTTP response parsing and logic, displays challenge site instantly.
+
+Have the browser solve the challenge by following the URL listed on HTTP [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh) instantly.
+
+
+### meta-refresh
+
+Requires HTTP and HTML response parsing and logic, displays challenge site instantly.
+
+Have the browser solve the challenge by following the URL listed on HTML `<meta http-equiv=refresh>` tag instantly. Equivalent to above.
+
+### resource-load
+
+Requires HTTP and HTML response parsing and logic, displays challenge site.
+
+Servers a challenge page with a linked resource that is loaded by the browser, which solves the challenge. Page refreshes a few seconds later via [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh).
+
+## Custom JavaScript
+
+### js-pow-sha256
+
+Requires JavaScript and workers, displays challenge site.
+
+Has the user solve a Proof of Work using SHA256 hashes, with configurable difficulty.
+
+Example:
+```yaml
+  js-pow-sha256:
+    runtime: js
+    parameters:
+      # specifies the folder path that assets are under
+      # can be either embedded or external path
+      # defaults to name of challenge
+      path: "js-pow-sha256"
+      # needs to be under static folder
+      js-loader: load.mjs
+      # needs to be under runtime folder
+      wasm-runtime: runtime.wasm
+      wasm-runtime-settings:
+        difficulty: 20
+      verify-probability: 0.02
+```
+

Dockerfile

@@ -1,5 +1,5 @@
-ARG from_builder=golang:1.24-alpine3.21
-ARG from=alpine:3.21
+ARG from_builder=docker.io/golang:1.24-alpine3.21
+ARG from=docker.io/alpine:3.21
 
 ARG BUILDPLATFORM
 
@@ -7,6 +7,7 @@ FROM --platform=$BUILDPLATFORM ${from_builder} AS build
 
 ARG TARGETARCH
 ARG TARGETOS
+ARG GOTOOLCHAIN=local
 
 RUN apk update && apk add --no-cache \
     bash \
@@ -22,35 +23,45 @@ RUN ./build-compress.sh
 ENV CGO_ENABLED=0
 ENV GOOS=${TARGETOS}
 ENV GOARCH=${TARGETARCH}
+ENV GOTOOLCHAIN=${GOTOOLCHAIN}
 
-RUN go build -pgo=auto -v -trimpath -o "${GOBIN}/go-away" ./cmd/go-away
+RUN go build -pgo=auto -v -trimpath -ldflags=-buildid= -o "${GOBIN}/go-away" ./cmd/go-away
 RUN test -e "${GOBIN}/go-away"
 
 
 FROM --platform=$TARGETPLATFORM ${from}
 
 COPY --from=build /go/bin/go-away /bin/go-away
+COPY examples/snippets/ /snippets/
+COPY docker-entrypoint.sh /
 
 ENV TZ UTC
 
+ENV GOAWAY_METRICS_BIND=""
+ENV GOAWAY_DEBUG_BIND=""
+
 ENV GOAWAY_BIND=":8080"
 ENV GOAWAY_BIND_NETWORK="tcp"
 ENV GOAWAY_SOCKET_MODE="0770"
+ENV GOAWAY_CONFIG=""
 ENV GOAWAY_POLICY="/policy.yml"
+ENV GOAWAY_POLICY_SNIPPETS=""
 ENV GOAWAY_CHALLENGE_TEMPLATE="anubis"
 ENV GOAWAY_CHALLENGE_TEMPLATE_THEME=""
 ENV GOAWAY_SLOG_LEVEL="WARN"
 ENV GOAWAY_CLIENT_IP_HEADER=""
+ENV GOAWAY_BACKEND_IP_HEADER=""
 ENV GOAWAY_JWT_PRIVATE_KEY_SEED=""
 ENV GOAWAY_BACKEND=""
+ENV GOAWAY_ACME_AUTOCERT=""
+ENV GOAWAY_CACHE="/cache"
+
 
 EXPOSE 8080/tcp
 EXPOSE 8080/udp
+EXPOSE 9090/tcp
+EXPOSE 6060/tcp
 
 ENV JWT_PRIVATE_KEY_SEED="${GOAWAY_JWT_PRIVATE_KEY_SEED}"
 
-ENTRYPOINT  /bin/go-away --bind ${GOAWAY_BIND} --bind-network ${GOAWAY_BIND_NETWORK} --socket-mode ${GOAWAY_SOCKET_MODE} \
-            --policy ${GOAWAY_POLICY} --client-ip-header ${GOAWAY_CLIENT_IP_HEADER} \
-            --challenge-template ${GOAWAY_CHALLENGE_TEMPLATE} --challenge-template-theme ${GOAWAY_CHALLENGE_TEMPLATE_THEME} \
-            --slog-level ${GOAWAY_SLOG_LEVEL} \
-            --backend ${GOAWAY_BACKEND}
+ENTRYPOINT ["/docker-entrypoint.sh"]

README.md

@@ -0,0 +1,498 @@
+### <a id=why></a>
+# go-away
+
+Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots. Uses conventional non-nuclear options.
+
+[![Build Status](https://ci.gammaspectra.live/api/badges/git/go-away/status.svg)](https://ci.gammaspectra.live/git/go-away)
+[![Go Reference](https://pkg.go.dev/badge/git.gammaspectra.live/git/go-away.svg)](https://pkg.go.dev/git.gammaspectra.live/git/go-away)
+
+go-away sits in between your site and the Internet / upstream proxy.
+
+Incoming requests can be selected by [rules](#rich-rule-matching) to be [actioned](#extended-rule-actions) or [challenged](CHALLENGES.md#challenges) to filter suspicious requests.
+
+The tool is designed highly flexible so the operator can minimize impact to legit users, while surgically targeting heavy endpoints or scrapers.
+
+[Challenges](CHALLENGES.md#challenges) can be transparent (not shown to user, depends on backend or other logic), [non-JavaScript](#non-javascript-challenges) (challenges common browser properties), or [custom JavaScript](#custom-javascript-wasm-challenges) (from Proof of Work to fingerprinting or Captcha is supported)
+
+See _[Why do this?](#why-do-this)_ section for the challenges and reasoning behind this tool. 
+
+This documentation and go-away are in active development. See [What's left?](#what-s-left) section for a breakdown.
+
+## Support
+
+If you have some suggestion or issue, feel free to open a [New Issue](https://git.gammaspectra.live/git/go-away/issues/new) on the repository. 
+
+[Pull Requests](https://git.gammaspectra.live/git/go-away/pulls) are encouraged and desired.
+
+For real-time chat and other support join IRC on [#go-away](ircs://irc.libera.chat/#go-away) on Libera.Chat [[WebIRC]](https://web.libera.chat/?nick=Guest?#go-away). The channel may not be monitored at all times, feel free to ping the operators there.
+
+## Code Mirrors
+
+Source code is automatically pushed to the following mirrors. Packages are also mirrored on Codeberg and GitHub.
+
+[![GammaSpectra.live](https://img.shields.io/badge/GammaSpectra.live-main+packages-green?style=flat&logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgd2lkdGg9IjYwMCIgaGVpZ2h0PSI2MDAiIHZpZXdCb3g9Ii0zMDAgLTMwMCA2MDAgNjAwIj4KPGNpcmNsZSByPSI1MCIvPgo8cGF0aCBkPSJNNzUsMCBBIDc1LDc1IDAgMCwwIDM3LjUsLTY0Ljk1MiBMIDEyNSwtMjE2LjUwNiBBIDI1MCwyNTAgMCAwLDEgMjUwLDAgeiIgaWQ9ImJsZCIvPgo8dXNlIHhsaW5rOmhyZWY9IiNibGQiIHRyYW5zZm9ybT0icm90YXRlKDEyMCkiLz4KPHVzZSB4bGluazpocmVmPSIjYmxkIiB0cmFuc2Zvcm09InJvdGF0ZSgyNDApIi8+Cjwvc3ZnPg==&labelColor=fff)](https://git.gammaspectra.live/git/go-away) ![](https://git.gammaspectra.live/git/go-away/badges/stars.svg?style=flat) [![](https://git.gammaspectra.live/git/go-away/badges/issues/open.svg?style=flat)](https://git.gammaspectra.live/git/go-away/issues?state=open) [![](https://git.gammaspectra.live/git/go-away/badges/pulls/open.svg?style=flat)](https://git.gammaspectra.live/git/go-away/pulls?state=open)
+
+[![Codeberg](https://img.shields.io/badge/Codeberg-mirror+packages-2185D0?style=flat&logo=codeberg&labelColor=fff)](https://codeberg.org/gone/go-away) ![](https://codeberg.org/gone/go-away/badges/stars.svg?style=flat)
+
+[![GitHub](https://img.shields.io/badge/GitHub-mirror+packages-blue?style=flat&logo=github&labelColor=fff&logoColor=24292f)](https://github.com/WeebDataHoarder/go-away) ![](https://img.shields.io/github/stars/WeebDataHoarder/go-away?style=flat)
+
+[![sourcehut](https://img.shields.io/badge/sourcehut-mirror-blue?style=flat&logo=sourcehut&labelColor=fff&logoColor=000)](https://git.sr.ht/~datahoarder/go-away)
+
+Note that issues or pull requests should be issued on the [main Forge](https://git.gammaspectra.live/git/go-away).
+
+## Features
+
+### Rich rule matching
+
+[Common Expression Language (CEL)](https://cel.dev/overview/cel-overview) is used to allow arbitrary selection of client properties, not only limited to regex. Boolean operators are supported.
+
+Templates can be defined in the Policy to allow reuse of such conditions on rule matching. Challenges can also be gated behind conditions.
+
+See the [CEL Language Definition](https://github.com/google/cel-spec/blob/master/doc/langdef.md) for the syntax.
+
+Rules and conditions are served with this environment:
+
+```
+remoteAddress (net.IP) - Connecting client remote address from headers or properties
+  remoteAddress.network(networkName string) bool - Check whether a given IP is listed on the underlying defined network
+  remoteAddress.network(networkCIDR string) bool - Check whether a given IP is listed on the CIDR
+host (string) - HTTP Host
+method (string) - HTTP Method/Verb
+userAgent (string) - HTTP User-Agent header
+path (string) - HTTP request Path
+query (map[string]string) - HTTP request Query arguments
+headers (map[string]string) - HTTP request headers
+fp (map[string]string) - Available fingerprints
+  
+Only available when TLS is enabled
+   fp.ja3n (string) JA3N TLS Fingerprint
+   fp.ja4 (string) JA4 TLS Fingerprint
+```
+
+### Template support
+
+Internal or external templates can be loaded to customize the look of the challenge or error page. Additionally, themes can be configured to change the look of these quickly.
+
+These templates are included by default:
+
+* `anubis`: An anubis-like themed challenge.
+* `forgejo`: Uses the Forgejo template and assets from your own instance. Supports specifying themes like `forgejo-auto`, `forgejo-light` and `forgejo-dark`.
+
+External templates for your site can be loaded specifying a full path to the `.gohtml` file. See [embed/templates/](embed/templates/) for examples to follow.
+
+You can alter the language and strings in the templates directly from the [config.yml](#config) file if specified.
+
+### Extended rule actions
+
+In addition to the common PASS / CHALLENGE / DENY rules, go-away offers more actions that can be extended via code.
+
+|  Action   | Behavior                                                                | Terminating |
+|:---------:|:------------------------------------------------------------------------|:-----------:|
+|   NONE    | Do nothing, continue. Useful for specifying on checks or challenges.    |     No      |
+|   PASS    | Passes the request to the backend immediately                           |     Yes     |
+|   DENY    | Denies the request with a descriptive page                              |     Yes     |
+|   BLOCK   | Denies the request with a response code                                 |     Yes     |
+|   DROP    | Drops the connection without sending a reply                            |     Yes     |
+| CHALLENGE | Issues a challenge that when passed, acts like PASS                     |     Yes     |
+|   CHECK   | Issues a challenge that when passed, continues executing rules          |     No      |
+|   PROXY   | Proxies request to a different backend, with optional path replacements |     Yes     |
+|  CONTEXT  | Modify the request context and apply different options                  |     No      |
+
+
+CHECK allows the client to be challenged but continue matching rules after these, for example, chaining a list of challenges that must be passed. 
+For example, you could use this to implement browser in checks without explicitly allowing all requests, and later deferring to a secondary check/challenge.
+
+PROXY allows the operator to send matching requests to a different backend, for example, a poison generator or a scraping maze.
+
+### Multiple challenge matching
+
+Several challenges can be offered as options for rules. This allows users that have passed other challenges before to not be affected.
+
+For example:
+```yaml
+  - name: standard-browser
+    action: challenge
+    settings:
+      challenges: [http-cookie-check, preload-link, meta-refresh, resource-load, js-pow-sha256]
+    conditions:
+      - '($is-generic-browser)'
+```
+
+This rule has the user be checked against a backend, then attempts pass a few browser challenges.
+
+In this case the processing would stop at `meta-refresh` due to the behavior of earlier challenges (cookie check and preload link allow failing / continue due to being silent, while meta-refresh requires displaying a challenge page).
+
+Any of these listed challenges being passed in the past will allow the client through, including non-offered `resource-load` and `js-pow-sha256`.
+
+### Non-Javascript challenges
+
+Several challenges that do not require JavaScript are offered, some targeting the HTTP stack and others a general browser behavior, or consulting with a backend service.
+
+These can be used for light checking of requests that eliminate most of the low effort scraping.
+
+See [Challenges](CHALLENGES.md#challenges) for a list of them.
+
+### Custom JavaScript / WASM challenges
+
+A WASM interface for server-side proof generation and checking is offered. We provide `js-pow-sha256` as an example of one.
+
+An internal test has shown you can implement Captchas or other browser fingerprinting tests within this interface.
+
+If you are interested in creating your own, see the [Development](#development) section below.
+
+### Upstream PROXY support
+
+Support for [HAProxy PROXY protocol](https://github.com/haproxy/haproxy/blob/master/doc/proxy-protocol.txt) can be enabled.
+
+This allows sending the client IP without altering the connection or HTTP headers.
+
+Supported by HAProxy, [Caddy](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#proxy_protocol), [nginx](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_protocol) and others.
+
+### Automatic TLS support and HTTP/2 support
+
+You can enable automatic certificate generation and TLS for the site via any ACME directory, which enables HTTP/2.
+
+Without TLS, HTTP/2 cleartext is supported, but you will need to configure the upstream proxy to send this protocol (`h2c://` on Caddy for example).
+
+
+### TLS Fingerprinting
+
+When running with TLS via autocert, TLS Fingerprinting of the incoming client is done.
+
+This can be targeted on conditions or other application logic.
+
+Read more about [JA3](https://medium.com/salesforce-engineering/tls-fingerprinting-with-ja3-and-ja3s-247362855967) and [JA4](https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/README.md).
+
+### Network range and automated filtering
+
+Some specific search spiders do follow _robots.txt_ and are well behaved. However, many actors can reuse user agents, so the origin network ranges must be checked.
+
+The samples provide example network range fetching and rules for Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot.
+
+Network ranges can be loaded via fetched JSON / TXT / HTML pages, or via lists. You can filter these using _jq_ or a regex.
+
+Example for _jq_:
+```yaml
+  aws-cloud:
+    - url: https://ip-ranges.amazonaws.com/ip-ranges.json
+      jq-path: '(.prefixes[] | select(has("ip_prefix")) | .ip_prefix), (.prefixes[] | select(has("ipv6_prefix")) | .ipv6_prefix)'
+```
+
+Example for _regex_:
+```yaml
+  cloudflare:
+    - url: https://www.cloudflare.com/ips-v4
+      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
+    - url: https://www.cloudflare.com/ips-v6
+      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"
+```
+
+
+### Sharing of signing seed across instances
+
+You can share the signing secret across multiple of your instances if you'd like to deploy multiple across the world.
+
+That way signed secrets will be verifiable across all the instances.
+
+By default, a random temporary key is generated every run.
+
+### Multiple backend support
+
+Multiple backends are supported, and rules specific on backend can be defined, and conditions and rules can match this as well.
+
+Subdomain wildcards like `*.example.com`, or full fallback wildcard `*` are supported.
+
+This allows one instance to run multiple domains or subdomains.
+
+### Package path
+
+You can modify the path where challenges are served and package name, if you don't want its presence to be easily discoverable.
+
+No source code editing or forking necessary!
+
+### IPv6 Happy Eyeballs challenge retry
+
+In case a client connects over IPv4 first then IPv6 due to [Fast Fallback / Happy Eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs), the challenge will automatically be retried.
+
+This is tracked by tagging challenges with a readable flag indicating the type of address.
+
+## Example policies
+
+### Forgejo
+
+The policy file at [examples/forgejo.yml](examples/forgejo.yml) provides a ready template to be used on your own Forgejo instance.
+
+Important notes:
+* Edit the `http-cookie-check` challenge, as this will fetch the listed backend with the given session cookie to check for user login.
+* Adjust the desired blocked networks or others. A template list of network ranges is provided, feel free to remove these if not needed.
+* Check the conditions and base rules to change your challenges offered and other ordering.
+* By default Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot are allowed by useragent and network ranges.
+
+### Generic
+
+The policy file at [examples/generic.yml](examples/generic.yml) provides a baseline to place on any site, that can be modified to fit your needs.
+
+Important notes:
+* Edit the `homesite` rule, as it's targeted to pages you always want to have available, like landing pages.
+* Edit the `is-static-asset` condition or the `allow-static-resources` rule to allow static file access as necessary.
+* If you have an API, add a PASS rule targeting it.
+* Check the conditions and base rules to change your challenges offered and other ordering.
+* Add or modify rules to target specific pages on your site as desired.
+* By default Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot are allowed by useragent and network ranges.
+
+### Snippets
+
+You can define snippets to be included. YAML anchors/aliases are supported.
+
+See [examples/snippets/](examples/snippets/) for some defaults including indexer bots, challenges and other general matches.
+
+
+
+## Why do this?
+In the past few years this small git instance has been hit by waves and waves of scraping.
+This was usually fought back by random useragent blocks for bots that did not follow [robots.txt](/robots.txt), until the past half year, where low-effort mass scraping was used more prominently.
+
+Recently these networks go from using residential IP blocks to sending requests at several hundred requests per second.
+
+If the server gets sluggish, more requests pile up. Even when denied they scrape for weeks later. Effectively spray and pray scraping, process later.
+
+At some point about 300Mbit/s of incoming requests (not including the responses) was hitting the server. And all of them nonsense URLs, or hitting archive/bundle downloads per commit.
+
+**If AI is so smart, why not just git clone the repositories?**
+
+* Wikimedia has posted about [How crawlers impact the operations of the Wikimedia projects](https://diff.wikimedia.org/2025/04/01/how-crawlers-impact-the-operations-of-the-wikimedia-projects/) [01/04/2025]
+
+* Xe (Anubis creator) has written about similar frustrations in several blogposts:
+  * [Amazon's AI crawler is making my git server unstable](https://xeiaso.net/notes/2025/amazon-crawler/) [01/17/2025]
+  * [Anubis works](https://xeiaso.net/notes/2025/anubis-works/) [04/12/2025]
+
+* Drew DeVault (sourcehut) has posted several articles and outages regarding the same issues:
+  * [Drew Blog: Please stop externalizing your costs directly into my face](https://drewdevault.com/2025/03/17/2025-03-17-Stop-externalizing-your-costs-on-me.html) [17/03/2025]
+    * (fun tidbit: I'm the one quoted as having the feedback discussion interrupted to deal with bots!)
+   * [sourcehut status: LLM crawlers continue to DDoS SourceHut](https://status.sr.ht/issues/2025-03-17-git.sr.ht-llms/) [17/03/2025]
+  * [sourcehut Blog: You cannot have our user's data](https://sourcehut.org/blog/2025-04-15-you-cannot-have-our-users-data/) [15/04/2025]
+
+* Others were also suffering at the same time [[1]](https://donotsta.re/notice/AreSNZlRlJv73AW7tI) [[2]](https://community.ipfire.org/t/suricata-ruleset-to-prevent-ai-scraping/11974) [[3]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[4]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[5]](https://blog.nytsoi.net/2025/03/01/obliterated-by-ai).
+
+---
+Initially I deployed Anubis, and yeah, it does work!
+
+This tool started as a way to replace [Anubis](https://anubis.techaro.lol/) as it was not found as featureful as desired, and the impact was too high.
+
+go-away may not be as straight to configure as Anubis but this was chosen to reduce impact on legitimate users, and offers many more options to dynamically target new waves.
+
+### Can't scrapers adapt?
+
+Yes, they can. At the moment their spray-and-pray approach is cheap for them.
+
+If they have to start adding an active browser in their scraping, that makes their collection expensive and slow.
+
+This would more or less eliminate the high rate low effort passive scraping and replace it with an active model.
+
+go-away offers a highly configurable set of challenges and rules that you can adapt to new ways.
+
+## What's left?
+
+go-away has most of the desired features from the original checklist that was made in its development. 
+However, a few points are left before go-away can be called v1.0.0:
+
+* [x] Several parts of the code are going through a refactor, which won't impact end users or operators.
+* [ ] Documentation is lacking and a more extensive one with inline example is in the works.
+* [x] Policy file syntax is going to stay mostly unchanged, except in the challenges definition section.
+* [ ] Allow end users to pick fallback challenges if any fail, specially with custom ones.
+* [ ] Replace Anubis-like default template with own one.
+* [x] Define strings and multi-language support for quick modification by operators without custom templates.
+* [ ] Have highly tested paths that match examples.
+* [x] Caching of temporary fetches, for example, network ranges.
+* [x] Allow live and dynamic policy reloading.
+* [x] Multiple domains / subdomains -> one backend handling, CEL rules for backends
+* [ ] Merge all rules and conditions into one large AST for higher performance.
+* [ ] Explore exposing a module for direct Caddy usage.
+* [x] More defined way of picking HTTP/HTTP(s) listeners and certificates.
+* [x] Expose metrics for challenge solve rates and acting on them.
+  * [ ] Metrics for common network ranges / AS / useragent
+
+## Setup
+
+go-away can take plaintext HTTP/1 and _HTTP/2_ / _h2c_ connections if desired over the same port. When doing this, it is recommended to have another reverse proxy above (for example [Caddy](https://caddyserver.com/), nginx, HAProxy) to handle HTTPs or similar.
+
+We also support the `autocert` parameter to configure HTTP(s). This will also allow TLS Fingerprinting to be done on incoming clients. This doesn't require any upstream proxies, and we recommend it's exposed directly or via SNI / Layer 4 proxying.
+
+### Config
+
+While most basic configuration can be passed via the command line, we support passing a [config.yml](examples/config.yml) with more advanced setup, including string replacement or custom backends configuration.
+
+### Binary / Go
+
+Requires Go 1.24+. Builds statically without CGo usage.
+
+We have Go 1.22+ support on the [go1.22 branch](https://git.gammaspectra.live/git/go-away/src/branch/go1.22).
+It will be regularly rebased to keep current with recent releases, at least until v1.0.0.
+Some features, such as TLS Fingerprinting, are not available on Go 1.22.
+
+```shell
+git clone https://git.gammaspectra.live/git/go-away.git && cd go-away
+
+CGO_ENABLED=0 go build -pgo=auto -v -trimpath -o ./go-away ./cmd/go-away
+
+# Run on port 8080, forwarding matching requests on git.example.com to http://forgejo:3000
+./go-away --bind :8080 \
+--backend git.example.com=http://forgejo:3000 \
+--policy examples/forgejo.yml \
+--challenge-template forgejo --challenge-template-theme forgejo-dark
+
+```
+
+### Dockerfile
+
+Available under [Dockerfile](Dockerfile). See the _docker compose_ below for the environment variables.
+
+### docker compose
+
+Example follows a hypothetical Forgejo server running on `http://forgejo:3000` serving `git.example.com`
+
+Container images are published under `git.gammaspectra.live/git/go-away`, `codeberg.org/gone/go-away` and `ghcr.io/weebdatahoarder/go-away`
+
+```yaml
+networks:
+  forgejo:
+    external: false
+    
+volumes:
+  goaway_cache:
+    
+services:
+  go-away:
+    # image: codeberg.org/gone/go-away:latest
+    # image: ghcr.io/weebdatahoarder/go-away:latest
+    image: git.gammaspectra.live/git/go-away:latest
+    restart: always
+    ports:
+      - "3000:8080"
+    networks:
+      - forgejo
+    depends_on:
+      - forgejo
+    volumes:
+      - "goaway_cache:/cache"
+      - "./examples/forgejo.yml:/policy.yml:ro"
+      #- "./your/snippets/:/policy/snippets/:ro"
+    environment:
+      #GOAWAY_BIND: ":8080"
+      # Supported tcp, unix, and proxy (for enabling PROXY module for request unwrapping)
+      #GOAWAY_BIND_NETWORK: "tcp"
+      #GOAWAY_SOCKET_MODE: "0770"
+
+      # Enable Prometheus metrics under /metrics on this bind
+      #GOAWAY_METRICS_BIND: ":9090"
+      # Enable Go debug profiles under this bind
+      #GOAWAY_DEBUG_BIND: ":6060"
+      
+      # set to letsencrypt or other directory URL to enable HTTPS. Above ports will be TLS only.
+      # enables request JA3N / JA4 client TLS fingerprinting
+      # TLS fingerprints are served on X-TLS-Fingerprint-JA3N and X-TLS-Fingerprint-JA4 headers
+      # TLS fingerprints can be matched against on CEL conditions
+      #GOAWAY_ACME_AUTOCERT: ""
+      
+      # Cache path for several services like certificates and caching network ranges
+      # Can be semi-ephemeral, recommended to be mapped to a permanent volume
+      #GOAWAY_CACHE="/cache"
+      
+      # default is WARN, set to INFO to also see challenge successes and others
+      #GOAWAY_SLOG_LEVEL: "INFO"
+      
+      # this value is used to sign cookies and challenges. by default a new one is generated each time
+      # set to generate to create one, then set the same value across all your instances
+      #GOAWAY_JWT_PRIVATE_KEY_SEED: ""
+      
+      # HTTP header that the client ip will be fetched from
+      # Defaults to the connection ip itself, if set here make sure your upstream proxy sets this properly
+      # Usually X-Forwarded-For is a good pick
+      # Not necessary with GOAWAY_BIND_NETWORK: proxy
+      GOAWAY_CLIENT_IP_HEADER: "X-Real-Ip"
+      
+      # HTTP header that go-away will set the obtained ip will be set to
+      # If left empty, the header on GOAWAY_CLIENT_IP_HEADER will be left as-is
+      #GOAWAY_BACKEND_IP_HEADER: ""
+      
+      # Alternate way of specifying parameters or more advanced settings
+      # Pass path to YAML file
+      #GOAWAY_CONFIG: ""
+      
+      GOAWAY_POLICY: "/policy.yml"
+
+      # Include extra snippets to load from this path.
+      # Note that the default snippets from example/snippets/ are included by default
+      #GOAWAY_POLICY_SNIPPETS: "/policy/snippets"
+      
+      # Template, and theme for the template to pick. defaults to an anubis-like one
+      # An file path can be specified. See embed/templates for a few examples
+      GOAWAY_CHALLENGE_TEMPLATE: forgejo
+      GOAWAY_CHALLENGE_TEMPLATE_THEME: forgejo-dark
+      
+      # Backend to match. Can be subdomain or full wildcards, "*.example.com" or "*"
+      GOAWAY_BACKEND: "git.example.com=http://forgejo:3000"
+      
+    # additional backends can be specified via more command arguments  
+    # command: ["--backend", "ci.example.com=http://ci:3000"]
+
+  forgejo:
+    # etc.
+
+```
+
+
+
+## Other Similar Projects
+
+|                                    Project                                    |                                                                                                                                                      Source Code                                                                                                                                                      | Description                                                                                             | Method                                       |
+|:-----------------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------|:---------------------------------------------|
+|                     [Anubis](https://anubis.techaro.lol/)                     |                                       [![GitHub](https://img.shields.io/badge/GitHub-TecharoHQ/anubis-blue?style=flat&logo=github&labelColor=fff&logoColor=24292f)](https://github.com/TecharoHQ/anubis)<br/>Go / [MIT](https://github.com/TecharoHQ/anubis/blob/main/LICENSE)                                        | Proxy that uses JavaScript proof of work to weight request based on simple match rules                  | JavaScript PoW (SHA-256)                     |
+|       [powxy](https://forge.lindenii.runxiyu.org/powxy/-/repos/powxy/)        |                  [![lindenii.runxiyu.org](https://img.shields.io/badge/lindenii-powxy-blue?style=flat&logo=git&labelColor=fff&logoColor=000)](https://forge.lindenii.runxiyu.org/powxy/-/repos/powxy/)<br/> Go / [BSD 2-Clause](https://forge.lindenii.runxiyu.org/powxy/-/repos/powxy/tree/LICENSE)                  | Powxy is a reverse proxy that protects your upstream service by challenging clients with proof-of-work. | JavaScript PoW (SHA-256) with manual program |
+| [PoW! Bot Deterrent](https://git.sequentialread.com/forest/pow-bot-deterrent) | [![SequentialRead](https://img.shields.io/badge/SequentialRead-forest/pow--bot--deterrent-blue?style=flat&logo=gitea&labelColor=fff&logoColor=000)](https://git.sequentialread.com/forest/pow-bot-deterrent)<br/> Go / [GPL v3.0](https://git.sequentialread.com/forest/pow-bot-deterrent/src/branch/main/LICENSE.md) | A proof-of-work based bot deterrent. Lightweight, self-hosted and copyleft licensed.                    | JavaScript PoW (WASM scrypt)                 |
+|                  [CSSWAF](https://github.com/yzqzss/csswaf)                   |                                            [![GitHub](https://img.shields.io/badge/GitHub-yzqzss/csswaf-blue?style=flat&logo=github&labelColor=fff&logoColor=24292f)](https://github.com/yzqzss/csswaf)<br/>Go / [MIT](https://github.com/yzqzss/csswaf/blob/main/LICENSE)                                            | A CSS-based NoJS Anti-BOT WAF (Proof of Concept)                                                        | Non-JS CSS Subresource loading order         |
+|            [anticrawl](https://flak.tedunangst.com/post/anticrawl)            |                                                        [![humungus.tedunangst.com](https://img.shields.io/badge/tedunangst-anticrawl-blue?style=flat&logo=mercurial&labelColor=fff&logoColor=000)](https://humungus.tedunangst.com/r/anticrawl)<br/>Go / None                                                         | Go http handler / proxy for regex based rules                                                           | Non-JS manual Challenge/Response             |
+
+
+## Development
+
+This Go package can be used as a command on `git.gammaspectra.live/git/go-away/cmd/go-away` or a library under `git.gammaspectra.live/git/go-away/lib`
+
+### Compiling WASM runtime challenge modules
+
+Custom WASM runtime modules follow the WASI `wasip1` preview syscall API.
+
+It is recommended using TinyGo to compile / refresh modules, and some function helpers are provided.
+
+If you want to use a different language or compiler, enable `wasip1` and the following interface must be exported:
+
+```
+// Allocation is a combination of pointer location in WASM memory and size of it
+type Allocation uint64
+
+func (p Allocation) Pointer() uint32 {
+	return uint32(p >> 32)
+}
+func (p Allocation) Size() uint32 {
+	return uint32(p)
+}
+
+
+// MakeChallenge MakeChallengeInput / MakeChallengeOutput are valid JSON.
+// See lib/challenge/wasm/interface/interface.go for a definition
+func MakeChallenge(in Allocation[MakeChallengeInput]) Allocation[MakeChallengeOutput]
+
+// VerifyChallenge VerifyChallengeInput is valid JSON.
+// See lib/challenge/wasm/interface/interface.go for a definition
+func VerifyChallenge(in Allocation[VerifyChallengeInput]) VerifyChallengeOutput
+
+func malloc(size uint32) uintptr
+func free(size uintptr)
+
+```
+
+Modules will be recreated for each call, so there is no state leftover.
+
+
+
+

build-poison.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o pipefail
-
-cd "$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
-
-
-go run ./generate-poison -path ./poison/

cmd/generate-poison/main.go

@@ -1,180 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"flag"
-	"fmt"
-	"github.com/andybalholm/brotli"
-	"github.com/klauspost/compress/zstd"
-	"io"
-	"math/rand/v2"
-	"os"
-	"path"
-	"slices"
-	"strings"
-	"sync"
-)
-
-type poisonCharacterGenerator struct {
-	Header       []byte
-	AllowedBytes []byte
-	Repeat       int
-	counter      int
-}
-
-func (r *poisonCharacterGenerator) Read(p []byte) (n int, err error) {
-	if len(r.Header) > 0 {
-		copy(p, r.Header)
-		nn := min(len(r.Header), len(p))
-		r.Header = r.Header[nn:]
-		p = p[nn:]
-	}
-
-	stride := min(len(p), r.Repeat)
-	for i := 0; i < len(p); i += stride {
-		copy(p[i:], bytes.Repeat([]byte{r.AllowedBytes[r.counter]}, stride))
-		r.counter = (r.counter + 1) % len(r.AllowedBytes)
-	}
-	return len(p), nil
-}
-
-type poisonValuesGenerator struct {
-	Header        []byte
-	AllowedValues [][]byte
-	counter       int
-}
-
-func (r *poisonValuesGenerator) Read(p []byte) (n int, err error) {
-	var i int
-
-	if len(r.Header) > 0 {
-		copy(p, r.Header)
-		nn := min(len(r.Header), len(p))
-		r.Header = r.Header[nn:]
-		i += nn
-
-		for i < len(p) {
-			copy(p[i:], r.AllowedValues[r.counter])
-			i += len(r.AllowedValues[r.counter])
-			r.counter = (r.counter + 1) % len(r.AllowedValues)
-			if r.counter == 0 {
-				break
-			}
-		}
-	}
-
-	for i < len(p) {
-		buf := slices.Repeat(r.AllowedValues[r.counter], len(r.AllowedValues)-r.counter)
-		copy(p[i:], buf)
-		i += len(buf)
-		r.counter = (r.counter + 1) % len(r.AllowedValues)
-	}
-	return len(p), nil
-}
-
-func main() {
-
-	outputPath := flag.String("path", "./", "path to poison files")
-
-	flag.Parse()
-
-	const Gigabyte = 1024 * 1024 * 1024
-
-	compressPoison(*outputPath, "text/html", &poisonValuesGenerator{
-		Header: []byte(fmt.Sprintf("<!DOCTYPE html><html><head><title>%d</title></head><body>", rand.Uint64())),
-		AllowedValues: [][]byte{
-			[]byte("<div><div class=\"\"><h2></h2></div><br>\n"),
-			[]byte("<span><span><p><span>\n"),
-			[]byte("<p></span></script><h3><p><span>\n"),
-			[]byte("<div><span><p></h1>"),
-			[]byte("</div></div></div>\n"),
-			[]byte("</p></p></p>"),
-			[]byte("<h1>Are you a bot?</h1><img>\n"),
-			[]byte("</span></span></span><script>{let a = (new XMLSerializer).serializeToString(document); console.log(a); let b = URL.createObjectURL(new Blob([a])); Array.from(document.getElementsByTagName(\"img\")).forEach((img) => {img.src = b;}); document.getElementsByTagName(\"body\")[0].prepend((new DOMParser()).parseFromString(a, \"text/html\"));}</script>"),
-		},
-	}, Gigabyte)
-}
-
-var poisonEncodings = []string{"br", "zstd", "gzip"}
-
-func compressPoison(outputPath, mime string, r io.Reader, maxSize int64) {
-	r = io.LimitReader(r, maxSize)
-
-	var closers []func()
-	var encoders []io.Writer
-	var writers []io.Writer
-	var readers []io.Reader
-
-	for _, encoding := range poisonEncodings {
-		f, err := os.Create(path.Join(outputPath, strings.ReplaceAll(mime, "/", "_")+"."+encoding+".poison"))
-		if err != nil {
-			panic(err)
-		}
-		switch encoding {
-		case "zstd":
-			w, err := zstd.NewWriter(f, zstd.WithEncoderLevel(zstd.SpeedBestCompression), zstd.WithEncoderCRC(false), zstd.WithWindowSize(zstd.MaxWindowSize))
-			if err != nil {
-				panic(err)
-			}
-			encoders = append(encoders, w)
-			closers = append(closers, func() {
-				w.Close()
-				f.Close()
-			})
-		case "br":
-			w := brotli.NewWriterLevel(f, brotli.BestCompression)
-			encoders = append(encoders, w)
-			closers = append(closers, func() {
-				w.Close()
-				f.Close()
-			})
-		case "gzip":
-			w, err := gzip.NewWriterLevel(f, gzip.BestCompression)
-			if err != nil {
-				panic(err)
-			}
-			encoders = append(encoders, w)
-			closers = append(closers, func() {
-				w.Close()
-				f.Close()
-			})
-		}
-		r, w := io.Pipe()
-		readers = append(readers, r)
-		writers = append(writers, w)
-	}
-
-	var wg sync.WaitGroup
-
-	for i := range poisonEncodings {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			_, err := io.Copy(encoders[i], readers[i])
-			if err != nil {
-				panic(err)
-			}
-			closers[i]()
-
-			// discard remaining data
-			_, _ = io.Copy(io.Discard, readers[i])
-		}()
-	}
-
-	_, err := io.Copy(io.MultiWriter(writers...), r)
-	if err != nil {
-		panic(err)
-	}
-
-	for _, w := range writers {
-		if pw, ok := w.(io.Closer); ok {
-			pw.Close()
-		} else {
-			panic("writer is not a Closer")
-		}
-	}
-
-	wg.Wait()
-}

cmd/go-away/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"context"
+	"bytes"
 	"crypto/ed25519"
 	"crypto/rand"
 	"encoding/hex"
@@ -10,65 +10,34 @@ import (
 	"fmt"
 	"git.gammaspectra.live/git/go-away/lib"
 	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/lib/settings"
 	"git.gammaspectra.live/git/go-away/utils"
-	"golang.org/x/net/http2"
-	"golang.org/x/net/http2/h2c"
-	"gopkg.in/yaml.v3"
-	"log"
+	"github.com/goccy/go-yaml"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"log/slog"
-	"maps"
-	"net"
 	"net/http"
+	"net/http/pprof"
 	"os"
+	"os/signal"
+	"path"
 	"runtime/debug"
-	"strconv"
 	"strings"
-	"sync"
-	"time"
+	"syscall"
 )
 
-func setupListener(network, address, socketMode string) (net.Listener, string) {
-	formattedAddress := ""
-	switch network {
-	case "unix":
-		formattedAddress = "unix:" + address
-	case "tcp":
-		formattedAddress = "http://localhost" + address
-	default:
-		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
-	}
-
-	listener, err := net.Listen(network, address)
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to bind to %s: %w", formattedAddress, err))
-	}
-
-	// additional permission handling for unix sockets
-	if network == "unix" {
-		mode, err := strconv.ParseUint(socketMode, 8, 0)
-		if err != nil {
-			listener.Close()
-			log.Fatal(fmt.Errorf("could not parse socket mode %s: %w", socketMode, err))
-		}
-
-		err = os.Chmod(address, os.FileMode(mode))
-		if err != nil {
-			listener.Close()
-			log.Fatal(fmt.Errorf("could not change socket mode: %w", err))
-		}
-	}
-
-	return listener, formattedAddress
-}
-
-var internalPackageName = func() string {
+var internalCmdName = "go-away"
+var internalMainName = "go-away"
+var internalMainVersion = "dev"
 
+func init() {
 	buildInfo, ok := debug.ReadBuildInfo()
 	if !ok {
-		return "go-away"
+		return
 	}
-	return buildInfo.Path
-}()
+	internalCmdName = buildInfo.Path
+	internalMainName = buildInfo.Main.Path
+	internalMainVersion = buildInfo.Main.Version
+}
 
 type MultiVar []string
 
@@ -81,42 +50,58 @@ func (v *MultiVar) Set(value string) error {
 	return nil
 }
 
-func newServer(handler http.Handler) *http.Server {
-	h2s := &http2.Server{}
-
-	// TODO: use Go 1.24 Server.Protocols to add H2C
-	// https://pkg.go.dev/net/http#Server.Protocols
-	h1s := &http.Server{
-		Handler: h2c.NewHandler(handler, h2s),
-	}
-
-	return h1s
+func fatal(err error) {
+	slog.Error(err.Error())
+	_, _ = fmt.Fprintln(os.Stderr, "================================================")
+	_, _ = fmt.Fprintln(os.Stderr, "Fatal error:")
+	_, _ = fmt.Fprintln(os.Stderr, err.Error())
+	os.Exit(1)
 }
 
 func main() {
-	bind := flag.String("bind", ":8080", "network address to bind HTTP to")
-	bindNetwork := flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
-	socketMode := flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
+
+	opt := settings.DefaultSettings
+
+	flag.StringVar(&opt.Bind.Address, "bind", opt.Bind.Address, "network address to bind HTTP/HTTP(s) to")
+	flag.StringVar(&opt.Bind.Network, "bind-network", opt.Bind.Network, "network family to bind HTTP to, e.g. unix, tcp")
+	flag.BoolVar(&opt.Bind.Proxy, "bind-proxy", opt.Bind.Proxy, "use PROXY protocol in front of the listener")
+	flag.StringVar(&opt.Bind.SocketMode, "socket-mode", opt.Bind.SocketMode, "socket mode (permissions) for unix domain sockets.")
+	flag.StringVar(&opt.BindMetrics, "metrics-bind", opt.BindMetrics, "network address to bind metrics on")
+	flag.StringVar(&opt.BindDebug, "debug-bind", opt.BindDebug, "network address to bind debug on")
 
 	slogLevel := flag.String("slog-level", "WARN", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
-	debugMode := flag.Bool("debug", false, "debug mode with logs and server timings")
-	passThrough := flag.Bool("passthrough", false, "passthrough mode sends all requests to matching backends until state is loaded")
+	flag.BoolVar(&opt.Bind.Passthrough, "passthrough", opt.Bind.Passthrough, "passthrough mode sends all requests to matching backends until state is loaded")
+	check := flag.Bool("check", false, "check configuration and policies, then exit")
+	flag.StringVar(&opt.Bind.TLSAcmeAutoCert, "acme-autocert", opt.Bind.TLSAcmeAutoCert, "enables HTTP(s) mode and uses the provided ACME server URL or available service (available: letsencrypt)")
 
 	clientIpHeader := flag.String("client-ip-header", "", "Client HTTP header to fetch their IP address from (X-Real-Ip, X-Client-Ip, X-Forwarded-For, Cf-Connecting-Ip, etc.)")
+	backendIpHeader := flag.String("backend-ip-header", "", "Backend HTTP header to set the client IP address from, if empty defaults to leaving Client header alone (X-Real-Ip, X-Client-Ip, X-Forwarded-For, Cf-Connecting-Ip, etc.)")
+
+	cachePath := flag.String("cache", path.Join(os.TempDir(), "go_away_cache"), "path to temporary cache directory")
 
 	policyFile := flag.String("policy", "", "path to policy YAML file")
-	challengeTemplate := flag.String("challenge-template", "anubis", "name or path of the challenge template to use (anubis, forgejo)")
-	challengeTemplateTheme := flag.String("challenge-template-theme", "", "name of the challenge template theme to use (forgejo => [forgejo-dark, forgejo-light, gitea...])")
+	var policySnippets MultiVar
+	flag.Var(&policySnippets, "policy-snippets", "path to YAML snippets folder (can be specified multiple times)")
 
-	packageName := flag.String("package-path", internalPackageName, "package name to expose in .well-known url path")
+	flag.StringVar(&opt.ChallengeTemplate, "challenge-template", opt.ChallengeTemplate, "name or path of the challenge template to use (anubis, forgejo)")
+
+	templateTheme := flag.String("challenge-template-theme", opt.ChallengeTemplateOverrides["Theme"], "name of the challenge template theme to use (forgejo => [forgejo-auto, forgejo-dark, forgejo-light, gitea...])")
+
+	basePath := flag.String("path", "/.well-known/."+internalCmdName, "base path where to expose go-away package onto, challenges will be served from here")
 
 	jwtPrivateKeySeed := flag.String("jwt-private-key-seed", "", "Seed for the jwt private key, or on JWT_PRIVATE_KEY_SEED env. One be generated by passing \"generate\" as a value, follows RFC 8032 private key definition. Defaults to random")
 
 	var backends MultiVar
 	flag.Var(&backends, "backend", "backend definition in the form of an.example.com=http://backend:1234 (can be specified multiple times)")
 
+	settingsFile := flag.String("config", "", "path to config override YAML file")
+
 	flag.Parse()
 
+	if *backendIpHeader == "" {
+		*backendIpHeader = *clientIpHeader
+	}
+
 	var err error
 
 	{
@@ -130,10 +115,37 @@ func main() {
 		leveler.Set(programLevel)
 
 		h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
-			AddSource: *debugMode,
+			AddSource: programLevel <= slog.LevelDebug,
 			Level:     leveler,
+			ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
+				if a.Key == "source" {
+					if src, ok := a.Value.Any().(*slog.Source); ok {
+						return slog.String(a.Key, fmt.Sprintf("%s:%d", src.File, src.Line))
+					}
+				}
+				return a
+			},
 		})
 		slog.SetDefault(slog.New(h))
+		// set default log logger to slog logger level
+		slog.SetLogLoggerLevel(programLevel)
+	}
+
+	slog.Info("go-away", "package", internalMainName, "version", internalMainVersion, "cmd", internalCmdName)
+
+	// preload missing settings
+	opt.ChallengeTemplateOverrides["Theme"] = *templateTheme
+
+	// load overrides
+	if *settingsFile != "" {
+		settingsData, err := os.ReadFile(*settingsFile)
+		if err != nil {
+			fatal(fmt.Errorf("could not read settings file: %w", err))
+		}
+		err = yaml.Unmarshal(settingsData, &opt)
+		if err != nil {
+			fatal(fmt.Errorf("could not parse settings file: %w", err))
+		}
 	}
 
 	var seed []byte
@@ -149,7 +161,7 @@ func main() {
 		if strings.ToLower(kValue) == "generate" {
 			_, priv, err := ed25519.GenerateKey(rand.Reader)
 			if err != nil {
-				log.Fatal(fmt.Errorf("failed to generate private key: %w", err))
+				fatal(fmt.Errorf("failed to generate private key: %w", err))
 			}
 			fmt.Printf("%x\n", priv.Seed())
 			os.Exit(0)
@@ -157,123 +169,207 @@ func main() {
 
 		seed, err = hex.DecodeString(kValue)
 		if err != nil {
-			log.Fatal(fmt.Errorf("failed to decode seed: %w", err))
+			fatal(fmt.Errorf("failed to decode seed: %w", err))
 		}
 
 		if len(seed) != ed25519.SeedSize {
-			log.Fatal(fmt.Errorf("invalid seed length: %d, expected %d", len(seed), ed25519.SeedSize))
+			fatal(fmt.Errorf("invalid seed length: %d, expected %d", len(seed), ed25519.SeedSize))
 		}
 
 	}
 
-	policyData, err := os.ReadFile(*policyFile)
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to read policy file: %w", err))
-	}
-
-	var p policy.Policy
-
-	if err = yaml.Unmarshal(policyData, &p); err != nil {
-		log.Fatal(fmt.Errorf("failed to parse policy file: %w", err))
-	}
-
 	createdBackends := make(map[string]http.Handler)
-
-	parsedBackends := make(map[string]string)
-	//TODO: deprecate
-	maps.Copy(parsedBackends, p.Backends)
 	for _, backend := range backends {
+		if backend == "" {
+			// skip empty to allow no values
+			continue
+		}
 		parts := strings.Split(backend, "=")
 		if len(parts) != 2 {
-			log.Fatal(fmt.Errorf("invalid backend definition: %s", backend))
+			fatal(fmt.Errorf("invalid backend definition: %s, expected 2 parts, got %v", backend, parts))
+		}
+
+		// make no-settings, default backend
+		opt.Backends[parts[0]] = settings.Backend{
+			URL:      parts[1],
+			IpHeader: *backendIpHeader,
 		}
-		parsedBackends[parts[0]] = parts[1]
 	}
 
-	for k, v := range parsedBackends {
-		backend, err := utils.MakeReverseProxy(v)
+	for k, v := range opt.Backends {
+		if v.IpHeader == "" {
+			//set default value
+			v.IpHeader = *backendIpHeader
+		}
+
+		backend, err := v.Create()
 		if err != nil {
-			log.Fatal(fmt.Errorf("backend %s: failed to make reverse proxy: %w", k, err))
+			fatal(fmt.Errorf("backend %s: failed to make reverse proxy: %w", k, err))
 		}
 
 		backend.ErrorLog = slog.NewLogLogger(slog.With("backend", k).Handler(), slog.LevelError)
 		createdBackends[k] = backend
 	}
 
-	var wg sync.WaitGroup
+	if len(createdBackends) == 0 {
+		fatal(fmt.Errorf("no backends defined in cmdline or settings file"))
+	}
 
-	passThroughCtx, cancelFunc := context.WithCancel(context.Background())
-	defer cancelFunc()
-
-	if *passThrough {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			server := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				backend, ok := createdBackends[r.Host]
-				if !ok {
-					http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
-					return
-				}
-
-				backend.ServeHTTP(w, r)
-			}))
-
-			listener, listenUrl := setupListener(*bindNetwork, *bind, *socketMode)
-			slog.Warn(
-				"listening passthrough",
-				"url", listenUrl,
-			)
-			defer listener.Close()
-
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-				if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
-					log.Fatal(err)
-				}
-			}()
-
-			<-passThroughCtx.Done()
-
-			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-			defer cancel()
-
-			if err := server.Shutdown(ctx); err != nil {
-				log.Fatal(err)
+	var cache utils.Cache
+	var acmeCache string
+	if *cachePath != "" {
+		err = os.MkdirAll(*cachePath, 0755)
+		if err != nil {
+			fatal(fmt.Errorf("failed to create cache directory: %w", err))
+		}
+		for _, n := range []string{"networks", "acme"} {
+			err = os.MkdirAll(path.Join(*cachePath, n), 0755)
+			if err != nil {
+				fatal(fmt.Errorf("failed to create cache sub directory %s: %w", n, err))
 			}
-			_ = server.Close()
-		}()
+		}
+
+		cache, err = utils.CacheDirectory(*cachePath)
+		if err != nil {
+			fatal(fmt.Errorf("failed to open cache directory: %w", err))
+		}
+
+		acmeCache = path.Join(*cachePath, "acme")
 	}
 
-	state, err := lib.NewState(p, lib.StateSettings{
-		Backends:               createdBackends,
-		Debug:                  *debugMode,
-		PackageName:            *packageName,
-		ChallengeTemplate:      *challengeTemplate,
-		ChallengeTemplateTheme: *challengeTemplateTheme,
-		PrivateKeySeed:         seed,
-		ClientIpHeader:         *clientIpHeader,
-	})
+	loadPolicyState := func() (http.Handler, error) {
+		policyData, err := os.ReadFile(*policyFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read policy file: %w", err)
+		}
 
-	if err != nil {
-		log.Fatal(fmt.Errorf("failed to create state: %w", err))
+		p, err := policy.NewPolicy(bytes.NewReader(policyData), policySnippets...)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse policy file: %w", err)
+		}
+
+		stateSettings := policy.StateSettings{
+			Cache:                 cache,
+			Backends:              createdBackends,
+			MainName:              internalMainName,
+			MainVersion:           internalMainVersion,
+			BasePath:              *basePath,
+			PrivateKeySeed:        seed,
+			ClientIpHeader:        *clientIpHeader,
+			BackendIpHeader:       *backendIpHeader,
+			ChallengeResponseCode: http.StatusTeapot,
+		}
+
+		state, err := lib.NewState(*p, opt, stateSettings)
+
+		if err != nil {
+			return nil, fmt.Errorf("failed to create state: %w", err)
+		}
+		return state, nil
 	}
 
-	// cancel the existing server listener
-	cancelFunc()
-	wg.Wait()
+	if *check {
+		_, err := loadPolicyState()
+		if err != nil {
+			fatal(err)
+		}
+		slog.Info("load ok")
+		os.Exit(0)
+	}
 
-	listener, listenUrl := setupListener(*bindNetwork, *bind, *socketMode)
+	listener, listenUrl := opt.Bind.Listener()
 	slog.Warn(
 		"listening",
 		"url", listenUrl,
 	)
 
-	server := newServer(state)
-
-	if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
-		log.Fatal(err)
+	server, swap, err := opt.Bind.Server(createdBackends, acmeCache)
+	if err != nil {
+		fatal(fmt.Errorf("failed to create server: %w", err))
 	}
+
+	server.ErrorLog = slog.NewLogLogger(slog.With("server", "http").Handler(), slog.LevelError)
+
+	go func() {
+		handler, err := loadPolicyState()
+		if err != nil {
+			fatal(fmt.Errorf("failed to load policy state: %w", err))
+		}
+
+		swap(handler)
+		slog.Warn(
+			"handler configuration loaded",
+		)
+
+		// allow reloading from now on
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, syscall.SIGHUP)
+		for sig := range c {
+			if sig != syscall.SIGHUP {
+				continue
+			}
+			handler, err = loadPolicyState()
+			if err != nil {
+				slog.Error("handler configuration reload error", "err", err)
+				continue
+			}
+
+			swap(handler)
+			slog.Warn("handler configuration reloaded")
+		}
+	}()
+
+	if opt.BindDebug != "" {
+		go func() {
+			mux := http.NewServeMux()
+			mux.HandleFunc("/debug/pprof/", pprof.Index)
+			mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+			mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+			debugServer := http.Server{
+				Addr:     opt.BindDebug,
+				Handler:  mux,
+				ErrorLog: slog.NewLogLogger(slog.With("server", "debug").Handler(), slog.LevelError),
+			}
+
+			slog.Warn(
+				"listening debug",
+				"bind", opt.BindDebug,
+			)
+			if err = debugServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+				fatal(err)
+			}
+		}()
+	}
+
+	if opt.BindMetrics != "" {
+		go func() {
+			mux := http.NewServeMux()
+			mux.Handle("/metrics", promhttp.Handler())
+			metricsServer := http.Server{
+				Addr:     opt.BindMetrics,
+				Handler:  mux,
+				ErrorLog: slog.NewLogLogger(slog.With("server", "metrics").Handler(), slog.LevelError),
+			}
+
+			slog.Warn(
+				"listening metrics",
+				"bind", opt.BindMetrics,
+			)
+			if err = metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+				fatal(err)
+			}
+		}()
+	}
+
+	if server.TLSConfig != nil {
+		if err := server.ServeTLS(listener, "", ""); !errors.Is(err, http.ErrServerClosed) {
+			fatal(err)
+		}
+	} else {
+		if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
+			fatal(err)
+		}
+	}
+
 }

docker-entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+set -e
+
+if [ "${1#-}" != "$1" ]; then
+        set -- /bin/go-away \
+                --bind "${GOAWAY_BIND}" --bind-network "${GOAWAY_BIND_NETWORK}" --socket-mode "${GOAWAY_SOCKET_MODE}" \
+                --metrics-bind "${GOAWAY_METRICS_BIND}" --debug-bind "${GOAWAY_DEBUG_BIND}" \
+                --config "${GOAWAY_CONFIG}" \
+                --policy "${GOAWAY_POLICY}" --policy-snippets "/snippets" --policy-snippets "${GOAWAY_POLICY_SNIPPETS}" \
+                --client-ip-header "${GOAWAY_CLIENT_IP_HEADER}" --backend-ip-header "${GOAWAY_BACKEND_IP_HEADER}" \
+                --cache "${GOAWAY_CACHE}" \
+                --challenge-template "${GOAWAY_CHALLENGE_TEMPLATE}" --challenge-template-theme "${GOAWAY_CHALLENGE_TEMPLATE_THEME}" \
+                --slog-level "${GOAWAY_SLOG_LEVEL}" \
+                --acme-autocert "${GOAWAY_ACME_AUTOCERT}" \
+                --backend "${GOAWAY_BACKEND}" \
+                "$@"
+fi
+
+if [ "$1" = "go-away" ]; then
+        shift
+        set -- /bin/go-away "$@"
+fi
+
+exec "$@"

embed/assets/static/anubis/style.css

@@ -103,3 +103,17 @@ footer {
         padding: 0.5em 10px;
     }
 }
+
+body,
+html {
+    height: 100%;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+.centered-div {
+    text-align: center;
+}

embed/embed.go

@@ -1,15 +1,59 @@
 package embed
 
-import "embed"
+import (
+	"embed"
+	"errors"
+	"io/fs"
+	"os"
+)
 
 //go:embed assets
-var AssetsFs embed.FS
+var assetsFs embed.FS
 
 //go:embed challenge
-var ChallengeFs embed.FS
+var challengeFs embed.FS
 
 //go:embed templates
-var TemplatesFs embed.FS
+var templatesFs embed.FS
 
-//go:embed poison/*.poison
-var PoisonFs embed.FS
+type FSInterface interface {
+	fs.FS
+	fs.ReadDirFS
+	fs.ReadFileFS
+}
+
+func trimPrefix(embedFS embed.FS, prefix string) FSInterface {
+	subFS, err := fs.Sub(embedFS, prefix)
+	if err != nil {
+		panic(err)
+	}
+	if properFS, ok := subFS.(FSInterface); ok {
+		return properFS
+	} else {
+		panic("unsupported")
+	}
+}
+
+var ChallengeFs = trimPrefix(challengeFs, "challenge")
+
+var TemplatesFs = trimPrefix(templatesFs, "templates")
+var AssetsFs = trimPrefix(assetsFs, "assets")
+
+func GetFallbackFS(embedFS FSInterface, prefix string) (FSInterface, error) {
+	var outFs fs.FS
+	if stat, err := os.Stat(prefix); err == nil && stat.IsDir() {
+		outFs = embedFS
+	} else if _, err := embedFS.ReadDir(prefix); err == nil {
+		outFs, err = fs.Sub(embedFS, prefix)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, err
+	}
+	if properFS, ok := outFs.(FSInterface); ok {
+		return properFS, nil
+	} else {
+		return nil, errors.New("unsupported FS")
+	}
+}

embed/templates/challenge-anubis.gohtml

@@ -4,142 +4,13 @@
     <title>{{ .Title }}</title>
     <link rel="stylesheet" href="{{ .Path }}/assets/static/anubis/style.css?cacheBust={{ .Random }}"/>
     <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
-    {{ range $key, $value := .Meta }}
-        {{ if eq $key "refresh"}}
-        <meta http-equiv="{{ $key }}" content="{{ $value }}"/>
-        {{else}}
-        <meta name="{{ $key }}" content="{{ $value }}"/>
-        {{end}}
+    <meta name="referrer" content="origin"/>
+    {{ range .Meta }}
+    <meta {{ range $key, $value := . }}{{ $key | attr }}="{{ $value }}" {{end}}/>
     {{ end }}
-    {{ range .Tags }}
-        {{ . }}
+    {{ range .HeaderTags }}
+    {{ . }}
     {{ end }}
-    <style>
-        body,
-        html {
-            height: 100%;
-            display: flex;
-            justify-content: center;
-            align-items: center;
-            margin-left: auto;
-            margin-right: auto;
-        }
-
-        .centered-div {
-            text-align: center;
-        }
-
-        .lds-roller,
-        .lds-roller div,
-        .lds-roller div:after {
-            box-sizing: border-box;
-        }
-
-        .lds-roller {
-            display: inline-block;
-            position: relative;
-            width: 80px;
-            height: 80px;
-        }
-
-        .lds-roller div {
-            animation: lds-roller 1.2s cubic-bezier(0.5, 0, 0.5, 1) infinite;
-            transform-origin: 40px 40px;
-        }
-
-        .lds-roller div:after {
-            content: " ";
-            display: block;
-            position: absolute;
-            width: 7.2px;
-            height: 7.2px;
-            border-radius: 50%;
-            background: currentColor;
-            margin: -3.6px 0 0 -3.6px;
-        }
-
-        .lds-roller div:nth-child(1) {
-            animation-delay: -0.036s;
-        }
-
-        .lds-roller div:nth-child(1):after {
-            top: 62.62742px;
-            left: 62.62742px;
-        }
-
-        .lds-roller div:nth-child(2) {
-            animation-delay: -0.072s;
-        }
-
-        .lds-roller div:nth-child(2):after {
-            top: 67.71281px;
-            left: 56px;
-        }
-
-        .lds-roller div:nth-child(3) {
-            animation-delay: -0.108s;
-        }
-
-        .lds-roller div:nth-child(3):after {
-            top: 70.90963px;
-            left: 48.28221px;
-        }
-
-        .lds-roller div:nth-child(4) {
-            animation-delay: -0.144s;
-        }
-
-        .lds-roller div:nth-child(4):after {
-            top: 72px;
-            left: 40px;
-        }
-
-        .lds-roller div:nth-child(5) {
-            animation-delay: -0.18s;
-        }
-
-        .lds-roller div:nth-child(5):after {
-            top: 70.90963px;
-            left: 31.71779px;
-        }
-
-        .lds-roller div:nth-child(6) {
-            animation-delay: -0.216s;
-        }
-
-        .lds-roller div:nth-child(6):after {
-            top: 67.71281px;
-            left: 24px;
-        }
-
-        .lds-roller div:nth-child(7) {
-            animation-delay: -0.252s;
-        }
-
-        .lds-roller div:nth-child(7):after {
-            top: 62.62742px;
-            left: 17.37258px;
-        }
-
-        .lds-roller div:nth-child(8) {
-            animation-delay: -0.288s;
-        }
-
-        .lds-roller div:nth-child(8):after {
-            top: 56px;
-            left: 12.28719px;
-        }
-
-        @keyframes lds-roller {
-            0% {
-                transform: rotate(0deg);
-            }
-
-            100% {
-                transform: rotate(360deg);
-            }
-        }
-    </style>
 </head>
 <body id="top">
 <main>
@@ -154,44 +25,29 @@
                 src="{{ .Path }}/assets/static/logo.png?cacheBust={{ .Random }}"
         />
         {{if .Challenge }}
-        <p id="status">Loading challenge <em>{{ .Challenge }}</em>...</p>
-        <script async type="module" src="{{ .Path }}/challenge/{{ .Challenge }}/challenge.mjs?cacheBust={{ .Random }}"></script>
+        <p id="status">{{ .Strings.Get "status_loading_challenge" }} <em>{{ .Challenge }}</em>...</p>
         {{else if .Error}}
-        <p id="status">Error: {{ .Error }}</p>
+        <p id="status">{{ .Strings.Get "status_error" }} {{ .Error }}</p>
         {{else}}
-        <p id="status">Loading...</p>
+        <p id="status">{{ .Strings.Get "status_loading" }}</p>
         {{end}}
-        {{if not .HideSpinner }}
-        <div id="spinner" class="lds-roller">
-            <div></div>
-            <div></div>
-            <div></div>
-            <div></div>
-            <div></div>
-            <div></div>
-            <div></div>
-            <div></div>
-        </div>
-        {{end}}
-        <details style="padding-bottom: 2em;">
-            <summary>Why am I seeing this?</summary>
-            <p>You are seeing this because the administrator of this website has set up <a href="https://git.gammaspectra.live/git/go-away">go-away</a> to protect the server against the scourge of <a href="https://thelibre.news/foss-infrastructure-is-under-attack-by-ai-companies/">AI companies aggressively scraping websites</a>. This can and does cause downtime for the websites, which makes their resources inaccessible for everyone.</p>
-            <p>Please note that some challenges requires the use of modern JavaScript features and some plugins may will disable. Please disable such plugins for this domain (for example, JShelter).</p>
-            <p>If you have any issues contact the administrator and provide this Request Id: <em>{{ .Id }}</em></p>
+        <details>
+            <summary>{{ .Strings.Get "details_title" }}</summary>
+
+            {{.Strings.Get "details_text"}}
         </details>
 
-        <noscript>
-            <p>
-                Sadly, you may need to enable JavaScript to get past this challenge. This is required because AI companies have changed
-                the social contract around how website hosting works.
-            </p>
-        </noscript>
-
         {{if .Redirect }}
-            <a role="button" href="{{ .Redirect }}">Refresh page</a>
+            <a style="margin-top: 2em; margin-bottom: 2em;" role="button" href="{{ .Redirect }}">{{ .Strings.Get "button_refresh_page" }}</a>
         {{end}}
 
-        <div id="testarea"></div>
+        {{if .EndTags }}
+            <noscript>
+                {{ .Strings.Get "noscript_warning" }}
+            </noscript>
+        {{end}}
+
+        <p><small>{{ .Strings.Get "details_contact_admin_with_request_id" }}: <em>{{ .Id }}</em></small></p>
     </div>
 
 
@@ -199,9 +55,18 @@
         <center>
             <p>
                 Protected by <a href="https://git.gammaspectra.live/git/go-away">go-away</a> :: Request Id <em>{{ .Id }}</em>
+
+                {{ range .Links }}
+                    :: <a href="{{ .URL }}">{{ .Name }}</a>
+                {{ end }}
             </p>
         </center>
     </footer>
+
+
+    {{ range .EndTags }}
+        {{ . }}
+    {{ end }}
 </main>
 </body>
 </html>

embed/templates/challenge-forgejo.gohtml

@@ -1,24 +1,16 @@
 <!DOCTYPE html>
-{{$theme := "forgejo-dark"}}
-{{ if .Theme }}
-    {{$theme = .Theme}}
-{{ end }}
+{{$theme := "forgejo-auto"}}{{ if .Theme }}{{$theme = .Theme}}{{ end }}
 <html lang="en-US" data-theme="{{ $theme }}">
 <head>
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
     <title>{{ .Title }}</title>
-    <meta name="referrer" content="no-referrer">
-
-    {{ range $key, $value := .Meta }}
-        {{ if eq $key "refresh"}}
-            <meta http-equiv="{{ $key }}" content="{{ $value }}"/>
-        {{else}}
-            <meta name="{{ $key }}" content="{{ $value }}"/>
-        {{end}}
+    <meta name="referrer" content="origin">
+    {{ range .Meta }}
+    <meta {{ range $key, $value := . }}{{ $key | attr }}="{{ $value }}" {{end}}/>
     {{ end }}
-    {{ range .Tags }}
-        {{ . }}
+    {{ range .HeaderTags }}
+    {{ . }}
     {{ end }}
 
 
@@ -61,37 +53,32 @@
                     </h2>
 
                     {{if .Challenge }}
-                    <h3 id="status">Loading challenge <em>{{ .Challenge }}</em>...</h3>
-                    <script async type="module" src="{{ .Path }}/challenge/{{ .Challenge }}/challenge.mjs?cacheBust={{ .Random }}"></script>
+                    <h3 id="status">{{ .Strings.Get "status_loading_challenge" }} <em>{{ .Challenge }}</em>...</h3>
                     {{else if .Error}}
-                    <h3 id="status">Error: {{ .Error }}</h3>
+                    <h3 id="status">{{ .Strings.Get "status_error" }} {{ .Error }}</h3>
                     {{else}}
-                    <h3 id="status">Loading...</h3>
+                    <h3 id="status">{{ .Strings.Get "status_loading" }}</h3>
                     {{end}}
-                    <div id="spinner"></div>
 
-                    <details style="padding-bottom: 2em;">
-                        <summary>Why am I seeing this?</summary>
-                        <p>You are seeing this because the administrator of this website has set up <a href="https://git.gammaspectra.live/git/go-away">go-away</a> to protect the server against the scourge of <a href="https://thelibre.news/foss-infrastructure-is-under-attack-by-ai-companies/">AI companies aggressively scraping websites</a>. This can and does cause downtime for the websites, which makes their resources inaccessible for everyone.</p>
-                        <p>Please note that some challenges requires the use of modern JavaScript features and some plugins may will disable. Please disable such plugins for this domain (for example, JShelter).</p>
-                        <p>If you have any issues contact the administrator and provide the Request Id: <em>{{ .Id }}</em></p>
+                    <details>
+                        <summary>{{ .Strings.Get "details_title" }}</summary>
+
+                        {{.Strings.Get "details_text"}}
                     </details>
 
-                    <noscript>
-                        <p>
-                            Sadly, you may need to enable JavaScript to get past this challenge. This is required because AI companies have changed
-                            the social contract around how website hosting works.
-                        </p>
-                    </noscript>
-
                     {{if .Redirect }}
-                    <div class="button-row">
-                        <a role="button" class="ui small primary button" href="{{ .Redirect }}">Refresh page</a>
-                    </div>
+                        <div class="button-row" style="margin-top: 2em; margin-bottom: 2em;" >
+                            <a role="button" class="ui small primary button" href="{{ .Redirect }}">{{ .Strings.Get "button_refresh_page" }}</a>
+                        </div>
                     {{end}}
 
+                    {{if .EndTags }}
+                        <noscript>
+                            {{ .Strings.Get "noscript_warning" }}
+                        </noscript>
+                    {{end}}
 
-                    <div id="testarea"></div>
+                    <p><small>{{ .Strings.Get "details_contact_admin_with_request_id" }}: <em>{{ .Id }}</em></small></p>
                 </div>
             </div>
         </div>
@@ -107,8 +94,15 @@
 <footer class="page-footer" role="group" aria-label="">
     <div class="left-links" role="contentinfo" aria-label="">
         Protected by <a href="https://git.gammaspectra.live/git/go-away">go-away</a> :: Request Id <em>{{ .Id }}</em>
+        {{ range .Links }}
+            :: <a href="{{ .URL }}">{{ .Name }}</a>
+        {{ end }}
     </div>
 </footer>
 
+{{ range .EndTags }}
+    {{ . }}
+{{ end }}
+
 </body>
 </html>

examples/config.yml

@@ -0,0 +1,101 @@
+# Configuration file
+# Parameters that exist both on config and cmdline will have cmdline as preference
+
+bind:
+  #address: ":8080"
+  #network: "tcp"
+  #socket-mode": "0770"
+
+  # Enable PROXY mode on this listener, to allow passing origin info. Default false
+  #proxy: true
+
+  # Enable passthrough mode, which will allow traffic onto the backends while rules load. Default false
+  #passthrough: true
+
+  # Enable TLS on this listener and obtain certificates via an ACME directory URL, or letsencrypt
+  #tls-acme-autocert: "letsencrypt"
+
+  # Enable TLS on this listener and obtain certificates via a certificate and key file on disk
+  # Only set one of tls-acme-autocert or tls-certificate+tls-key
+  #tls-certificate: ""
+  #tls-key: ""
+
+# Bind the Go debug port
+#bind-debug: ":6060"
+
+# Bind the Prometheus metrics onto /metrics path on this port
+#bind-metrics ":9090"
+
+# These links will be shown on the presented challenge or error pages
+links:
+  #- name: Privacy
+  #  url: "/privacy.html"
+
+  #- name: Contact
+  #  url: "mailto:admin@example.com"
+
+  #- name: Donations
+  #  url: "https://donations.example.com/abcd"
+
+# HTML Template to use for challenge or error pages
+# External templates can be included by providing a disk path
+# Bundled templates:
+#  anubis: An Anubis-like template with no configuration parameters
+#  forgejo: Looks like native Forgejo. Includes logos and resources from your instance. Supports Theme.
+#
+#challenge-template: "anubis"
+
+# Allows overriding specific settings set on templates. Key-Values will be passed to templates as-is
+challenge-template-overrides:
+  # Set template theme if supported
+  #Theme: "forgejo-auto"
+
+# Advanced backend configuration
+# Backends setup via cmdline will be added here
+backends:
+  # Example HTTP backend and setting client ip header
+  #"git.example.com":
+  #  url: "http://forgejo:3000"
+  #  ip-header: "X-Client-Ip"
+
+
+  # Example HTTPS backend with host/SNI override, HTTP/2 and no certificate verification
+  #"ssl.example.com":
+  #  url: "https://127.0.0.1:8443"
+  #  host: ssl.example.com
+  #  http2-enabled: true
+  #  tls-skip-verify: true
+
+# List of strings you can replace to alter the presentation on challenge/error templates
+# Can use other languages.
+# Note raw HTML is allowed, be careful with it.
+# Default strings exist in code, uncomment any to set it
+strings:
+  #title_challenge: "Checking you are not a bot"
+  #title_error: "Oh no!"
+  #noscript_warning: "<p>Sadly, you may need to enable JavaScript to get past this challenge. This is required because AI companies have changed the social contract around how website hosting works.</p>"
+  #details_title: "Why am I seeing this?"
+  #details_text: >
+  #  <p>
+  #  You are seeing this because the administrator of this website has set up <a href="https://git.gammaspectra.live/git/go-away">go-away</a>
+  #  to protect the server against the scourge of <a href="https://thelibre.news/foss-infrastructure-is-under-attack-by-ai-companies/">AI companies aggressively scraping websites</a>.
+  #  </p>
+  #  <p>
+  #  Mass scraping can and does cause downtime for the websites, which makes their resources inaccessible for everyone.
+  #  </p>
+  #  <p>
+  #  Please note that some challenges requires the use of modern JavaScript features and some plugins may disable these.
+  #  Disable such plugins for this domain (for example, JShelter) if you encounter any issues.
+  #  </p>
+
+  #details_contact_admin_with_request_id: "If you have any issues contact the site administrator and provide the following Request Id"
+
+  #button_refresh_page: "Refresh page"
+
+  #status_loading_challenge: "Loading challenge"
+  #status_starting_challenge: "Starting challenge"
+  #status_loading: "Loading..."
+  #status_calculating: "Calculating..."
+  #status_challenge_success: "Challenge success!"
+  #status_challenge_done_took: "Done! Took"
+  #status_error: "Error:"

examples/forgejo.yml

@@ -1,191 +1,41 @@
 # Example cmdline (forward requests from upstream to port :8080)
-# $ go-away --bind :8080 --backend git.example.com:http://forgejo:3000 --policy examples/forgejo.yml --challenge-template forgejo --challenge-template-theme forgejo-dark
+# $ go-away --bind :8080 --backend git.example.com=http://forgejo:3000 --policy examples/forgejo.yml --policy-snippets example/snippets/ --challenge-template forgejo --challenge-template-theme forgejo-auto
 
 
 
 # Define networks to be used later below
 networks:
-  # todo: support direct ASN lookups
-  # todo: cache these values
+  # Networks will get included from snippets
+
   huawei-cloud:
-    # AS136907
-    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/136907/aggregated.json
-      jq-path: '.subnets.ipv4[], .subnets.ipv6[]'
+    - asn: 136907
   alibaba-cloud:
-    # AS45102
-    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/45102/aggregated.json
-      jq-path: '.subnets.ipv4[], .subnets.ipv6[]'
+    - asn: 45102
   zenlayer-inc:
-    # AS21859
-    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/21859/aggregated.json
-      jq-path: '.subnets.ipv4[], .subnets.ipv6[]'
-  aws-cloud:
-    - url: https://ip-ranges.amazonaws.com/ip-ranges.json
-      jq-path: '(.prefixes[] | select(has("ip_prefix")) | .ip_prefix), (.prefixes[] | select(has("ipv6_prefix")) | .ipv6_prefix)'
-  google-cloud:
-    - url: https://www.gstatic.com/ipranges/cloud.json
-      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
-  oracle-cloud:
-    - url: https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json
-      jq-path: '.regions[] | .cidrs[] | .cidr'
-  azure-cloud:
-    # todo: https://www.microsoft.com/en-us/download/details.aspx?id=56519 does not provide direct JSON
-    - url: https://raw.githubusercontent.com/femueller/cloud-ip-ranges/refs/heads/master/microsoft-azure-ip-ranges.json
-      jq-path: '.values[] | .properties.addressPrefixes[]'
-
-  digitalocean:
-    - url: https://www.digitalocean.com/geo/google.csv
-      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
-  linode:
-    - url: https://geoip.linode.com/
-      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
-  vultr:
-    - url: "https://geofeed.constant.com/?json"
-      jq-path: '.subnets[] | .ip_prefix'
-  cloudflare:
-    - url: https://www.cloudflare.com/ips-v4
-      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
-    - url: https://www.cloudflare.com/ips-v6
-      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"
-
-  icloud-private-relay:
-    - url: https://mask-api.icloud.com/egress-ip-ranges.csv
-      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
-  tunnelbroker-relay:
-    # HE Tunnelbroker
-    - url: https://tunnelbroker.net/export/google
-      regex: "(?P<prefix>([0-9a-f:]+::)/[0-9]+),"
+    - asn: 21859
 
 
-  googlebot:
-    - url: https://developers.google.com/static/search/apis/ipranges/googlebot.json
-      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
-  bingbot:
-    - url: https://www.bing.com/toolbox/bingbot.json
-      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
-  qwantbot:
-    - url: https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
-      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
-  duckduckbot:
-    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
-      regex: "<li>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</li>"
-  yandexbot:
-    # todo: detected as bot
-    # - url: https://yandex.com/ips
-    #  regex: "<span>(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+)[ \\\\t]*</span><br/>"
-    - prefixes:
-        - "5.45.192.0/18"
-        - "5.255.192.0/18"
-        - "37.9.64.0/18"
-        - "37.140.128.0/18"
-        - "77.88.0.0/18"
-        - "84.252.160.0/19"
-        - "87.250.224.0/19"
-        - "90.156.176.0/22"
-        - "93.158.128.0/18"
-        - "95.108.128.0/17"
-        - "141.8.128.0/18"
-        - "178.154.128.0/18"
-        - "185.32.187.0/24"
-        - "2a02:6b8::/29"
-  kagibot:
-    - url: https://kagi.com/bot
-      regex: "\\n(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+) "
-
-
-# todo: define interface
 challenges:
-  js-pow-sha256:
-    # Asset must be under challenges/{name}/static/{asset}
-    # Other files here will be available under that path
-    mode: js
-    asset: load.mjs
-    parameters:
-      difficulty: 20
-    runtime:
-      mode: wasm
-      # Verify must be under challenges/{name}/runtime/{asset}
-      asset: runtime.wasm
-      probability: 0.02
-
-  # Challenges with a cookie, self redirect (non-JS, requires HTTP parsing)
-  self-cookie:
-    mode: "cookie"
-
-
-  # Challenges with a redirect via Link header with rel=preload and early hints (non-JS, requires HTTP parsing, fetching and logic)
-  # Works on HTTP/2 and above!
-  self-preload-link:
-    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
-    mode: "preload-link"
-    runtime:
-      # verifies that result = key
-      mode: "key"
-      probability: 0.1
-    parameters:
-      preload-early-hint-deadline: 3s
-      key-code: 200
-      key-mime: text/css
-      key-content: ""
-
-  # Challenges with a redirect via Refresh header (non-JS, requires HTTP parsing and logic)
-  self-header-refresh:
-    mode: "header-refresh"
-    runtime:
-      # verifies that result = key
-      mode: "key"
-      probability: 0.1
-
-  # Challenges with a redirect via Refresh meta (non-JS, requires HTML parsing and logic)
-  self-meta-refresh:
-    mode: "meta-refresh"
-    runtime:
-      # verifies that result = key
-      mode: "key"
-      probability: 0.1
-
-  # Challenges with loading a random CSS or image document (non-JS, requires HTML parsing and logic)
-  self-resource-load:
-    mode: "resource-load"
-    runtime:
-      # verifies that result = key
-      mode: "key"
-      probability: 0.1
-    parameters:
-      key-code: 200
-      key-mime: text/css
-      key-content: ""
-
+  # Challenges will get included from snippets
 
   # Verifies the existence of a cookie and confirms it against some backend request, passing the entire client cookie contents
   http-cookie-check:
-    mode: http
-    url: http://forgejo:3000/user/stopwatches
-    # url: http://forgejo:3000/repo/search
-    # url: http://forgejo:3000/notifications/new
+    runtime: http
     parameters:
+      http-url: http://forgejo:3000/user/stopwatches
+      # http-url: http://forgejo:3000/repo/search
+      # http-url: http://forgejo:3000/notifications/new
       http-method: GET
       http-cookie: i_like_gitea
       http-code: 200
-      # todo: archive value of session within token to bind it
+      verify-probability: 0.1
 
 conditions:
   # Conditions will get replaced on rules AST when found as ($condition-name)
-  # Checks to detect a headless chromium via headers only
-  is-headless-chromium:
-    - 'userAgent.contains("HeadlessChrome") || userAgent.contains("HeadlessChromium")'
-    - '"Sec-Ch-Ua" in headers && (headers["Sec-Ch-Ua"].contains("HeadlessChrome") || headers["Sec-Ch-Ua"].contains("HeadlessChromium"))'
-    #- '(userAgent.contains("Chrome/") || userAgent.contains("Chromium/")) && (!("Accept-Language" in headers) || !("Accept-Encoding" in headers))'
 
-  is-generic-browser:
-    - 'userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")'
-
-  is-well-known-asset:
-    - 'path == "/robots.txt"'
-    - 'path.startsWith("/.well-known")'
+  # Conditions will get included from snippets
 
   is-static-asset:
-    - 'path == "/favicon.ico"'
     - 'path == "/apple-touch-icon.png"'
     - 'path == "/apple-touch-icon-precomposed.png"'
     - 'path.startsWith("/assets/")'
@@ -195,44 +45,18 @@ conditions:
     - 'path.startsWith("/user/avatar/")'
     - 'path.startsWith("/attachments/")'
 
-  is-git-ua:
-    - 'userAgent.startsWith("git/") || userAgent.contains("libgit")'
-    - 'userAgent.startsWith("go-git")'
-    - 'userAgent.startsWith("JGit/") || userAgent.startsWith("JGit-")'
-    # Golang proxy and initial fetch
-    - 'userAgent.startsWith("GoModuleMirror/")'
-    - 'userAgent.startsWith("Go-http-client/") && "go-get" in query && query["go-get"] == "1"'
-    - '"Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"'
   is-git-path:
     - 'path.matches("^/[^/]+/[^/]+/(git-upload-pack|git-receive-pack|HEAD|info/refs|info/lfs|objects)")'
 
-  is-generic-robot-ua:
-    - 'userAgent.contains("compatible;") && !userAgent.contains("Trident/")'
-    - 'userAgent.matches("\\+https?://")'
-    - 'userAgent.matches("[bB]ot/[0-9]")'
-
-  is-tool-ua:
-    - 'userAgent.startsWith("python-requests/")'
-    - 'userAgent.startsWith("Python-urllib/")'
-    - 'userAgent.startsWith("python-httpx/")'
-    - 'userAgent.contains("aoihttp/")'
-    - 'userAgent.startsWith("http.rb/")'
-    - 'userAgent.startsWith("curl/")'
-    - 'userAgent.startsWith("Wget/")'
-    - 'userAgent.startsWith("libcurl/")'
-    - 'userAgent.startsWith("okhttp/")'
-    - 'userAgent.startsWith("Java/")'
-    - 'userAgent.startsWith("Apache-HttpClient//")'
-    - 'userAgent.startsWith("Go-http-client/")'
-    - 'userAgent.startsWith("node-fetch/")'
-    - 'userAgent.startsWith("reqwest/")'
-
   is-suspicious-crawler:
+    # TLS Fingerprint for specific agent without ALPN
+    - '(userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")) && ("ja4" in fp && fp.ja4.matches("^t[0-9a-z]+00_"))'
+    # Old engines
     - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
     # Old IE browsers
     - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
     # Old Linux browsers
-    - 'userAgent.contains("Linux i686")'
+    - 'userAgent.matches("Linux i[63]86") || userAgent.matches("FreeBSD i[63]86")'
     # Old Windows browsers
     - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
     # Old mobile browsers
@@ -241,6 +65,7 @@ conditions:
     - 'userAgent.startsWith("Opera/")'
     #- 'userAgent.matches("Gecko/(201[0-9]|200[0-9])")'
     - 'userAgent.matches("^Mozilla/[1-4]")'
+
   is-heavy-resource:
     - 'path.startsWith("/explore/")'
     - 'path.matches("^/[^/]+/[^/]+/src/commit/")'
@@ -255,28 +80,33 @@ conditions:
     # user activity tab
     - 'path.matches("^/[^/]+$") && "tab" in query && query.tab == "activity"'
 
-# Rules and conditions are served this environment
-#   remoteAddress (net.IP) - Connecting client remote address from headers or properties
-#   host (string) - HTTP Host
-#   method (string) - HTTP Method/Verb
-#   userAgent (string) - HTTP User-Agent header
-#   path (string) - HTTP request Path
-#   query (map[string]string) - HTTP request Query arguments
-#   headers (map[string]string) - HTTP request headers
-#
-# Additionally these functions are available
-#   inNetwork(networkName string, address net.IP) bool
-#   inNetwork(networkCIDR string, address net.IP) bool
+
+# Rules are checked sequentially in order, from top to bottom
 rules:
   - name: allow-well-known-resources
     conditions:
       - '($is-well-known-asset)'
     action: pass
 
+  - name: allow-static-resources
+    conditions:
+      - '($is-static-asset)'
+    action: pass
+
+  - name: desired-crawlers
+    conditions:
+      - *is-bot-googlebot
+      - *is-bot-bingbot
+      - *is-bot-duckduckbot
+      - *is-bot-kagibot
+      - *is-bot-qwantbot
+      - *is-bot-yandexbot
+    action: pass
+
   - name: undesired-networks
     conditions:
-      - 'inNetwork("huawei-cloud", remoteAddress) || inNetwork("alibaba-cloud", remoteAddress) || inNetwork("zenlayer-inc", remoteAddress)'
-    action: poison
+      - 'remoteAddress.network("huawei-cloud") || remoteAddress.network("alibaba-cloud") || remoteAddress.network("zenlayer-inc")'
+    action: drop
 
   - name: undesired-crawlers
     conditions:
@@ -287,7 +117,7 @@ rules:
       - 'userAgent.matches("^Opera/[0-9.]+\\.\\(")'
       # AI bullshit stuff, they do not respect robots.txt even while they read it
       # TikTok Bytedance AI training
-      - 'userAgent.contains("Bytedance") || userAgent.contains("Bytespider")'
+      - 'userAgent.contains("Bytedance") || userAgent.contains("Bytespider") || userAgent.contains("TikTokSpider")'
       # Meta AI training; The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.
       - 'userAgent.contains("meta-externalagent/") || userAgent.contains("meta-externalfetcher/") || userAgent.contains("FacebookBot")'
       # Anthropic AI training and usage
@@ -300,7 +130,7 @@ rules:
       - 'userAgent.contains("Amazonbot") || userAgent.contains("Google-Extended") || userAgent.contains("PanguBot") || userAgent.contains("AI2Bot") || userAgent.contains("Diffbot") || userAgent.contains("cohere-training-data-crawler") || userAgent.contains("Applebot-Extended")'
       # SEO / Ads and marketing
       - 'userAgent.contains("BLEXBot")'
-    action: poison
+    action: drop
 
   - name: unknown-crawlers
     conditions:
@@ -309,32 +139,29 @@ rules:
     action: deny
 
   # check a sequence of challenges for non logged in
-  - name: suspicious-crawlers/0
+  - name: suspicious-crawlers
     conditions: ['($is-suspicious-crawler)']
-    action: check
-    challenges: [js-pow-sha256, http-cookie-check]
-  - name: suspicious-crawlers/1
-    conditions: ['($is-suspicious-crawler)']
-    action: check
-    challenges: [self-preload-link]
-  - name: suspicious-crawlers/2
-    conditions: ['($is-suspicious-crawler)']
-    action: check
-    challenges: [self-header-refresh]
-  - name: suspicious-crawlers/3
-    conditions: ['($is-suspicious-crawler)']
-    action: check
-    challenges: [self-resource-load]
-
-  - name: allow-static-resources
-    conditions:
-      - '($is-static-asset)'
-    action: pass
+    action: none
+    children:
+      - name: 0
+        action: check
+        settings:
+          challenges: [js-pow-sha256, http-cookie-check]
+      - name: 1
+        action: check
+        settings:
+          challenges: [preload-link, resource-load]
+      - name: 2
+        action: check
+        settings:
+          challenges: [header-refresh]
 
   - name: always-pow-challenge
     conditions:
-      # login paths
-      - 'path.startsWith("/user/sign_up") || path.startsWith("/user/login") || path.startsWith("/user/oauth2/")'
+      # login and sign up paths
+      - 'path.startsWith("/user/sign_up")'
+      - 'path.startsWith("/user/login") || path.startsWith("/user/oauth2/")'
+      - 'path.startsWith("/user/activate")'
       # repo / org / mirror creation paths
       - 'path == "/repo/create" || path == "/repo/migrate" || path ==  "/org/create"'
       # user profile info edit paths
@@ -344,7 +171,8 @@ rules:
       # Match archive downloads from browsers and not tools
       - 'path.matches("^/[^/]+/[^/]+/archive/.*\\.(bundle|zip|tar\\.gz)") && ($is-generic-browser)'
     action: challenge
-    challenges: [ js-pow-sha256 ]
+    settings:
+      challenges: [ js-pow-sha256 ]
 
   - name: allow-git-operations
     conditions:
@@ -379,50 +207,51 @@ rules:
       # OCI packages API and package managers
       - 'path.startsWith("/api/packages/") || path == "/api/packages"'
       - 'path.startsWith("/v2/") || path == "/v2"'
+      - 'path.endsWith("/branches/list") || path.endsWith("/tags/list")'
     action: pass
 
   - name: preview-fetchers
     conditions:
+      # These summary cards are included in most previews at the end of the url
       - 'path.endsWith("/-/summary-card")'
       #- 'userAgent.contains("facebookexternalhit/")'
-      - 'userAgent.contains("Twitterbot/")'
-      - '"X-Purpose" in headers && headers["X-Purpose"] == "preview"'
-    action: pass
-
-  - name: desired-crawlers
-    conditions:
-      - 'userAgent.contains("+https://kagi.com/bot") && inNetwork("kagibot", remoteAddress)'
-      - '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-InspectionTool")) && inNetwork("googlebot", remoteAddress)'
-      - 'userAgent.contains("+http://www.bing.com/bingbot.htm") && inNetwork("bingbot", remoteAddress)'
-      - 'userAgent.contains("+http://duckduckgo.com/duckduckbot.html") && inNetwork("duckduckbot", remoteAddress)'
-      - 'userAgent.contains("+https://help.qwant.com/bot/") && inNetwork("qwantbot", remoteAddress)'
-      - 'userAgent.contains("+http://yandex.com/bots") && inNetwork("yandexbot", remoteAddress)'
+      #- 'userAgent.contains("Twitterbot/")'
     action: pass
 
+  # Allow loading and embedding of core pages without challenges
+  # Extended pages like linking to files or tabs are not covered here, but might be included in other challenges
   - name: homesite
     conditions:
+      # Match root of site
       - 'path == "/"'
+
+      # Match root of any repository or user, or issue or pr
       # generic /*/*/ match gave too many options for scrapers to trigger random endpoints
-      # edit this with preferential users/orgs for now
-      # todo: create negative match?
-      - 'path.matches("(?i)^/(WeebDataHoarder|P2Pool|mirror|git|S\\.O\\.N\\.G|FM10K|Sillycom|pwgen2155|kaitou|metonym)/[^/]+$")'
+      # this is a negative match of endpoints that Forgejo holds as reserved as users or orgs
+      # see https://codeberg.org/forgejo/forgejo/src/branch/forgejo/models/user/user.go#L582
+      - '(path.matches("^/[^/]+/[^/]+/?$") || path.matches("^/[^/]+/[^/]+/badges/") || path.matches("^/[^/]+/[^/]+/(issues|pulls)/[0-9]+$") || (path.matches("^/[^/]+/?$") && size(query) == 0)) && !path.matches("(?i)^/(api|metrics|v2|assets|attachments|avatar|avatars|repo-avatars|captcha|login|org|repo|user|admin|devtest|explore|issues|pulls|milestones|notifications|ghost)(/|$)")'
     action: pass
 
-  - name: suspicious-fetchers
-    action: challenge
-    challenges: [js-pow-sha256, http-cookie-check]
-    conditions:
-      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
-
   # check a sequence of challenges
-  - name: heavy-operations/0
-    action: check
-    challenges: [self-preload-link, self-header-refresh, js-pow-sha256, http-cookie-check]
+  - name: heavy-operations
     conditions: ['($is-heavy-resource)']
-  - name: heavy-operations/1
+    action: none
+    children:
+      - name: 0
+        action: check
+        settings:
+          challenges: [preload-link, header-refresh, js-pow-sha256, http-cookie-check]
+      - name: 1
+        action: check
+        settings:
+          challenges: [ resource-load, js-pow-sha256, http-cookie-check ]
+
+  - name: standard-bots
     action: check
-    challenges: [self-resource-load, js-pow-sha256, http-cookie-check]
-    conditions: ['($is-heavy-resource)']
+    settings:
+      challenges: [meta-refresh, resource-load]
+    conditions:
+      - '($is-generic-robot-ua)'
 
   # Allow all source downloads not caught in browser above
   # todo: limit this as needed?
@@ -430,27 +259,70 @@ rules:
     conditions:
       - 'path.matches("^/[^/]+/[^/]+/raw/branch/")'
       - 'path.matches("^/[^/]+/[^/]+/archive/")'
-      - 'path.matches("^/[^/]+/[^/]+/media/")'
       - 'path.matches("^/[^/]+/[^/]+/releases/download/")'
+      - 'path.matches("^/[^/]+/[^/]+/media/") && ($is-generic-browser)'
     action: pass
 
+  # check DNSBL and serve harder challenges
+  # todo: make this specific to score
+  - name: undesired-dnsbl
+    action: check
+    settings:
+      challenges: [dnsbl]
+      # if DNSBL fails, check additional challenges
+      fail: check
+      fail-settings:
+        challenges: [js-pow-sha256, http-cookie-check]
+
+  - name: suspicious-fetchers
+    action: check
+    settings:
+      challenges: [js-pow-sha256]
+    conditions:
+      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
+
   # Allow PUT/DELETE/PATCH/POST requests in general
   - name: non-get-request
     action: pass
     conditions:
       - '!(method == "HEAD" || method == "GET")'
 
+  # Enable fetching OpenGraph and other tags from backend on these paths
+  - name: enable-meta-tags
+    action: context
+    settings:
+      context-set:
+        # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
+        proxy-meta-tags: "true"
+        
+      # Set additional response headers
+      #response-headers:
+      # X-Clacks-Overhead:
+      #  - GNU Terry Pratchett
 
 
+
+  - name: plaintext-browser
+    action: challenge
+    settings:
+      challenges: [http-cookie-check, meta-refresh, cookie]
+    conditions:
+      - 'userAgent.startsWith("Lynx/")'
+
+  # Comment this rule out to not challenge tool-like user agents
   - name: standard-tools
     action: challenge
-    challenges: [self-meta-refresh]
+    settings:
+      challenges: [cookie]
     conditions:
-      - '($is-generic-robot-ua)'
       - '($is-tool-ua)'
+      - '!($is-generic-browser)'
 
   - name: standard-browser
     action: challenge
-    challenges: [http-cookie-check, self-preload-link, self-meta-refresh, self-resource-load, js-pow-sha256]
+    settings:
+      challenges: [http-cookie-check, preload-link, meta-refresh, resource-load, js-pow-sha256]
     conditions:
       - '($is-generic-browser)'
+
+# If end of rules is reached, default is PASS

examples/generic.yml

@@ -0,0 +1,177 @@
+# Example cmdline (forward requests from upstream to port :8080)
+# $ go-away --bind :8080 --backend site.example.com=http://site:3000 --policy examples/generic.yml --policy-snippets example/snippets/ --challenge-template anubis
+
+
+
+# Define networks to be used later below
+networks:
+  # Networks will get included from snippets
+
+
+challenges:
+  # Challenges will get included from snippets
+  
+conditions:
+  # Conditions will get replaced on rules AST when found as ($condition-name)
+
+  # Conditions will get included from snippets
+
+
+  is-static-asset:
+    - 'path == "/apple-touch-icon.png"'
+    - 'path == "/apple-touch-icon-precomposed.png"'
+    - 'path.matches("\\.(manifest|ttf|woff|woff2|jpg|jpeg|gif|png|webp|avif|svg|mp4|webm|css|js|mjs|wasm)$")'
+
+  is-suspicious-crawler:
+    - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
+    # Old IE browsers
+    - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
+    # Old Linux browsers
+    - 'userAgent.contains("Linux i[63]86") || userAgent.contains("FreeBSD i[63]86")'
+    # Old Windows browsers
+    - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
+    # Old mobile browsers
+    - 'userAgent.matches("Android [1-5]\\.") || userAgent.matches("(iPad|iPhone) OS [1-9]_")'
+    # Old generic browsers
+    - 'userAgent.startsWith("Opera/")'
+    #- 'userAgent.matches("Gecko/(201[0-9]|200[0-9])")'
+    - 'userAgent.matches("^Mozilla/[1-4]")'
+
+
+# Rules are checked sequentially in order, from top to bottom
+rules:
+  - name: allow-well-known-resources
+    conditions:
+      - '($is-well-known-asset)'
+    action: pass
+
+  - name: allow-static-resources
+    conditions:
+      - '($is-static-asset)'
+    action: pass
+
+  - name: desired-crawlers
+    conditions:
+      - *is-bot-googlebot
+      - *is-bot-bingbot
+      - *is-bot-duckduckbot
+      - *is-bot-kagibot
+      - *is-bot-qwantbot
+      - *is-bot-yandexbot
+    action: pass
+
+  - name: undesired-crawlers
+    conditions:
+      - '($is-headless-chromium)'
+      - 'userAgent.startsWith("Lightpanda/")'
+      - 'userAgent.startsWith("masscan/")'
+      # Typo'd opera botnet
+      - 'userAgent.matches("^Opera/[0-9.]+\\.\\(")'
+      # AI bullshit stuff, they do not respect robots.txt even while they read it
+      # TikTok Bytedance AI training
+      - 'userAgent.contains("Bytedance") || userAgent.contains("Bytespider") || userAgent.contains("TikTokSpider")'
+      # Meta AI training; The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.
+      - 'userAgent.contains("meta-externalagent/") || userAgent.contains("meta-externalfetcher/") || userAgent.contains("FacebookBot")'
+      # Anthropic AI training and usage
+      - 'userAgent.contains("ClaudeBot") || userAgent.contains("Claude-User")|| userAgent.contains("Claude-SearchBot")'
+      # Common Crawl AI crawlers
+      - 'userAgent.contains("CCBot")'
+      # ChatGPT AI crawlers https://platform.openai.com/docs/bots
+      - 'userAgent.contains("GPTBot") || userAgent.contains("OAI-SearchBot") || userAgent.contains("ChatGPT-User")'
+      # Other AI crawlers
+      - 'userAgent.contains("Amazonbot") || userAgent.contains("Google-Extended") || userAgent.contains("PanguBot") || userAgent.contains("AI2Bot") || userAgent.contains("Diffbot") || userAgent.contains("cohere-training-data-crawler") || userAgent.contains("Applebot-Extended")'
+      # SEO / Ads and marketing
+      - 'userAgent.contains("BLEXBot")'
+    action: drop
+
+  - name: unknown-crawlers
+    conditions:
+      # No user agent set
+      - 'userAgent == ""'
+    action: deny
+
+  # check a sequence of challenges
+  - name: suspicious-crawlers
+    conditions: ['($is-suspicious-crawler)']
+    action: none
+    children:
+      - name: 0
+        action: check
+        settings:
+          challenges: [js-pow-sha256]
+      - name: 1
+        action: check
+        settings:
+          challenges: [preload-link, resource-load]
+      - name: 2
+        action: check
+        settings:
+          challenges: [header-refresh]
+
+  - name: homesite
+    conditions:
+      - 'path == "/"'
+    action: pass
+
+  # check DNSBL and serve harder challenges
+  # todo: make this specific to score
+  - name: undesired-dnsbl
+    action: check
+    settings:
+      challenges: [dnsbl]
+      # if DNSBL fails, check additional challenges
+      fail: check
+      fail-settings:
+        challenges: [js-pow-sha256]
+
+  - name: suspicious-fetchers
+    action: check
+    settings:
+      challenges: [js-pow-sha256]
+    conditions:
+      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
+
+  # Allow PUT/DELETE/PATCH/POST requests in general
+  - name: non-get-request
+    action: pass
+    conditions:
+      - '!(method == "HEAD" || method == "GET")'
+
+  # Enable fetching OpenGraph and other tags from backend on these paths
+  - name: enable-meta-tags
+    action: context
+    settings:
+      context-set:
+        # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
+        proxy-meta-tags: "true"
+
+      # Set additional response headers
+      #response-headers:
+      # X-Clacks-Overhead:
+      #  - GNU Terry Pratchett
+
+  - name: plaintext-browser
+    action: challenge
+    settings:
+      challenges: [meta-refresh, cookie]
+    conditions:
+      - 'userAgent.startsWith("Lynx/")'
+
+  # Uncomment this rule out to challenge tool-like user agents
+  #- name: standard-tools
+  #  action: challenge
+  #  settings:
+  #    challenges: [cookie]
+  #  conditions:
+  #    - '($is-generic-robot-ua)'
+  #    - '($is-tool-ua)'
+  #    - '!($is-generic-browser)'
+
+  - name: standard-browser
+    action: challenge
+    settings:
+      challenges: [preload-link, meta-refresh, resource-load, js-pow-sha256]
+    conditions:
+      - '($is-generic-browser)'
+
+# If end of rules is reached, default is PASS

examples/snippets/bot-bingbot.yml

@@ -0,0 +1,8 @@
+networks:
+  bingbot:
+    - url: https://www.bing.com/toolbox/bingbot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+
+conditions:
+  is-bot-bingbot:
+      - &is-bot-bingbot 'userAgent.contains("+http://www.bing.com/bingbot.htm") && remoteAddress.network("bingbot")'

examples/snippets/bot-duckduckbot.yml

@@ -0,0 +1,8 @@
+networks:
+  duckduckbot:
+    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
+      regex: "<li><div>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</div></li>"
+
+conditions:
+  is-bot-duckduckbot:
+      - &is-bot-duckduckbot 'userAgent.contains("+http://duckduckgo.com/duckduckbot.html") && remoteAddress.network("duckduckbot")'

examples/snippets/bot-googlebot.yml

@@ -0,0 +1,8 @@
+networks:
+  googlebot:
+    - url: https://developers.google.com/static/search/apis/ipranges/googlebot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+
+conditions:
+  is-bot-googlebot:
+      - &is-bot-googlebot '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-PageRenderer") || userAgent.contains("Google-InspectionTool") || userAgent.contains("Googlebot")) && remoteAddress.network("googlebot")'

examples/snippets/bot-kagibot.yml

@@ -0,0 +1,8 @@
+networks:
+  kagibot:
+    - url: https://kagi.com/bot
+      regex: "\\n(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+) "
+
+conditions:
+  is-bot-kagibot:
+      - &is-bot-kagibot 'userAgent.contains("+https://kagi.com/bot") && remoteAddress.network("kagibot")'

examples/snippets/bot-qwantbot.yml

@@ -0,0 +1,8 @@
+networks:
+  qwantbot:
+    - url: https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+
+conditions:
+  is-bot-qwantbot:
+      - &is-bot-qwantbot 'userAgent.contains("+https://help.qwant.com/bot/") && remoteAddress.network("qwantbot")'

examples/snippets/bot-uptimerobot.yml

@@ -0,0 +1,8 @@
+networks:
+  uptimerobot:
+    - url: https://uptimerobot.com/inc/files/ips/IPv4andIPv6.txt
+      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+(/[0-9]+)?|[0-9a-f:]+:.+)"
+
+conditions:
+  is-bot-uptimerobot:
+      - &is-bot-uptimerobot 'userAgent.contains("http://www.uptimerobot.com/") && remoteAddress.network("uptimerobot")'

examples/snippets/bot-yandexbot.yml

@@ -0,0 +1,24 @@
+networks:
+  yandexbot:
+    # todo: detected as bot
+    # - url: https://yandex.com/ips
+    #  regex: "<span>(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+)[ \\\\t]*</span><br/>"
+    - prefixes:
+      - "5.45.192.0/18"
+      - "5.255.192.0/18"
+      - "37.9.64.0/18"
+      - "37.140.128.0/18"
+      - "77.88.0.0/18"
+      - "84.252.160.0/19"
+      - "87.250.224.0/19"
+      - "90.156.176.0/22"
+      - "93.158.128.0/18"
+      - "95.108.128.0/17"
+      - "141.8.128.0/18"
+      - "178.154.128.0/18"
+      - "185.32.187.0/24"
+      - "2a02:6b8::/29"
+
+conditions:
+  is-bot-yandexbot:
+      - &is-bot-yandexbot 'userAgent.contains("+http://yandex.com/bots") && remoteAddress.network("yandexbot")'

examples/snippets/challenge-dnsbl.yml

@@ -0,0 +1,6 @@
+challenges:
+  dnsbl:
+    runtime: dnsbl
+    parameters:
+      dnsbl-decay: 1h
+      dnsbl-timeout: 1s

examples/snippets/challenge-js-pow-sha256.yml

@@ -0,0 +1,15 @@
+challenges:
+  js-pow-sha256:
+    runtime: js
+    parameters:
+      # specifies the folder path that assets are under
+      # can be either embedded or external path
+      # defaults to name of challenge
+      path: "js-pow-sha256"
+      # needs to be under static folder
+      js-loader: load.mjs
+      # needs to be under runtime folder
+      wasm-runtime: runtime.wasm
+      wasm-runtime-settings:
+        difficulty: 20
+      verify-probability: 0.02

examples/snippets/challenges-non-js.yml

@@ -0,0 +1,28 @@
+challenges:
+  # Challenges with a cookie, self redirect (non-JS, requires HTTP parsing)
+  cookie:
+    runtime: "cookie"
+
+  # Challenges with a redirect via Link header with rel=preload and early hints (non-JS, requires HTTP parsing, fetching and logic)
+  # Works on HTTP/2 and above!
+  preload-link:
+    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
+    runtime: "preload-link"
+    parameters:
+      preload-early-hint-deadline: 3s
+
+  # Challenges with a redirect via Refresh header (non-JS, requires HTTP parsing and logic)
+  header-refresh:
+    runtime: "refresh"
+    parameters:
+      refresh-via: "header"
+
+  # Challenges with a redirect via Refresh meta (non-JS, requires HTML parsing and logic)
+  meta-refresh:
+    runtime: "refresh"
+    parameters:
+      refresh-via: "meta"
+
+  # Challenges with loading a random CSS or image document (non-JS, requires HTML parsing and logic)
+  resource-load:
+    runtime: "resource-load"

examples/snippets/conditions-generic.yml

@@ -0,0 +1,45 @@
+conditions:
+  is-well-known-asset:
+    - 'path == "/robots.txt"'
+    - 'path == "/favicon.ico"'
+    - 'path.startsWith("/.well-known")'
+
+  is-git-ua:
+    - 'userAgent.startsWith("git/") || userAgent.contains("libgit")'
+    - 'userAgent.startsWith("go-git")'
+    - 'userAgent.startsWith("JGit/") || userAgent.startsWith("JGit-")'
+    # Golang proxy and initial fetch
+    - 'userAgent.startsWith("GoModuleMirror/")'
+    - 'userAgent.startsWith("Go-http-client/") && "go-get" in query && query["go-get"] == "1"'
+    - '"Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"'
+
+  is-generic-browser:
+    - 'userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")'
+
+  is-generic-robot-ua:
+    - 'userAgent.matches("compatible[;)]") && !userAgent.contains("Trident/")'
+    - 'userAgent.matches("\\+https?://")'
+    - 'userAgent.contains("@")'
+    - 'userAgent.matches("[bB]ot/[0-9]")'
+
+  is-tool-ua:
+    - 'userAgent.startsWith("python-requests/")'
+    - 'userAgent.startsWith("Python-urllib/")'
+    - 'userAgent.startsWith("python-httpx/")'
+    - 'userAgent.contains("aoihttp/")'
+    - 'userAgent.startsWith("http.rb/")'
+    - 'userAgent.startsWith("curl/")'
+    - 'userAgent.startsWith("Wget/")'
+    - 'userAgent.startsWith("libcurl/")'
+    - 'userAgent.startsWith("okhttp/")'
+    - 'userAgent.startsWith("Java/")'
+    - 'userAgent.startsWith("Apache-HttpClient//")'
+    - 'userAgent.startsWith("Go-http-client/")'
+    - 'userAgent.startsWith("node-fetch/")'
+    - 'userAgent.startsWith("reqwest/")'
+
+  # Checks to detect a headless chromium via headers only
+  is-headless-chromium:
+    - 'userAgent.contains("HeadlessChrome") || userAgent.contains("HeadlessChromium")'
+    - '"Sec-Ch-Ua" in headers && (headers["Sec-Ch-Ua"].contains("HeadlessChrome") || headers["Sec-Ch-Ua"].contains("HeadlessChromium"))'
+    #- '(userAgent.contains("Chrome/") || userAgent.contains("Chromium/")) && (!("Accept-Language" in headers) || !("Accept-Encoding" in headers))'

examples/snippets/networks-other.yml

@@ -0,0 +1,37 @@
+networks:
+#  aws-cloud:
+#    - url: https://ip-ranges.amazonaws.com/ip-ranges.json
+#      jq-path: '(.prefixes[] | select(has("ip_prefix")) | .ip_prefix), (.prefixes[] | select(has("ipv6_prefix")) | .ipv6_prefix)'
+#  google-cloud:
+#    - url: https://www.gstatic.com/ipranges/cloud.json
+#      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+#  oracle-cloud:
+#    - url: https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json
+#      jq-path: '.regions[] | .cidrs[] | .cidr'
+#  azure-cloud:
+#    # todo: https://www.microsoft.com/en-us/download/details.aspx?id=56519 does not provide direct JSON
+#    - url: https://raw.githubusercontent.com/femueller/cloud-ip-ranges/refs/heads/master/microsoft-azure-ip-ranges.json
+#      jq-path: '.values[] | .properties.addressPrefixes[]'
+#
+#  digitalocean:
+#    - url: https://www.digitalocean.com/geo/google.csv
+#      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
+#  linode:
+#    - url: https://geoip.linode.com/
+#      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
+#  vultr:
+#    - url: "https://geofeed.constant.com/?json"
+#      jq-path: '.subnets[] | .ip_prefix'
+#  cloudflare:
+#    - url: https://www.cloudflare.com/ips-v4
+#      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
+#    - url: https://www.cloudflare.com/ips-v6
+#      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"
+#
+#  icloud-private-relay:
+#    - url: https://mask-api.icloud.com/egress-ip-ranges.csv
+#      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
+#  tunnelbroker-relay:
+#    # HE Tunnelbroker
+#    - url: https://tunnelbroker.net/export/google
+#      regex: "(?P<prefix>([0-9a-f:]+::)/[0-9]+),"

examples/spa.yml

@@ -0,0 +1,87 @@
+# Example cmdline (forward requests from upstream to port :8080)
+# $ go-away --bind :8080 --backend site.example.com=http://site:3000 --policy examples/spa.yml --policy-snippets example/snippets/ --challenge-template anubis
+
+
+
+# Define networks to be used later below
+networks:
+  # Networks will get included from snippets
+
+
+challenges:
+  # Challenges will get included from snippets
+  
+conditions:
+  # Conditions will get replaced on rules AST when found as ($condition-name)
+
+
+  is-static-asset:
+    - 'path == "/apple-touch-icon.png"'
+    - 'path == "/apple-touch-icon-precomposed.png"'
+    - 'path.matches("\\.(manifest|ttf|woff|woff2|jpg|jpeg|gif|png|webp|avif|svg|mp4|webm|css|js|mjs|wasm)$")'
+    # Add other paths where you have static assets
+    # - 'path.startsWith("/static/") || path.startsWith("/assets/")'
+
+
+# Rules are checked sequentially in order, from top to bottom
+rules:
+  - name: allow-well-known-resources
+    conditions:
+      - '($is-well-known-asset)'
+    action: pass
+
+  - name: allow-static-resources
+    conditions:
+      - '($is-static-asset)'
+    action: pass
+
+  - name: unknown-crawlers
+    conditions:
+      # No user agent set
+      - 'userAgent == ""'
+    action: deny
+
+  # Enable fetching OpenGraph and other tags from backend on index
+  - name: enable-meta-tags
+    action: context
+    conditions:
+      - 'path == "/" || path == "/index.html"'
+    settings:
+      context-set:
+        # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
+        proxy-meta-tags: "true"
+
+  # Challenge incoming visitors so challenge is remembered on api endpoints
+  # API requests will have this challenge stored
+  - name: index
+    conditions:
+      - 'path == "/" || path == "/index.html"'
+    settings:
+      challenges: [ preload-link, header-refresh ]
+    action: challenge
+
+  # Allow PUT/DELETE/PATCH/POST requests in general
+  - name: non-get-request
+    action: pass
+    conditions:
+      - '!(method == "HEAD" || method == "GET")'
+
+  # Challenge rest of endpoints (SPA API etc.)
+  # Above rule on index ensures clients have passed a challenge beforehand
+  - name: standard-browser
+    action: challenge
+    settings:
+      challenges: [ preload-link, header-refresh ]
+      # Fallback on cookie challenge
+      fail: challenge
+      fail-settings:
+        challenges: [ cookie ]
+    conditions:
+      - '($is-generic-browser)'
+
+  - name: other-fetchers
+    action: challenge
+    settings:
+      challenges: [ cookie ]
+    conditions:
+      - '!($is-generic-browser)'

go.mod

@@ -1,45 +1,41 @@
 module git.gammaspectra.live/git/go-away
 
-go 1.22.0
+go 1.24.0
 
-toolchain go1.22.12
+toolchain go1.24.2
 
 require (
+	codeberg.org/gone/http-cel v1.0.0
 	codeberg.org/meta/gzipped/v2 v2.0.0-20231111234332-aa70c3194756
-	github.com/andybalholm/brotli v1.1.1
-	github.com/go-jose/go-jose/v4 v4.0.5
-	github.com/google/cel-go v0.24.1
+	github.com/alphadose/haxmap v1.4.1
+	github.com/go-jose/go-jose/v4 v4.1.0
+	github.com/goccy/go-yaml v1.17.1
+	github.com/google/cel-go v0.25.0
 	github.com/itchyny/gojq v0.12.17
-	github.com/klauspost/compress v1.18.0
+	github.com/pires/go-proxyproto v0.8.0
+	github.com/prometheus/client_golang v1.22.0
 	github.com/tetratelabs/wazero v1.9.0
 	github.com/yl2chen/cidranger v1.0.2
-	golang.org/x/net v0.26.0
-	gopkg.in/yaml.v3 v3.0.1
+	golang.org/x/crypto v0.37.0
 )
 
 require (
 	cel.dev/expr v0.23.1 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.63.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
-	golang.org/x/crypto v0.33.0 // indirect
-	golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac // indirect
-	golang.org/x/text v0.22.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
+	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
+	golang.org/x/net v0.39.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 )
-
-// Used by github.com/antlr4-go/antlr v4.13.0 via github.com/google/cel-go
-// Ensure we have no other exp package usages by only proxying the slices functions in that package
-// Newer versions than v0.0.0-20250210185358-939b2ce775ac are not supported by Go 1.22
-replace golang.org/x/exp v0.0.0 => ./utils/exp
-
-// Pin latest versions to support Go 1.22 to prevent a package update from changing them
-// TODO: remove this when Go 1.22+ is supported by other higher users
-replace (
-	golang.org/x/crypto => golang.org/x/crypto v0.33.0
-	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7
-	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7
-)

go.sum

@@ -1,20 +1,28 @@
 cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg=
 cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
+codeberg.org/gone/http-cel v1.0.0 h1:flEv/KzEye4W7vjwkdAkwo7VCbuj9xZLjyTn/rjWFDQ=
+codeberg.org/gone/http-cel v1.0.0/go.mod h1:uRkxygsQp5EFE3e9dRkJ4HK453G5YZDHCq9DEG5CoDw=
 codeberg.org/meta/gzipped/v2 v2.0.0-20231111234332-aa70c3194756 h1:bDqEUEYt4UJy8mfLCZeJuXx+xNJvdqTbkE4Ci11NQYU=
 codeberg.org/meta/gzipped/v2 v2.0.0-20231111234332-aa70c3194756/go.mod h1:aJ/ghJW7viYfwZ6OizDst+uJgbb6r/Hvoqhmi1OPTTw=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/alphadose/haxmap v1.4.1 h1:VtD6VCxUkjNIfJk/aWdYFfOzrRddDFjmvmRmILg7x8Q=
+github.com/alphadose/haxmap v1.4.1/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
 github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
-github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
-github.com/google/cel-go v0.24.1 h1:jsBCtxG8mM5wiUJDSGUqU0K7Mtr3w7Eyv00rw4DiZxI=
-github.com/google/cel-go v0.24.1/go.mod h1:Hdf9TqOaTNSFQA1ybQaRqATVoK7m/zcf7IMhGXP5zI8=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY=
+github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw=
+github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY=
+github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY=
+github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/itchyny/gojq v0.12.17 h1:8av8eGduDb5+rvEdaOO+zQUjA04MS0m3Ps8HiD+fceg=
 github.com/itchyny/gojq v0.12.17/go.mod h1:WBrEMkgAfAGO1LUcGOckBl5O726KPp+OlkKug0I/FEY=
 github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
@@ -23,8 +31,22 @@ github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 h1:Pdirg1gwhEcGjM
 github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43/go.mod h1:ahLMuLCUyDdXqtqGyuwGev7/PGtO7r7ocvdwDuEN/3E=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pires/go-proxyproto v0.8.0 h1:5unRmEAPbHXHuLjDg01CxJWf91cw3lKHc/0xzKpXEe0=
+github.com/pires/go-proxyproto v0.8.0/go.mod h1:iknsfgnH8EkjrMeMyvfKByp9TiBZCKZM0jx2xmKqnVY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k=
+github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
 github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -38,25 +60,24 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
 github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
 github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs=
-golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw=
-google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
+golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f h1:tjZsroqekhC63+WMqzmWyW5Twj/ZfR5HAlpd5YQ1Vs0=
+google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:Cd8IzgPo5Akum2c9R6FsXNaZbH3Jpa2gpHlW89FqlyQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f h1:N/PrbTw4kdkqNRzVfWPrBekzLuarFREcbFOiOLkXon4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
 google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go.work

@@ -1,6 +0,0 @@
-go 1.22.0
-
-use (
-	.
-	utils/exp
-)

go.work.sum

@@ -1,26 +0,0 @@
-github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
-golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
-google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

lib/action/block.go

@@ -0,0 +1,36 @@
+package action
+
+import (
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionBLOCK] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return Block{
+			Code:     http.StatusForbidden,
+			RuleHash: ruleHash,
+		}, nil
+	}
+}
+
+type Block struct {
+	Code     int
+	RuleHash string
+}
+
+func (a Block) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	logger.Info("request blocked")
+	data := challenge.RequestDataFromContext(r.Context())
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("Connection", "close")
+	w.WriteHeader(a.Code)
+	_, _ = w.Write([]byte(fmt.Errorf("access blocked: blocked by administrative rule %s/%s", data.Id.String(), a.RuleHash).Error()))
+
+	return false, nil
+}

lib/action/challenge.go

@@ -0,0 +1,193 @@
+package action
+
+import (
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+	"strings"
+)
+
+func init() {
+	i := func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node, cont bool) (Handler, error) {
+		params := ChallengeDefaultSettings
+
+		if settings != nil {
+			ymlData, err := settings.MarshalYAML()
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.Unmarshal(ymlData, &params)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		if params.Code == 0 {
+			params.Code = state.Settings().ChallengeResponseCode
+		}
+
+		var regs []*challenge.Registration
+		for _, regName := range params.Challenges {
+			if reg, ok := state.GetChallengeByName(regName); ok {
+				regs = append(regs, reg)
+			} else {
+				return nil, fmt.Errorf("challenge %s not found", regName)
+			}
+		}
+
+		if len(regs) == 0 {
+			return nil, fmt.Errorf("no registered challenges found in rule %s", ruleName)
+		}
+
+		passAction := policy.RuleAction(strings.ToUpper(params.PassAction))
+		passHandler, ok := Register[passAction]
+		if !ok {
+			return nil, fmt.Errorf("unknown pass action %s", params.PassAction)
+		}
+
+		passActionHandler, err := passHandler(state, ruleName, ruleHash, params.PassSettings)
+		if err != nil {
+			return nil, err
+		}
+
+		failAction := policy.RuleAction(strings.ToUpper(params.FailAction))
+		failHandler, ok := Register[failAction]
+		if !ok {
+			return nil, fmt.Errorf("unknown pass action %s", params.FailAction)
+		}
+
+		failActionHandler, err := failHandler(state, ruleName, ruleHash, params.FailSettings)
+		if err != nil {
+			return nil, err
+		}
+
+		return Challenge{
+			RuleHash:   ruleHash,
+			Code:       params.Code,
+			Continue:   cont,
+			Challenges: regs,
+
+			PassAction:        passAction,
+			PassActionHandler: passActionHandler,
+			FailAction:        failAction,
+			FailActionHandler: failActionHandler,
+		}, nil
+	}
+	Register[policy.RuleActionCHALLENGE] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return i(state, ruleName, ruleHash, settings, false)
+	}
+	Register[policy.RuleActionCHECK] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return i(state, ruleName, ruleHash, settings, true)
+	}
+}
+
+var ChallengeDefaultSettings = ChallengeSettings{
+	PassAction: string(policy.RuleActionPASS),
+	FailAction: string(policy.RuleActionDENY),
+}
+
+type ChallengeSettings struct {
+	Code       int      `yaml:"http-code"`
+	Challenges []string `yaml:"challenges"`
+
+	PassAction   string   `yaml:"pass"`
+	PassSettings ast.Node `yaml:"pass-settings"`
+
+	// FailAction Executed in case no challenges match or
+	FailAction   string   `yaml:"fail"`
+	FailSettings ast.Node `yaml:"fail-settings"`
+}
+
+type Challenge struct {
+	RuleHash   string
+	Code       int
+	Continue   bool
+	Challenges []*challenge.Registration
+
+	PassAction        policy.RuleAction
+	PassActionHandler Handler
+	FailAction        policy.RuleAction
+	FailActionHandler Handler
+}
+
+func (a Challenge) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	data := challenge.RequestDataFromContext(r.Context())
+	for _, reg := range a.Challenges {
+		if data.HasValidChallenge(reg.Id()) {
+
+			data.State.ChallengeChecked(r, reg, r.URL.String(), logger)
+
+			if a.Continue {
+				return true, nil
+			}
+
+			// we passed!
+			data.State.ActionHit(r, a.PassAction, logger)
+			return a.PassActionHandler.Handle(logger.With("challenge", reg.Name), w, r, done)
+		}
+	}
+	// none matched, issue challenges in sequential priority
+	for _, reg := range a.Challenges {
+		result := data.ChallengeVerify[reg.Id()]
+		state := data.ChallengeState[reg.Id()]
+		if result.Ok() || result == challenge.VerifyResultSkip || state == challenge.VerifyStatePass {
+			// skip already ok'd challenges for some reason (TODO: why)
+			// also skip skipped challenges due to preconditions
+			continue
+		}
+
+		expiry := data.Expiration(reg.Duration)
+		key := challenge.GetChallengeKeyForRequest(data.State, reg, expiry, r)
+		result = reg.IssueChallenge(w, r, key, expiry)
+		if result != challenge.VerifyResultSkip {
+			data.State.ChallengeIssued(r, reg, r.URL.String(), logger)
+		}
+		data.ChallengeVerify[reg.Id()] = result
+		data.ChallengeState[reg.Id()] = challenge.VerifyStatePass
+		switch result {
+		case challenge.VerifyResultOK:
+			data.State.ChallengePassed(r, reg, r.URL.String(), logger)
+			if a.Continue {
+				return true, nil
+			}
+
+			data.State.ActionHit(r, a.PassAction, logger)
+			return a.PassActionHandler.Handle(logger.With("challenge", reg.Name), w, r, done)
+		case challenge.VerifyResultNotOK:
+			// we have had the challenge checked, but it's not ok!
+			// safe to continue
+			continue
+		case challenge.VerifyResultFail:
+			err := fmt.Errorf("challenge %s failed on issuance", reg.Name)
+			data.State.ChallengeFailed(r, reg, err, r.URL.String(), logger)
+
+			if reg.Class == challenge.ClassTransparent {
+				// allow continuing transparent challenges
+				continue
+			}
+
+			data.State.ActionHit(r, a.FailAction, logger)
+			return a.FailActionHandler.Handle(logger, w, r, done)
+		case challenge.VerifyResultNone:
+			// challenge was issued
+			if reg.Class == challenge.ClassTransparent {
+				// allow continuing transparent challenges
+				continue
+			}
+			// we cannot continue after issuance
+			return false, nil
+
+		case challenge.VerifyResultSkip:
+			// continue onto next one due to precondition
+			continue
+		}
+	}
+
+	// nothing matched, execute default action
+	data.State.ActionHit(r, a.FailAction, logger)
+	return a.FailActionHandler.Handle(logger, w, r, done)
+}

lib/action/code.go

@@ -0,0 +1,47 @@
+package action
+
+import (
+	"errors"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionCODE] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		params := CodeDefaultSettings
+
+		if settings != nil {
+			ymlData, err := settings.MarshalYAML()
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.Unmarshal(ymlData, &params)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		if params.Code == 0 {
+			return nil, errors.New("http-code not set")
+		}
+
+		return Code(params.Code), nil
+	}
+}
+
+var CodeDefaultSettings = CodeSettings{}
+
+type CodeSettings struct {
+	Code int `yaml:"http-code"`
+}
+
+type Code int
+
+func (a Code) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	w.WriteHeader(int(a))
+	return false, nil
+}

lib/action/context.go

@@ -0,0 +1,55 @@
+package action
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionCONTEXT] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		params := ContextDefaultSettings
+
+		if settings != nil {
+			ymlData, err := settings.MarshalYAML()
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.Unmarshal(ymlData, &params)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return Context{
+			opts: params,
+		}, nil
+	}
+}
+
+var ContextDefaultSettings = ContextSettings{}
+
+type ContextSettings struct {
+	ContextSet      map[string]string `yaml:"context-set"`
+	ResponseHeaders map[string]string `yaml:"response-headers"`
+}
+
+type Context struct {
+	opts ContextSettings
+}
+
+func (a Context) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	data := challenge.RequestDataFromContext(r.Context())
+	for k, v := range a.opts.ContextSet {
+		data.SetOpt(k, v)
+	}
+
+	for k, v := range a.opts.ResponseHeaders {
+		w.Header().Set(k, v)
+	}
+
+	return true, nil
+}

lib/action/deny.go

@@ -0,0 +1,31 @@
+package action
+
+import (
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionDENY] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return Deny{
+			Code:     http.StatusForbidden,
+			RuleHash: ruleHash,
+		}, nil
+	}
+}
+
+type Deny struct {
+	Code     int
+	RuleHash string
+}
+
+func (a Deny) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	logger.Info("request denied")
+	data := challenge.RequestDataFromContext(r.Context())
+	data.State.ErrorPage(w, r, a.Code, fmt.Errorf("access denied: denied by administrative rule %s/%s", data.Id.String(), a.RuleHash), "")
+	return false, nil
+}

lib/action/drop.go

@@ -0,0 +1,39 @@
+package action
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionDROP] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return Drop{}, nil
+	}
+}
+
+type Drop struct {
+}
+
+func (a Drop) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	logger.Info("request dropped")
+
+	if hj, ok := w.(http.Hijacker); ok {
+		if conn, _, err := hj.Hijack(); err == nil {
+			// drop without sending data
+			_ = conn.Close()
+			return false, nil
+		}
+	}
+
+	// fallback
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("Content-Length", "0")
+	w.Header().Set("Connection", "close")
+	w.WriteHeader(http.StatusForbidden)
+
+	return false, nil
+}

lib/action/none.go

@@ -0,0 +1,21 @@
+package action
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionNONE] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return None{}, nil
+	}
+}
+
+type None struct{}
+
+func (a None) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	return true, nil
+}

lib/action/pass.go

@@ -0,0 +1,23 @@
+package action
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+func init() {
+	Register[policy.RuleActionPASS] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		return Pass{}, nil
+	}
+}
+
+type Pass struct{}
+
+func (a Pass) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	logger.Debug("request passed")
+	done().ServeHTTP(w, r)
+	return false, nil
+}

lib/action/proxy.go

@@ -0,0 +1,80 @@
+package action
+
+import (
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+	"regexp"
+)
+
+func init() {
+	Register[policy.RuleActionPROXY] = func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error) {
+		params := ProxyDefaultSettings
+
+		if settings != nil {
+			ymlData, err := settings.MarshalYAML()
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.Unmarshal(ymlData, &params)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		if params.Match != "" {
+			expr, err := regexp.Compile(params.Match)
+			if err != nil {
+				return nil, err
+			}
+
+			return Proxy{
+				Match:   expr,
+				Rewrite: params.Rewrite,
+				Backend: params.Backend,
+			}, nil
+		}
+
+		return Proxy{
+			Backend: params.Backend,
+		}, nil
+	}
+}
+
+var ProxyDefaultSettings = ProxySettings{}
+
+type ProxySettings struct {
+	Match   string `yaml:"proxy-match"`
+	Rewrite string `yaml:"proxy-rewrite"`
+	Backend string `yaml:"proxy-backend"`
+}
+
+type Proxy struct {
+	Match   *regexp.Regexp
+	Rewrite string
+	Backend string
+}
+
+func (a Proxy) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
+	data := challenge.RequestDataFromContext(r.Context())
+
+	backend := data.State.GetBackend(a.Backend)
+	if backend == nil {
+		return false, fmt.Errorf("backend for %s not found", a.Backend)
+	}
+
+	if a.Match != nil {
+		// rewrite query
+		r.URL.Path = a.Match.ReplaceAllString(r.URL.Path, a.Rewrite)
+	}
+
+	// set headers, ignore reply
+	_ = done()
+	backend.ServeHTTP(w, r)
+
+	return false, nil
+}

lib/action/register.go

@@ -0,0 +1,20 @@
+package action
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/goccy/go-yaml/ast"
+	"log/slog"
+	"net/http"
+)
+
+type Handler interface {
+	// Handle An incoming request.
+	// If next is true, continue processing
+	// If next is false, stop processing. If passing to a backend, done() must be called beforehand to set headers.
+	Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error)
+}
+
+type NewFunc func(state challenge.StateInterface, ruleName, ruleHash string, settings ast.Node) (Handler, error)
+
+var Register = make(map[policy.RuleAction]NewFunc)

lib/challenge.go

@@ -1,82 +1,13 @@
 package lib
 
 import (
-	"crypto/sha256"
-	"encoding/binary"
-	"github.com/go-jose/go-jose/v4/jwt"
-	"net"
-	"net/http"
-	"strings"
-	"time"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/cookie"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/dnsbl"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/http"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/preload-link"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/refresh"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/resource-load"
+	_ "git.gammaspectra.live/git/go-away/lib/challenge/wasm"
 )
 
-type ChallengeInformation struct {
-	Name   string `json:"name"`
-	Key    []byte `json:"key"`
-	Result []byte `json:"result"`
-
-	Expiry    *jwt.NumericDate `json:"exp,omitempty"`
-	NotBefore *jwt.NumericDate `json:"nbf,omitempty"`
-	IssuedAt  *jwt.NumericDate `json:"iat,omitempty"`
-}
-
-func getRequestScheme(r *http.Request) string {
-	if proto := r.Header.Get("X-Forwarded-Proto"); proto == "http" || proto == "https" {
-		return proto
-	}
-
-	if r.TLS != nil {
-		return "https"
-	}
-
-	return "http"
-}
-
-func getRequestAddress(r *http.Request, clientHeader string) net.IP {
-	var ipStr string
-	if clientHeader != "" {
-		ipStr = r.Header.Get(clientHeader)
-	}
-	if ipStr != "" {
-		// handle X-Forwarded-For
-		ipStr = strings.Split(ipStr, ",")[0]
-	}
-
-	// fallback
-	if ipStr == "" {
-		parts := strings.Split(r.RemoteAddr, ":")
-		// drop port
-		ipStr = strings.Join(parts[:len(parts)-1], ":")
-	}
-	ipStr = strings.Trim(ipStr, "[]")
-	return net.ParseIP(ipStr)
-}
-
-func (state *State) GetChallengeKeyForRequest(challengeName string, until time.Time, r *http.Request) []byte {
-	hasher := sha256.New()
-	hasher.Write([]byte("challenge\x00"))
-	hasher.Write([]byte(challengeName))
-	hasher.Write([]byte{0})
-	hasher.Write(getRequestAddress(r, state.Settings.ClientIpHeader).To16())
-	hasher.Write([]byte{0})
-
-	// specific headers
-	for _, k := range []string{
-		"Accept-Language",
-		// General browser information
-		"User-Agent",
-		// TODO: not sent in preload
-		//"Sec-Ch-Ua",
-		//"Sec-Ch-Ua-Platform",
-	} {
-		hasher.Write([]byte(r.Header.Get(k)))
-		hasher.Write([]byte{0})
-	}
-	hasher.Write([]byte{0})
-	_ = binary.Write(hasher, binary.LittleEndian, until.UTC().Unix())
-	hasher.Write([]byte{0})
-	hasher.Write(state.publicKey)
-	hasher.Write([]byte{0})
-
-	return hasher.Sum(nil)
-}
+// This file loads embedded challenge runtimes so their init() is called

lib/challenge/awaiter.go

@@ -0,0 +1,47 @@
+package challenge
+
+import (
+	"context"
+	"github.com/alphadose/haxmap"
+	"sync/atomic"
+)
+
+type awaiterCallback func(result VerifyResult)
+
+type Awaiter[K ~string | ~int64 | ~uint64] haxmap.Map[K, awaiterCallback]
+
+func NewAwaiter[T ~string | ~int64 | ~uint64]() *Awaiter[T] {
+	return (*Awaiter[T])(haxmap.New[T, awaiterCallback]())
+}
+
+func (a *Awaiter[T]) Await(key T, ctx context.Context) VerifyResult {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var result atomic.Int64
+
+	a.m().Set(key, func(receivedResult VerifyResult) {
+		result.Store(int64(receivedResult))
+		cancel()
+	})
+	// cleanup
+	defer a.m().Del(key)
+
+	<-ctx.Done()
+
+	return VerifyResult(result.Load())
+}
+
+func (a *Awaiter[T]) Solve(key T, result VerifyResult) {
+	if f, ok := a.m().GetAndDel(key); ok && f != nil {
+		f(result)
+	}
+}
+
+func (a *Awaiter[T]) m() *haxmap.Map[T, awaiterCallback] {
+	return (*haxmap.Map[T, awaiterCallback])(a)
+}
+
+func (a *Awaiter[T]) Close() error {
+	return nil
+}

lib/challenge/cookie/cookie.go

@@ -0,0 +1,38 @@
+package cookie
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/goccy/go-yaml/ast"
+	"net/http"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes[Key] = FillRegistration
+}
+
+const Key = "cookie"
+
+func FillRegistration(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	reg.Class = challenge.ClassBlocking
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+		token, err := reg.IssueChallengeToken(state.PrivateKey(), key, nil, expiry, true)
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+
+		utils.SetCookie(challenge.RequestDataFromContext(r.Context()).CookiePrefix+reg.Name, token, expiry, w, r)
+
+		uri, err := challenge.RedirectUrl(r, reg)
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+
+		http.Redirect(w, r, uri.String(), http.StatusTemporaryRedirect)
+		return challenge.VerifyResultNone
+	}
+
+	return nil
+}

lib/challenge/data.go

@@ -0,0 +1,260 @@
+package challenge
+
+import (
+	http_cel "codeberg.org/gone/http-cel"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/traits"
+	"net/http"
+	"net/netip"
+	"net/textproto"
+	"strings"
+	"time"
+)
+
+type requestDataContextKey struct {
+}
+
+func RequestDataFromContext(ctx context.Context) *RequestData {
+	return ctx.Value(requestDataContextKey{}).(*RequestData)
+}
+
+type RequestId [16]byte
+
+func (id RequestId) String() string {
+	return hex.EncodeToString(id[:])
+}
+
+type RequestData struct {
+	Id              RequestId
+	Time            time.Time
+	ChallengeVerify map[Id]VerifyResult
+	ChallengeState  map[Id]VerifyState
+	RemoteAddress   netip.AddrPort
+	State           StateInterface
+	CookiePrefix    string
+
+	r *http.Request
+
+	fp     map[string]string
+	header traits.Mapper
+	query  traits.Mapper
+
+	opts map[string]string
+}
+
+func CreateRequestData(r *http.Request, state StateInterface) (*http.Request, *RequestData) {
+
+	var data RequestData
+	// generate random id, todo: is this fast?
+	_, _ = rand.Read(data.Id[:])
+	data.RemoteAddress = utils.GetRequestAddress(r, state.Settings().ClientIpHeader)
+	data.ChallengeVerify = make(map[Id]VerifyResult, len(state.GetChallenges()))
+	data.ChallengeState = make(map[Id]VerifyState, len(state.GetChallenges()))
+	data.Time = time.Now().UTC()
+	data.State = state
+
+	data.fp = make(map[string]string, 2)
+
+	if fp := utils.GetTLSFingerprint(r); fp != nil {
+		if ja3nPtr := fp.JA3N(); ja3nPtr != nil {
+			ja3n := ja3nPtr.String()
+			data.fp["ja3n"] = ja3n
+			r.Header.Set("X-TLS-Fingerprint-JA3N", ja3n)
+		}
+		if ja4Ptr := fp.JA4(); ja4Ptr != nil {
+			ja4 := ja4Ptr.String()
+			data.fp["ja4"] = ja4
+			r.Header.Set("X-TLS-Fingerprint-JA4", ja4)
+		}
+	}
+
+	q := r.URL.Query()
+	// delete query parameters that were set by go-away
+	for k := range q {
+		if strings.HasPrefix(k, QueryArgPrefix) {
+			q.Del(k)
+		}
+	}
+
+	data.query = http_cel.NewValuesMap(q)
+	data.header = http_cel.NewMIMEMap(textproto.MIMEHeader(r.Header))
+	data.opts = make(map[string]string)
+
+	sum := sha256.New()
+	sum.Write([]byte(r.Host))
+	sum.Write([]byte{0})
+	sum.Write(data.NetworkPrefix().AsSlice())
+	sum.Write([]byte{0})
+	sum.Write(state.PublicKey())
+	sum.Write([]byte{0})
+	data.CookiePrefix = utils.CookiePrefix + hex.EncodeToString(sum.Sum(nil)[:6]) + "-"
+
+	r = r.WithContext(context.WithValue(r.Context(), requestDataContextKey{}, &data))
+	r = utils.SetRemoteAddress(r, data.RemoteAddress)
+	data.r = r
+
+	return r, &data
+}
+
+func (d *RequestData) ResolveName(name string) (any, bool) {
+	switch name {
+	case "host":
+		return d.r.Host, true
+	case "method":
+		return d.r.Method, true
+	case "remoteAddress":
+		return d.RemoteAddress.Addr().AsSlice(), true
+	case "userAgent":
+		return d.r.UserAgent(), true
+	case "path":
+		return d.r.URL.Path, true
+	case "query":
+		return d.query, true
+	case "headers":
+		return d.header, true
+	case "fp":
+		return d.fp, true
+	default:
+		return nil, false
+	}
+}
+
+func (d *RequestData) Parent() cel.Activation {
+	return nil
+}
+
+func (d *RequestData) NetworkPrefix() netip.Addr {
+	address := d.RemoteAddress.Addr().Unmap()
+	if address.Is4() {
+		// Take a /24 for IPv4
+		prefix, _ := address.Prefix(24)
+		return prefix.Addr()
+	} else {
+		// Take a /64 for IPv6
+		prefix, _ := address.Prefix(64)
+		return prefix.Addr()
+	}
+}
+
+const (
+	RequestOptBackendHost   = "backend-host"
+	RequestOptCacheMetaTags = "proxy-meta-tags"
+)
+
+func (d *RequestData) SetOpt(n, v string) {
+	d.opts[n] = v
+}
+
+func (d *RequestData) GetOpt(n, def string) string {
+	v, ok := d.opts[n]
+	if !ok {
+		return def
+	}
+	return v
+}
+
+func (d *RequestData) GetOptBool(n string, def bool) bool {
+	v, ok := d.opts[n]
+	if !ok {
+		return def
+	}
+	switch v {
+	case "true", "t", "1", "yes", "yep", "y", "ok":
+		return true
+	case "false", "f", "0", "no", "nope", "n", "err":
+		return false
+	default:
+		return def
+	}
+}
+
+func (d *RequestData) BackendHost() (http.Handler, string) {
+	host := d.r.Host
+
+	if opt := d.GetOpt(RequestOptBackendHost, ""); opt != "" && opt != host {
+		host = d.r.Host
+	}
+
+	return d.State.GetBackend(host), host
+}
+
+func (d *RequestData) EvaluateChallenges(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	var issuedChallenge string
+	if q.Has(QueryArgChallenge) {
+		issuedChallenge = q.Get(QueryArgChallenge)
+	}
+	for _, reg := range d.State.GetChallenges() {
+		key := GetChallengeKeyForRequest(d.State, reg, d.Expiration(reg.Duration), r)
+		verifyResult, verifyState, err := reg.VerifyChallengeToken(d.State.PublicKey(), key, r)
+		if err != nil && !errors.Is(err, http.ErrNoCookie) {
+			// clear invalid cookie
+			utils.ClearCookie(d.CookiePrefix+reg.Name, w, r)
+		}
+
+		// prevent evaluating the challenge if not solved
+		if !verifyResult.Ok() && reg.Condition != nil {
+			out, _, err := reg.Condition.Eval(d)
+			// verify eligibility
+			if err != nil {
+				d.State.Logger(r).Error(err.Error(), "challenge", reg.Name)
+			} else if out != nil && out.Type() == types.BoolType {
+				if out.Equal(types.True) != types.True {
+					// skip challenge match due to precondition!
+					verifyResult = VerifyResultSkip
+					continue
+				}
+			}
+		}
+
+		if !verifyResult.Ok() && issuedChallenge == reg.Name {
+			// we issued the challenge, must skip to prevent loops
+			verifyResult = VerifyResultSkip
+		}
+		d.ChallengeVerify[reg.Id()] = verifyResult
+		d.ChallengeState[reg.Id()] = verifyState
+	}
+
+	if d.State.Settings().BackendIpHeader != "" {
+		if d.State.Settings().ClientIpHeader != "" {
+			r.Header.Del(d.State.Settings().ClientIpHeader)
+		}
+		r.Header.Set(d.State.Settings().BackendIpHeader, d.RemoteAddress.String())
+	}
+
+	// send these to client so we consistently get the headers
+	//w.Header().Set("Accept-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
+	//w.Header().Set("Critical-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
+}
+
+func (d *RequestData) Expiration(duration time.Duration) time.Time {
+	return d.Time.Add(duration).Round(duration)
+}
+
+func (d *RequestData) HasValidChallenge(id Id) bool {
+	return d.ChallengeVerify[id].Ok()
+}
+
+func (d *RequestData) RequestHeaders(headers http.Header) {
+	headers.Set("X-Away-Id", d.Id.String())
+
+	for id, result := range d.ChallengeVerify {
+		if result.Ok() {
+			c, ok := d.State.GetChallenge(id)
+			if !ok {
+				panic("challenge not found")
+			}
+
+			headers.Set(fmt.Sprintf("X-Away-Challenge-%s-Result", c.Name), result.String())
+			headers.Set(fmt.Sprintf("X-Away-Challenge-%s-State", c.Name), d.ChallengeState[id].String())
+		}
+	}
+}

lib/challenge/dnsbl/dnsbl.go

@@ -0,0 +1,145 @@
+package http
+
+import (
+	"context"
+	"errors"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"net"
+	"net/http"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes[Key] = FillRegistration
+}
+
+const Key = "dnsbl"
+
+type Parameters struct {
+	VerifyProbability float64       `yaml:"verify-probability"`
+	Host              string        `yaml:"dnsbl-host"`
+	Timeout           time.Duration `yaml:"dnsbl-timeout"`
+	Decay             time.Duration `yaml:"dnsbl-decay"`
+}
+
+var DefaultParameters = Parameters{
+	VerifyProbability: 0.10,
+	Timeout:           time.Second * 1,
+	Decay:             time.Hour * 1,
+	Host:              "dnsbl.dronebl.org",
+}
+
+func lookup(ctx context.Context, decay, timeout time.Duration, dnsbl *utils.DNSBL, decayMap *utils.DecayMap[[net.IPv6len]byte, utils.DNSBLResponse], ip net.IP) (utils.DNSBLResponse, error) {
+	var key [net.IPv6len]byte
+	copy(key[:], ip.To16())
+
+	result, ok := decayMap.Get(key)
+	if ok {
+		return result, nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	result, err := dnsbl.Lookup(ctx, ip)
+	if err != nil {
+
+	}
+	decayMap.Set(key, result, decay)
+
+	return result, err
+}
+
+type closer chan struct{}
+
+func (c closer) Close() error {
+	select {
+	case <-c:
+	default:
+		close(c)
+	}
+	return nil
+}
+
+func FillRegistration(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	params := DefaultParameters
+
+	if parameters != nil {
+		ymlData, err := parameters.MarshalYAML()
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(ymlData, &params)
+		if err != nil {
+			return err
+		}
+	}
+
+	if params.Host == "" {
+		return errors.New("empty host")
+	}
+
+	reg.Class = challenge.ClassTransparent
+
+	if params.VerifyProbability <= 0 {
+		//20% default
+		params.VerifyProbability = 0.20
+	} else if params.VerifyProbability > 1.0 {
+		params.VerifyProbability = 1.0
+	}
+	reg.VerifyProbability = params.VerifyProbability
+
+	decayMap := utils.NewDecayMap[[net.IPv6len]byte, utils.DNSBLResponse]()
+
+	dnsbl := utils.NewDNSBL(params.Host, &net.Resolver{
+		PreferGo: true,
+	})
+
+	ob := make(closer)
+
+	go func() {
+		ticker := time.NewTicker(params.Timeout / 3)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				decayMap.Decay()
+			case <-ob:
+				return
+			}
+		}
+	}()
+
+	// allow freeing the ticker/decay map
+	reg.Object = ob
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+
+		data := challenge.RequestDataFromContext(r.Context())
+
+		result, err := lookup(r.Context(), params.Decay, params.Timeout, dnsbl, decayMap, data.RemoteAddress.Addr().Unmap().AsSlice())
+		if err != nil {
+			data.State.Logger(r).Debug("dnsbl lookup failed", "address", data.RemoteAddress.Addr().String(), "result", result, "err", err)
+		}
+
+		if result.Bad() {
+			token, err := reg.IssueChallengeToken(state.PrivateKey(), key, nil, expiry, false)
+			if err != nil {
+				return challenge.VerifyResultFail
+			}
+			utils.SetCookie(data.CookiePrefix+reg.Name, token, expiry, w, r)
+			return challenge.VerifyResultNotOK
+		} else {
+			token, err := reg.IssueChallengeToken(state.PrivateKey(), key, nil, expiry, true)
+			if err != nil {
+				return challenge.VerifyResultFail
+			}
+			utils.SetCookie(data.CookiePrefix+reg.Name, token, expiry, w, r)
+			return challenge.VerifyResultOK
+		}
+	}
+
+	return nil
+}

lib/challenge/helper.go

@@ -0,0 +1,203 @@
+package challenge
+
+import (
+	"crypto/subtle"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/utils"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+var ErrInvalidToken = errors.New("invalid token")
+var ErrMismatchedToken = errors.New("mismatched token")
+var ErrMismatchedTokenHappyEyeballs = errors.New("mismatched token: IPv4 to IPv6 upgrade detected, retrying")
+
+func NewKeyVerifier() (verify VerifyFunc, issue func(key Key) string) {
+	return func(key Key, token []byte, r *http.Request) (VerifyResult, error) {
+			expectedKey, err := hex.DecodeString(string(token))
+			if err != nil {
+				return VerifyResultFail, err
+			}
+			if len(expectedKey) != KeySize {
+				return VerifyResultFail, ErrInvalidToken
+			}
+			if subtle.ConstantTimeCompare(key[:], expectedKey) == 1 {
+				return VerifyResultOK, nil
+			}
+
+			kk := Key(expectedKey)
+			// IPv4 -> IPv6 Happy Eyeballs
+			if key.Get(KeyFlagIsIPv4) == 0 && kk.Get(KeyFlagIsIPv4) > 0 {
+				return VerifyResultOK, ErrMismatchedTokenHappyEyeballs
+			}
+
+			return VerifyResultFail, ErrMismatchedToken
+		}, func(key Key) string {
+			return hex.EncodeToString(key[:])
+		}
+}
+
+const (
+	QueryArgPrefix    = "__goaway"
+	QueryArgReferer   = QueryArgPrefix + "_referer"
+	QueryArgRedirect  = QueryArgPrefix + "_redirect"
+	QueryArgRequestId = QueryArgPrefix + "_id"
+	QueryArgChallenge = QueryArgPrefix + "_challenge"
+	QueryArgToken     = QueryArgPrefix + "_token"
+)
+
+const MakeChallengeUrlSuffix = "/make-challenge"
+const VerifyChallengeUrlSuffix = "/verify-challenge"
+
+func GetVerifyInformation(r *http.Request, reg *Registration) (requestId RequestId, redirect, token string, err error) {
+
+	q := r.URL.Query()
+
+	if q.Get(QueryArgChallenge) != reg.Name {
+		return RequestId{}, "", "", fmt.Errorf("unexpected challenge: got \"%s\"", q.Get(QueryArgChallenge))
+	}
+
+	requestIdHex := q.Get(QueryArgRequestId)
+
+	if len(requestId) != hex.DecodedLen(len(requestIdHex)) {
+		return RequestId{}, "", "", errors.New("invalid request id")
+	}
+	n, err := hex.Decode(requestId[:], []byte(requestIdHex))
+	if err != nil {
+		return RequestId{}, "", "", err
+	} else if n != len(requestId) {
+		return RequestId{}, "", "", errors.New("invalid request id")
+	}
+
+	token = q.Get(QueryArgToken)
+	redirect, err = utils.EnsureNoOpenRedirect(q.Get(QueryArgRedirect))
+	if err != nil {
+		return RequestId{}, "", "", err
+	}
+	return
+}
+
+func VerifyUrl(r *http.Request, reg *Registration, token string) (*url.URL, error) {
+
+	redirectUrl, err := RedirectUrl(r, reg)
+	if err != nil {
+		return nil, err
+	}
+
+	uri := new(url.URL)
+	uri.Path = reg.Path + VerifyChallengeUrlSuffix
+
+	data := RequestDataFromContext(r.Context())
+	values := uri.Query()
+	values.Set(QueryArgRequestId, data.Id.String())
+	values.Set(QueryArgRedirect, redirectUrl.String())
+	values.Set(QueryArgToken, token)
+	values.Set(QueryArgChallenge, reg.Name)
+	uri.RawQuery = values.Encode()
+
+	return uri, nil
+}
+
+func RedirectUrl(r *http.Request, reg *Registration) (*url.URL, error) {
+	uri, err := url.ParseRequestURI(r.URL.String())
+	if err != nil {
+		return nil, err
+	}
+
+	data := RequestDataFromContext(r.Context())
+	values := uri.Query()
+	values.Set(QueryArgRequestId, data.Id.String())
+	if ref := r.Referer(); ref != "" {
+		values.Set(QueryArgReferer, r.Referer())
+	}
+	values.Set(QueryArgChallenge, reg.Name)
+	uri.RawQuery = values.Encode()
+
+	return uri, nil
+}
+
+func VerifyHandlerChallengeResponseFunc(state StateInterface, data *RequestData, w http.ResponseWriter, r *http.Request, verifyResult VerifyResult, err error, redirect string) {
+	if err != nil {
+		// Happy Eyeballs! auto retry
+		if errors.Is(err, ErrMismatchedTokenHappyEyeballs) {
+			reqUri := *r.URL
+			q := reqUri.Query()
+
+			ref := q.Get(QueryArgReferer)
+			// delete query parameters that were set by go-away
+			for k := range q {
+				if strings.HasPrefix(k, QueryArgPrefix) {
+					q.Del(k)
+				}
+			}
+			if ref != "" {
+				q.Set(QueryArgReferer, ref)
+			}
+			reqUri.RawQuery = q.Encode()
+
+			http.Redirect(w, r, reqUri.String(), http.StatusTemporaryRedirect)
+			return
+		}
+		state.ErrorPage(w, r, http.StatusBadRequest, err, redirect)
+		return
+	} else if !verifyResult.Ok() {
+		state.ErrorPage(w, r, http.StatusForbidden, fmt.Errorf("access denied: failed challenge"), redirect)
+		return
+	}
+	http.Redirect(w, r, redirect, http.StatusTemporaryRedirect)
+}
+
+func VerifyHandlerFunc(state StateInterface, reg *Registration, verify VerifyFunc, responseFunc func(state StateInterface, data *RequestData, w http.ResponseWriter, r *http.Request, verifyResult VerifyResult, err error, redirect string)) http.HandlerFunc {
+	if verify == nil {
+		verify = reg.Verify
+	}
+	if responseFunc == nil {
+		responseFunc = VerifyHandlerChallengeResponseFunc
+	}
+	return func(w http.ResponseWriter, r *http.Request) {
+		data := RequestDataFromContext(r.Context())
+		requestId, redirect, token, err := GetVerifyInformation(r, reg)
+		if err != nil {
+			state.ChallengeFailed(r, reg, err, "", nil)
+			responseFunc(state, data, w, r, VerifyResultFail, fmt.Errorf("internal error: %w", err), "")
+			return
+		}
+		data.Id = requestId
+
+		err = func() (err error) {
+			expiration := data.Expiration(reg.Duration)
+			key := GetChallengeKeyForRequest(state, reg, expiration, r)
+
+			verifyResult, err := verify(key, []byte(token), r)
+			if err != nil {
+				return err
+			} else if !verifyResult.Ok() {
+				utils.ClearCookie(data.CookiePrefix+reg.Name, w, r)
+				state.ChallengeFailed(r, reg, nil, redirect, nil)
+				responseFunc(state, data, w, r, verifyResult, nil, redirect)
+				return nil
+			}
+
+			challengeToken, err := reg.IssueChallengeToken(state.PrivateKey(), key, []byte(token), expiration, true)
+			if err != nil {
+				utils.ClearCookie(data.CookiePrefix+reg.Name, w, r)
+			} else {
+				utils.SetCookie(data.CookiePrefix+reg.Name, challengeToken, expiration, w, r)
+			}
+			data.ChallengeVerify[reg.id] = verifyResult
+			state.ChallengePassed(r, reg, redirect, nil)
+
+			responseFunc(state, data, w, r, verifyResult, nil, redirect)
+			return nil
+		}()
+		if err != nil {
+			utils.ClearCookie(data.CookiePrefix+reg.Name, w, r)
+			state.ChallengeFailed(r, reg, err, redirect, nil)
+			responseFunc(state, data, w, r, VerifyResultFail, fmt.Errorf("access denied: error in challenge %s: %w", reg.Name, err), redirect)
+			return
+		}
+	}
+}

lib/challenge/http/http.go

@@ -0,0 +1,160 @@
+package http
+
+import (
+	"crypto/sha256"
+	"crypto/subtle"
+	"errors"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"io"
+	"net/http"
+	"slices"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes[Key] = FillRegistration
+}
+
+const Key = "http"
+
+type Parameters struct {
+	VerifyProbability float64 `yaml:"verify-probability"`
+
+	HttpMethod string `yaml:"http-method"`
+	HttpCode   int    `yaml:"http-code"`
+	HttpCookie string `yaml:"http-cookie"`
+	Url        string `yaml:"http-url"`
+}
+
+var DefaultParameters = Parameters{
+	VerifyProbability: 0.20,
+	HttpMethod:        http.MethodGet,
+	HttpCode:          http.StatusOK,
+}
+
+func FillRegistration(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	params := DefaultParameters
+
+	if parameters != nil {
+		ymlData, err := parameters.MarshalYAML()
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(ymlData, &params)
+		if err != nil {
+			return err
+		}
+	}
+
+	if params.Url == "" {
+		return errors.New("empty url")
+	}
+
+	reg.Class = challenge.ClassTransparent
+
+	bindAuthValue := func(key challenge.Key, r *http.Request) ([]byte, error) {
+		var cookieValue string
+		if cookie, err := r.Cookie(params.HttpCookie); err != nil || cookie == nil {
+			// skip check if we don't have cookie or it's expired
+			return nil, http.ErrNoCookie
+		} else {
+			cookieValue = cookie.Value
+		}
+
+		// bind hash of cookie contents
+		sum := sha256.New()
+		sum.Write([]byte(cookieValue))
+		sum.Write([]byte{0})
+		sum.Write(key[:])
+		return sum.Sum(nil), nil
+	}
+
+	if params.VerifyProbability <= 0 {
+		//20% default
+		params.VerifyProbability = 0.20
+	} else if params.VerifyProbability > 1.0 {
+		params.VerifyProbability = 1.0
+	}
+	reg.VerifyProbability = params.VerifyProbability
+
+	if params.HttpCookie != "" {
+		// re-verify the cookie value
+		// TODO: configure to verify with backend
+		reg.Verify = func(key challenge.Key, token []byte, r *http.Request) (challenge.VerifyResult, error) {
+			sum, err := bindAuthValue(key, r)
+			if err != nil {
+				return challenge.VerifyResultFail, err
+			}
+			if subtle.ConstantTimeCompare(sum, token) == 1 {
+				return challenge.VerifyResultOK, nil
+			}
+			return challenge.VerifyResultFail, errors.New("invalid cookie value")
+		}
+	}
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+		var sum []byte
+		if params.HttpCookie != "" {
+			if c, err := r.Cookie(params.HttpCookie); err != nil || c == nil {
+				// skip check if we don't have cookie or it's expired
+				return challenge.VerifyResultSkip
+			} else {
+				sum, err = bindAuthValue(key, r)
+				if err != nil {
+					return challenge.VerifyResultFail
+				}
+			}
+		}
+
+		request, err := http.NewRequest(params.HttpMethod, params.Url, nil)
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+
+		var excludeHeaders = []string{"Host", "Content-Length"}
+		for k, v := range r.Header {
+			if slices.Contains(excludeHeaders, k) {
+				// skip these parameters
+				continue
+			}
+			request.Header[k] = v
+		}
+		// set id
+		request.Header.Set("X-Away-Id", challenge.RequestDataFromContext(r.Context()).Id.String())
+
+		// set request info in X headers
+		request.Header.Set("X-Away-Host", r.Host)
+		request.Header.Set("X-Away-Path", r.URL.Path)
+		request.Header.Set("X-Away-Query", r.URL.RawQuery)
+
+		response, err := state.Client().Do(request)
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+		defer response.Body.Close()
+		defer io.Copy(io.Discard, response.Body)
+
+		data := challenge.RequestDataFromContext(r.Context())
+
+		if response.StatusCode != params.HttpCode {
+			token, err := reg.IssueChallengeToken(state.PrivateKey(), key, sum, expiry, false)
+			if err != nil {
+				return challenge.VerifyResultFail
+			}
+			utils.SetCookie(data.CookiePrefix+reg.Name, token, expiry, w, r)
+			return challenge.VerifyResultNotOK
+		} else {
+			token, err := reg.IssueChallengeToken(state.PrivateKey(), key, sum, expiry, true)
+			if err != nil {
+				return challenge.VerifyResultFail
+			}
+			utils.SetCookie(data.CookiePrefix+reg.Name, token, expiry, w, r)
+			return challenge.VerifyResultOK
+		}
+	}
+
+	return nil
+}

lib/challenge/key.go

@@ -0,0 +1,81 @@
+package challenge
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"net/http"
+	"time"
+)
+
+type Key [KeySize]byte
+
+const KeySize = sha256.Size
+
+func (k *Key) Set(flags KeyFlags) {
+	(*k)[0] |= uint8(flags)
+}
+func (k *Key) Get(flags KeyFlags) KeyFlags {
+	return KeyFlags((*k)[0] & uint8(flags))
+}
+func (k *Key) Unset(flags KeyFlags) {
+	(*k)[0] = (*k)[0] & ^(uint8(flags))
+}
+
+type KeyFlags uint8
+
+const (
+	KeyFlagIsIPv4 = KeyFlags(1 << iota)
+)
+
+func KeyFromString(s string) (Key, error) {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		return Key{}, err
+	}
+	if len(b) != KeySize {
+		return Key{}, errors.New("invalid challenge key")
+	}
+	return Key(b), nil
+}
+
+func GetChallengeKeyForRequest(state StateInterface, reg *Registration, until time.Time, r *http.Request) Key {
+	data := RequestDataFromContext(r.Context())
+
+	hasher := sha256.New()
+	hasher.Write([]byte("challenge\x00"))
+	hasher.Write([]byte(reg.Name))
+	hasher.Write([]byte{0})
+	keyAddr := data.NetworkPrefix().As16()
+	hasher.Write(keyAddr[:])
+	hasher.Write([]byte{0})
+
+	// specific headers
+	for _, k := range []string{
+		"Accept-Language",
+		// General browser information
+		"User-Agent",
+		// TODO: not sent in preload
+		//"Sec-Ch-Ua",
+		//"Sec-Ch-Ua-Platform",
+	} {
+		hasher.Write([]byte(r.Header.Get(k)))
+		hasher.Write([]byte{0})
+	}
+	hasher.Write([]byte{0})
+	_ = binary.Write(hasher, binary.LittleEndian, until.UTC().Unix())
+	hasher.Write([]byte{0})
+	hasher.Write(state.PublicKey())
+	hasher.Write([]byte{0})
+
+	sum := Key(hasher.Sum(nil))
+
+	sum[0] = 0
+
+	if data.RemoteAddress.Addr().Unmap().Is4() {
+		// Is IPv4, mark
+		sum.Set(KeyFlagIsIPv4)
+	}
+	return Key(sum)
+}

lib/challenge/preload-link/preload-link.go

@@ -0,0 +1,128 @@
+package preload_link
+
+import (
+	"context"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"net/http"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes[Key] = FillRegistration
+}
+
+const Key = "preload-link"
+
+type Parameters struct {
+	Deadline time.Duration `yaml:"preload-early-hint-deadline"`
+}
+
+var DefaultParameters = Parameters{
+	Deadline: time.Second * 3,
+}
+
+func FillRegistration(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	params := DefaultParameters
+
+	if parameters != nil {
+		ymlData, err := parameters.MarshalYAML()
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(ymlData, &params)
+		if err != nil {
+			return err
+		}
+	}
+
+	verifier, issuer := challenge.NewKeyVerifier()
+	reg.Verify = verifier
+
+	reg.Class = challenge.ClassTransparent
+
+	ob := challenge.NewAwaiter[string]()
+
+	reg.Object = ob
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+		// this only works on HTTP/2 and HTTP/3
+
+		if r.ProtoMajor < 2 {
+			// this can happen if we are an upgraded request from HTTP/1.1 to HTTP/2 in H2C
+			if _, ok := w.(http.Pusher); !ok {
+				return challenge.VerifyResultSkip
+			}
+		}
+
+		issuerKey := issuer(key)
+
+		uri, err := challenge.VerifyUrl(r, reg, issuerKey)
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+
+		// remove redirect args
+		values := uri.Query()
+		values.Del(challenge.QueryArgRedirect)
+		uri.RawQuery = values.Encode()
+
+		// Redirect URI must be absolute to work
+		uri.Scheme = utils.GetRequestScheme(r)
+		uri.Host = r.Host
+
+		w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"preload\"; as=\"style\"; fetchpriority=high", uri.String()))
+		defer func() {
+			// remove old header so it won't show on response!
+			w.Header().Del("Link")
+		}()
+		w.WriteHeader(http.StatusEarlyHints)
+
+		ctx, cancel := context.WithTimeout(r.Context(), params.Deadline)
+		defer cancel()
+		if result := ob.Await(issuerKey, ctx); result.Ok() {
+			// this should serve!
+			return challenge.VerifyResultOK
+		} else if result == challenge.VerifyResultNone {
+			// we hit timeout
+			return challenge.VerifyResultFail
+		} else {
+			return result
+		}
+	}
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("GET "+reg.Path+challenge.VerifyChallengeUrlSuffix, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/css; charset=utf-8")
+		w.Header().Set("Content-Length", "0")
+
+		data := challenge.RequestDataFromContext(r.Context())
+		key := challenge.GetChallengeKeyForRequest(state, reg, data.Expiration(reg.Duration), r)
+		issuerKey := issuer(key)
+
+		_, _, token, err := challenge.GetVerifyInformation(r, reg)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+		}
+
+		verifyResult, _ := verifier(key, []byte(token), r)
+		if !verifyResult.Ok() {
+			w.WriteHeader(http.StatusUnauthorized)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+
+		ob.Solve(issuerKey, verifyResult)
+		if !verifyResult.Ok() {
+			// also give data on other failure when mismatched
+			ob.Solve(token, verifyResult)
+		}
+	})
+	reg.Handler = mux
+
+	return nil
+}

lib/challenge/refresh/refresh.go

@@ -0,0 +1,67 @@
+package refresh
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"net/http"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes["refresh"] = FillRegistration
+}
+
+type Parameters struct {
+	Mode string `yaml:"refresh-via"`
+}
+
+var DefaultParameters = Parameters{
+	Mode: "header",
+}
+
+func FillRegistration(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	params := DefaultParameters
+
+	if parameters != nil {
+		ymlData, err := parameters.MarshalYAML()
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(ymlData, &params)
+		if err != nil {
+			return err
+		}
+	}
+
+	reg.Class = challenge.ClassBlocking
+
+	verifier, issuer := challenge.NewKeyVerifier()
+	reg.Verify = verifier
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+		uri, err := challenge.VerifyUrl(r, reg, issuer(key))
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+
+		if params.Mode == "meta" {
+			state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
+				"Meta": []map[string]string{
+					{
+						"http-equiv": "refresh",
+						"content":    "0; url=" + uri.String(),
+					},
+				},
+			})
+		} else {
+			// self redirect!
+			w.Header().Set("Refresh", "0; url="+uri.String())
+
+			state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, nil)
+		}
+		return challenge.VerifyResultNone
+	}
+
+	return nil
+}

lib/challenge/register.go

@@ -0,0 +1,251 @@
+package challenge
+
+import (
+	"bytes"
+	http_cel "codeberg.org/gone/http-cel"
+	"crypto/ed25519"
+	"errors"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
+	"github.com/goccy/go-yaml/ast"
+	"github.com/google/cel-go/cel"
+	"io"
+	"math/rand/v2"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+)
+
+type Register map[Id]*Registration
+
+func (r Register) Get(id Id) (*Registration, bool) {
+	c, ok := r[id]
+	return c, ok
+}
+
+func (r Register) GetByName(name string) (*Registration, Id, bool) {
+	for id, c := range r {
+		if c.Name == name {
+			return c, id, true
+		}
+	}
+
+	return nil, 0, false
+}
+
+var idCounter Id
+
+// DefaultDuration TODO: adjust
+const DefaultDuration = time.Hour * 24 * 7
+
+func (r Register) Create(state StateInterface, name string, pol policy.Challenge, replacer *strings.Replacer) (*Registration, Id, error) {
+	runtime, ok := Runtimes[pol.Runtime]
+	if !ok {
+		return nil, 0, fmt.Errorf("unknown challenge runtime %s", pol.Runtime)
+	}
+
+	reg := &Registration{
+		Name:     name,
+		Path:     path.Join(state.UrlPath(), "challenge", name),
+		Duration: pol.Duration,
+	}
+
+	if reg.Duration == 0 {
+		reg.Duration = DefaultDuration
+	}
+
+	// allow nesting
+	var conditions []string
+	for _, cond := range pol.Conditions {
+		if replacer != nil {
+			cond = replacer.Replace(cond)
+		}
+		conditions = append(conditions, cond)
+	}
+
+	if len(conditions) > 0 {
+		ast, err := http_cel.NewAst(state.ProgramEnv(), http_cel.OperatorOr, conditions...)
+		if err != nil {
+			return nil, 0, fmt.Errorf("error compiling conditions: %v", err)
+		}
+		reg.Condition, err = http_cel.ProgramAst(state.ProgramEnv(), ast)
+		if err != nil {
+			return nil, 0, fmt.Errorf("error compiling program: %v", err)
+		}
+	}
+
+	if _, oldId, ok := r.GetByName(reg.Name); ok {
+		reg.id = oldId
+	} else {
+		idCounter++
+		reg.id = idCounter
+	}
+
+	err := runtime(state, reg, pol.Parameters)
+	if err != nil {
+		return nil, 0, fmt.Errorf("error filling registration: %v", err)
+	}
+	r[reg.id] = reg
+	return reg, reg.id, nil
+}
+
+func (r Register) Add(c *Registration) Id {
+	if _, oldId, ok := r.GetByName(c.Name); ok {
+		c.id = oldId
+		r[oldId] = c
+		return oldId
+	} else {
+		idCounter++
+		c.id = idCounter
+		r[idCounter] = c
+		return idCounter
+	}
+}
+
+type Registration struct {
+	// id The assigned internal identifier
+	id Id
+
+	// Name The unique name for this challenge
+	Name string
+
+	// Class whether this challenge is transparent or otherwise
+	Class Class
+
+	// Condition A CEL condition which is passed the same environment as general rules.
+	// If nil, always true
+	// If non-nil, must return true for this challenge to be allowed to be executed
+	Condition cel.Program
+
+	// Path The url path that this challenge is hosted under for the Handler to be called.
+	Path string
+
+	// Duration How long this challenge will be valid when passed
+	Duration time.Duration
+
+	// Handler An HTTP handler for all requests coming on the Path
+	// This handler will need to handle MakeChallengeUrlSuffix and VerifyChallengeUrlSuffix as well if needed
+	// Recommended to use http.ServeMux
+	Handler http.Handler
+
+	// Verify Verify an issued token
+	Verify            VerifyFunc
+	VerifyProbability float64
+
+	// IssueChallenge Issues a challenge to a request.
+	// If Class is ClassTransparent and VerifyResult is !VerifyResult.Ok(), continue with other challenges
+	// TODO: have this return error as well
+	IssueChallenge func(w http.ResponseWriter, r *http.Request, key Key, expiry time.Time) VerifyResult
+
+	// Object used to handle state or similar
+	// Can be nil if no state is needed
+	// If non-nil must implement io.Closer even if there's nothing to do
+	Object io.Closer
+}
+
+type VerifyFunc func(key Key, token []byte, r *http.Request) (VerifyResult, error)
+
+type Token struct {
+	Name   string `json:"name"`
+	Key    []byte `json:"key"`
+	Result []byte `json:"result,omitempty"`
+	Ok     bool   `json:"ok"`
+
+	Expiry    jwt.NumericDate `json:"exp,omitempty"`
+	NotBefore jwt.NumericDate `json:"nbf,omitempty"`
+	IssuedAt  jwt.NumericDate `json:"iat,omitempty"`
+}
+
+func (reg Registration) Id() Id {
+	return reg.id
+}
+
+func (reg Registration) IssueChallengeToken(privateKey ed25519.PrivateKey, key Key, result []byte, until time.Time, ok bool) (token string, err error) {
+	signer, err := jose.NewSigner(jose.SigningKey{
+		Algorithm: jose.EdDSA,
+		Key:       privateKey,
+	}, nil)
+	if err != nil {
+		return "", err
+	}
+
+	token, err = jwt.Signed(signer).Claims(Token{
+		Name:      reg.Name,
+		Key:       key[:],
+		Result:    result,
+		Ok:        ok,
+		Expiry:    jwt.NumericDate(until.Unix()),
+		NotBefore: jwt.NumericDate(time.Now().UTC().AddDate(0, 0, -1).Unix()),
+		IssuedAt:  jwt.NumericDate(time.Now().UTC().Unix()),
+	}).Serialize()
+	if err != nil {
+		return "", err
+	}
+	return token, nil
+}
+
+var ErrVerifyKeyMismatch = errors.New("verify: key mismatch")
+var ErrVerifyVerifyMismatch = errors.New("verify: verification mismatch")
+var ErrTokenExpired = errors.New("token: expired")
+
+func (reg Registration) VerifyChallengeToken(publicKey ed25519.PublicKey, expectedKey Key, r *http.Request) (VerifyResult, VerifyState, error) {
+	cookie, err := r.Cookie(RequestDataFromContext(r.Context()).CookiePrefix + reg.Name)
+	if err != nil {
+		return VerifyResultNone, VerifyStateNone, err
+	}
+	if cookie == nil {
+		return VerifyResultNone, VerifyStateNone, http.ErrNoCookie
+	}
+
+	token, err := jwt.ParseSigned(cookie.Value, []jose.SignatureAlgorithm{jose.EdDSA})
+	if err != nil {
+		return VerifyResultFail, VerifyStateNone, err
+	}
+
+	var i Token
+	err = token.Claims(publicKey, &i)
+	if err != nil {
+		return VerifyResultFail, VerifyStateNone, err
+	}
+
+	if i.Name != reg.Name {
+		return VerifyResultFail, VerifyStateNone, errors.New("token invalid name")
+	}
+	if i.Expiry.Time().Compare(time.Now()) < 0 {
+		return VerifyResultFail, VerifyStateNone, ErrTokenExpired
+	}
+	if i.NotBefore.Time().Compare(time.Now()) > 0 {
+		return VerifyResultFail, VerifyStateNone, errors.New("token not valid yet")
+	}
+
+	if bytes.Compare(expectedKey[:], i.Key) != 0 {
+		return VerifyResultFail, VerifyStateNone, ErrVerifyKeyMismatch
+	}
+
+	if reg.Verify != nil {
+		if rand.Float64() < reg.VerifyProbability {
+			// random spot check
+			if ok, err := reg.Verify(expectedKey, i.Result, r); err != nil {
+				return VerifyResultFail, VerifyStateFull, err
+			} else if ok == VerifyResultNotOK {
+				return VerifyResultNotOK, VerifyStateFull, nil
+			} else if !ok.Ok() {
+				return ok, VerifyStateFull, ErrVerifyVerifyMismatch
+			} else {
+				return ok, VerifyStateFull, nil
+			}
+		}
+	}
+
+	if !i.Ok {
+		return VerifyResultNotOK, VerifyStateBrief, nil
+	}
+	return VerifyResultOK, VerifyStateBrief, nil
+}
+
+type FillRegistration func(state StateInterface, reg *Registration, parameters ast.Node) error
+
+var Runtimes = make(map[string]FillRegistration)

lib/challenge/resource-load/resource-load.go

@@ -0,0 +1,55 @@
+package resource_load
+
+import (
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"github.com/goccy/go-yaml/ast"
+	"html/template"
+	"net/http"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes["resource-load"] = FillRegistrationHeader
+}
+
+func FillRegistrationHeader(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	reg.Class = challenge.ClassBlocking
+
+	verifier, issuer := challenge.NewKeyVerifier()
+	reg.Verify = verifier
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+		uri, err := challenge.VerifyUrl(r, reg, issuer(key))
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
+
+		// self redirect!
+		//TODO: adjust deadline
+		w.Header().Set("Refresh", "2; url="+r.URL.String())
+
+		state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
+			"HeaderTags": []template.HTML{
+				template.HTML(fmt.Sprintf("<link href=\"%s\" rel=\"stylesheet\" crossorigin=\"use-credentials\">", uri.String())),
+			},
+		})
+		return challenge.VerifyResultNone
+	}
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("GET "+reg.Path+challenge.VerifyChallengeUrlSuffix, challenge.VerifyHandlerFunc(state, reg, nil, func(state challenge.StateInterface, data *challenge.RequestData, w http.ResponseWriter, r *http.Request, verifyResult challenge.VerifyResult, err error, redirect string) {
+		//TODO: add other types inside css that need to be loaded!
+		w.Header().Set("Content-Type", "text/css; charset=utf-8")
+		w.Header().Set("Content-Length", "0")
+		if !verifyResult.Ok() {
+			w.WriteHeader(http.StatusForbidden)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+	}))
+	reg.Handler = mux
+
+	return nil
+}

lib/challenge/script.go

@@ -0,0 +1,42 @@
+package challenge
+
+import (
+	_ "embed"
+	"encoding/json"
+	"git.gammaspectra.live/git/go-away/utils"
+	"net/http"
+	"text/template"
+)
+
+//go:embed script.mjs
+var scriptData []byte
+
+var scriptTemplate = template.Must(template.New("script.mjs").Parse(string(scriptData)))
+
+func ServeChallengeScript(w http.ResponseWriter, r *http.Request, reg *Registration, params any, script string) {
+	data := RequestDataFromContext(r.Context())
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+	w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
+
+	paramData, err := json.Marshal(params)
+	if err != nil {
+		//TODO: log
+		panic(err)
+	}
+
+	w.WriteHeader(http.StatusOK)
+
+	err = scriptTemplate.Execute(w, map[string]any{
+		"Id":              data.Id.String(),
+		"Path":            reg.Path,
+		"Parameters":      paramData,
+		"Random":          utils.CacheBust(),
+		"Challenge":       reg.Name,
+		"ChallengeScript": script,
+		"Strings":         data.State.Options().Strings,
+	})
+	if err != nil {
+		//TODO: log
+		panic(err)
+	}
+}

lib/challenge/script.mjs

@@ -14,9 +14,8 @@ const u = (url = "", params = {}) => {
 (async () => {
     const status = document.getElementById('status');
     const title = document.getElementById('title');
-    const spinner = document.getElementById('spinner');
 
-    status.innerText = 'Starting challenge {{ .Challenge }}...';
+    status.innerText = '{{ .Strings.Get "status_starting_challenge" }} {{ .Challenge }}...';
 
     try {
         const info = await setup({
@@ -25,15 +24,13 @@ const u = (url = "", params = {}) => {
         });
 
         if (info != "") {
-            status.innerText = 'Calculating... ' + info
+            status.innerText = '{{ .Strings.Get "status_calculating" }} ' + info
         } else {
-            status.innerText = 'Calculating...';
+            status.innerText = '{{ .Strings.Get "status_calculating" }}';
         }
     } catch (err) {
-        title.innerHTML = "Oh no!";
-        status.innerHTML = `Failed to initialize: ${err.message}`;
-        spinner.innerHTML = "";
-        spinner.style.display = "none";
+        title.innerHTML = '{{ .Strings.Get "title_error" }}';
+        status.innerHTML = `{{ .Strings.Get "status_error" }} ${err.message}`;
         return
     }
 
@@ -44,26 +41,25 @@ const u = (url = "", params = {}) => {
         const t1 = Date.now();
         console.log({ result, info });
 
-        title.innerHTML = "Challenge success!";
+        title.innerHTML = '{{ .Strings.Get "status_challenge_success" }}';
         if (info != "") {
-            status.innerHTML = `Done! Took ${t1 - t0}ms, ${info}`;
+            status.innerHTML = `{{ .Strings.Get "status_challenge_done_took" }} ${t1 - t0}ms, ${info}`;
         } else {
-            status.innerHTML = `Done! Took ${t1 - t0}ms`;
+            status.innerHTML = `{{ .Strings.Get "status_challenge_done_took" }} ${t1 - t0}ms`;
         }
 
         setTimeout(() => {
             const redir = window.location.href;
             window.location.href = u("{{ .Path }}/verify-challenge", {
-                result: result,
-                redirect: redir,
-                requestId: "{{ .Id }}",
-                elapsedTime: t1 - t0,
+                __goaway_token: result,
+                __goaway_challenge: "{{ .Challenge }}",
+                __goaway_redirect: redir,
+                __goaway_id: "{{ .Id }}",
+                __goaway_elapsedTime: t1 - t0,
             });
         }, 500);
     } catch (err) {
-        title.innerHTML = "Oh no!";
-        status.innerHTML = `Failed to challenge: ${err.message}`;
-        spinner.innerHTML = "";
-        spinner.style.display = "none";
+        title.innerHTML = '{{ .Strings.Get "title_error" }}';
+        status.innerHTML = `{{ .Strings.Get "status_error" }} ${err.message}`;
     }
 })();

lib/challenge/state.go

@@ -1,176 +0,0 @@
-package challenge
-
-import (
-	"bytes"
-	"crypto/ed25519"
-	"errors"
-	"git.gammaspectra.live/git/go-away/utils"
-	"github.com/go-jose/go-jose/v4"
-	"github.com/go-jose/go-jose/v4/jwt"
-	"github.com/google/cel-go/cel"
-	"math/rand/v2"
-	"net/http"
-	"time"
-)
-
-type Result int
-
-const (
-	// ResultStop Stop testing other challenges and return
-	ResultStop = Result(iota)
-	// ResultContinue Test next
-	ResultContinue
-	// ResultPass  passed, return and proxy
-	ResultPass
-)
-
-type Id int
-
-type Challenge struct {
-	Id      Id
-	Program cel.Program
-	Name    string
-	Path    string
-
-	Verify            func(key []byte, result string, r *http.Request) (bool, error)
-	VerifyProbability float64
-
-	ServeStatic http.Handler
-
-	ServeChallenge func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) Result
-
-	ServeScript     http.Handler
-	ServeScriptPath string
-
-	ServeMakeChallenge   http.Handler
-	ServeVerifyChallenge http.Handler
-}
-
-type Token struct {
-	Name   string `json:"name"`
-	Key    []byte `json:"key"`
-	Result []byte `json:"result,omitempty"`
-
-	Expiry    *jwt.NumericDate `json:"exp,omitempty"`
-	NotBefore *jwt.NumericDate `json:"nbf,omitempty"`
-	IssuedAt  *jwt.NumericDate `json:"iat,omitempty"`
-}
-
-func (c Challenge) IssueChallengeToken(privateKey ed25519.PrivateKey, key, result []byte, until time.Time) (token string, err error) {
-	signer, err := jose.NewSigner(jose.SigningKey{
-		Algorithm: jose.EdDSA,
-		Key:       privateKey,
-	}, nil)
-	if err != nil {
-		return "", err
-	}
-
-	expiry := jwt.NumericDate(until.Unix())
-	notBefore := jwt.NumericDate(time.Now().UTC().AddDate(0, 0, -1).Unix())
-	issuedAt := jwt.NumericDate(time.Now().UTC().Unix())
-
-	token, err = jwt.Signed(signer).Claims(Token{
-		Name:      c.Name,
-		Key:       key,
-		Result:    result,
-		Expiry:    &expiry,
-		NotBefore: &notBefore,
-		IssuedAt:  &issuedAt,
-	}).Serialize()
-	if err != nil {
-		return "", err
-	}
-	return token, nil
-}
-
-type VerifyResult int
-
-const (
-	VerifyResultNONE = VerifyResult(iota)
-	VerifyResultFAIL
-	VerifyResultSKIP
-
-	// VerifyResultPASS Client just passed this challenge
-	VerifyResultPASS
-	VerifyResultOK
-	VerifyResultBRIEF
-	VerifyResultFULL
-)
-
-func (r VerifyResult) Ok() bool {
-	return r >= VerifyResultPASS
-}
-
-func (r VerifyResult) String() string {
-	switch r {
-	case VerifyResultNONE:
-		return "NONE"
-	case VerifyResultFAIL:
-		return "FAIL"
-	case VerifyResultSKIP:
-		return "SKIP"
-	case VerifyResultPASS:
-		return "PASS"
-	case VerifyResultOK:
-		return "OK"
-	case VerifyResultBRIEF:
-		return "BRIEF"
-	case VerifyResultFULL:
-		return "FULL"
-	default:
-		panic("unsupported")
-	}
-}
-
-var ErrVerifyKeyMismatch = errors.New("verify: key mismatch")
-var ErrVerifyVerifyMismatch = errors.New("verify: verification mismatch")
-
-func (c Challenge) VerifyChallengeToken(publicKey ed25519.PublicKey, expectedKey []byte, r *http.Request) (VerifyResult, error) {
-	cookie, err := r.Cookie(utils.CookiePrefix + c.Name)
-	if err != nil {
-		return VerifyResultNONE, err
-	}
-	if cookie == nil {
-		return VerifyResultNONE, http.ErrNoCookie
-	}
-
-	token, err := jwt.ParseSigned(cookie.Value, []jose.SignatureAlgorithm{jose.EdDSA})
-	if err != nil {
-		return VerifyResultFAIL, err
-	}
-
-	var i Token
-	err = token.Claims(publicKey, &i)
-	if err != nil {
-		return VerifyResultFAIL, err
-	}
-
-	if i.Name != c.Name {
-		return VerifyResultFAIL, errors.New("token invalid name")
-	}
-	if i.Expiry == nil && i.Expiry.Time().Compare(time.Now()) < 0 {
-		return VerifyResultFAIL, errors.New("token expired")
-	}
-	if i.NotBefore == nil && i.NotBefore.Time().Compare(time.Now()) > 0 {
-		return VerifyResultFAIL, errors.New("token not valid yet")
-	}
-
-	if bytes.Compare(expectedKey, i.Key) != 0 {
-		return VerifyResultFAIL, ErrVerifyKeyMismatch
-	}
-
-	if c.Verify != nil {
-		if rand.Float64() < c.VerifyProbability {
-			// random spot check
-			if ok, err := c.Verify(expectedKey, string(i.Result), r); err != nil {
-				return VerifyResultFAIL, err
-			} else if !ok {
-				return VerifyResultFAIL, ErrVerifyVerifyMismatch
-			}
-			return VerifyResultFULL, nil
-		} else {
-			return VerifyResultBRIEF, nil
-		}
-	}
-	return VerifyResultOK, nil
-}

lib/challenge/types.go

@@ -0,0 +1,120 @@
+package challenge
+
+import (
+	"crypto/ed25519"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/lib/settings"
+	"github.com/google/cel-go/cel"
+	"log/slog"
+	"net/http"
+)
+
+type Id int64
+
+type Class uint8
+
+const (
+	// ClassTransparent Transparent challenges work inline in the execution process.
+	// These can pass or continue, so more challenges or requests can ve served afterward.
+	ClassTransparent = Class(iota)
+
+	// ClassBlocking Blocking challenges must serve a different response to challenge the requester.
+	// These can pass or stop, for example, due to serving a challenge
+	ClassBlocking
+)
+
+type VerifyState uint8
+
+const (
+	VerifyStateNone = VerifyState(iota)
+	// VerifyStatePass Challenge was just passed on this request
+	VerifyStatePass
+	// VerifyStateBrief Challenge token was verified but didn't check the challenge
+	VerifyStateBrief
+	// VerifyStateFull Challenge token was verified and challenge verification was done
+	VerifyStateFull
+)
+
+func (r VerifyState) String() string {
+	switch r {
+	case VerifyStatePass:
+		return "PASS"
+	case VerifyStateBrief:
+		return "BRIEF"
+	case VerifyStateFull:
+		return "FULL"
+	default:
+		panic("unsupported")
+	}
+}
+
+type VerifyResult uint8
+
+const (
+	// VerifyResultNone A negative pass result, without a token
+	VerifyResultNone = VerifyResult(iota)
+	// VerifyResultFail A negative pass result, with an invalid token
+	VerifyResultFail
+	// VerifyResultSkip Challenge was skipped due to precondition
+	VerifyResultSkip
+	// VerifyResultNotOK A negative pass result, with a valid token
+	VerifyResultNotOK
+
+	// VerifyResultOK A positive pass result, with a valid token
+	VerifyResultOK
+)
+
+func (r VerifyResult) Ok() bool {
+	return r >= VerifyResultOK
+}
+
+func (r VerifyResult) String() string {
+	switch r {
+	case VerifyResultNone:
+		return "None"
+	case VerifyResultFail:
+		return "Fail"
+	case VerifyResultSkip:
+		return "Skip"
+	case VerifyResultNotOK:
+		return "NotOK"
+	case VerifyResultOK:
+		return "OK"
+	default:
+		panic("unsupported")
+	}
+}
+
+type StateInterface interface {
+	ProgramEnv() *cel.Env
+
+	Client() *http.Client
+	PrivateKey() ed25519.PrivateKey
+	PublicKey() ed25519.PublicKey
+
+	UrlPath() string
+
+	ChallengeFailed(r *http.Request, reg *Registration, err error, redirect string, logger *slog.Logger)
+	ChallengePassed(r *http.Request, reg *Registration, redirect string, logger *slog.Logger)
+	ChallengeIssued(r *http.Request, reg *Registration, redirect string, logger *slog.Logger)
+	ChallengeChecked(r *http.Request, reg *Registration, redirect string, logger *slog.Logger)
+
+	RuleHit(r *http.Request, name string, logger *slog.Logger)
+	RuleMiss(r *http.Request, name string, logger *slog.Logger)
+	ActionHit(r *http.Request, name policy.RuleAction, logger *slog.Logger)
+
+	Logger(r *http.Request) *slog.Logger
+
+	ChallengePage(w http.ResponseWriter, r *http.Request, status int, reg *Registration, params map[string]any)
+	ErrorPage(w http.ResponseWriter, r *http.Request, status int, err error, redirect string)
+
+	GetChallenge(id Id) (*Registration, bool)
+	GetChallengeByName(name string) (*Registration, bool)
+	GetChallenges() Register
+
+	Settings() policy.StateSettings
+
+	Options() settings.Settings
+
+	GetBackend(host string) http.Handler
+}

lib/challenge/wasm/interface/interface.go

@@ -111,6 +111,7 @@ type VerifyChallengeInput struct {
 
 type VerifyChallengeOutput uint64
 
+// TODO: expand allowed values
 const (
 	VerifyChallengeOutputOK = VerifyChallengeOutput(iota)
 	VerifyChallengeOutputFailed

lib/challenge/wasm/registration.go

@@ -0,0 +1,186 @@
+package wasm
+
+import (
+	"codeberg.org/meta/gzipped/v2"
+	"context"
+	"errors"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/embed"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	_interface "git.gammaspectra.live/git/go-away/lib/challenge/wasm/interface"
+	"git.gammaspectra.live/git/go-away/utils"
+	"git.gammaspectra.live/git/go-away/utils/inline"
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"github.com/tetratelabs/wazero/api"
+	"html/template"
+	"io"
+	"io/fs"
+	"net/http"
+	"path"
+	"time"
+)
+
+func init() {
+	challenge.Runtimes["js"] = FillJavaScriptRegistration
+}
+
+type Parameters struct {
+	Path string `yaml:"path"`
+	// Loader path to js/mjs file to use as challenge issuer
+	Loader string `yaml:"js-loader"`
+
+	// Runtime path to WASM wasip1 runtime
+	Runtime string `yaml:"wasm-runtime"`
+
+	Settings map[string]string `yaml:"wasm-runtime-settings"`
+
+	NativeCompiler bool `yaml:"wasm-native-compiler"`
+
+	VerifyProbability float64 `yaml:"verify-probability"`
+}
+
+var DefaultParameters = Parameters{
+	VerifyProbability: 0.1,
+	NativeCompiler:    true,
+}
+
+func FillJavaScriptRegistration(state challenge.StateInterface, reg *challenge.Registration, parameters ast.Node) error {
+	params := DefaultParameters
+
+	if parameters != nil {
+		ymlData, err := parameters.MarshalYAML()
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(ymlData, &params)
+		if err != nil {
+			return err
+		}
+	}
+
+	reg.Class = challenge.ClassBlocking
+
+	mux := http.NewServeMux()
+
+	if params.Path == "" {
+		params.Path = reg.Name
+	}
+
+	assetsFs, err := embed.GetFallbackFS(embed.ChallengeFs, params.Path)
+	if err != nil {
+		return err
+	}
+
+	if params.VerifyProbability <= 0 {
+		//10% default
+		params.VerifyProbability = 0.1
+	} else if params.VerifyProbability > 1.0 {
+		params.VerifyProbability = 1.0
+	}
+
+	reg.VerifyProbability = params.VerifyProbability
+
+	ob := NewRunner(params.NativeCompiler)
+	reg.Object = ob
+
+	wasmData, err := assetsFs.ReadFile(path.Join("runtime", params.Runtime))
+	if err != nil {
+		return fmt.Errorf("could not load runtime: %w", err)
+	}
+
+	err = ob.Compile("runtime", wasmData)
+	if err != nil {
+		return fmt.Errorf("compiling runtime: %w", err)
+	}
+
+	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
+		state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
+			"EndTags": []template.HTML{
+				template.HTML(fmt.Sprintf("<script async type=\"module\" src=\"%s?cacheBust=%s\"></script>", reg.Path+"/script.mjs", utils.CacheBust())),
+			},
+		})
+		return challenge.VerifyResultNone
+	}
+
+	reg.Verify = func(key challenge.Key, token []byte, r *http.Request) (challenge.VerifyResult, error) {
+		var ok bool
+		err = ob.Instantiate("runtime", func(ctx context.Context, mod api.Module) (err error) {
+			in := _interface.VerifyChallengeInput{
+				Key:        key[:],
+				Parameters: params.Settings,
+				Result:     token,
+			}
+
+			out, err := VerifyChallengeCall(ctx, mod, in)
+			if err != nil {
+				return err
+			}
+
+			if out == _interface.VerifyChallengeOutputError {
+				return errors.New("error checking challenge")
+			}
+			ok = out == _interface.VerifyChallengeOutputOK
+			return nil
+		})
+		if err != nil {
+			return challenge.VerifyResultFail, err
+		}
+		if ok {
+			return challenge.VerifyResultOK, nil
+		}
+		return challenge.VerifyResultFail, nil
+	}
+
+	// serve assets if existent
+	if staticFs, err := fs.Sub(assetsFs, "static"); err != nil {
+		return fmt.Errorf("no static assets: %w", err)
+	} else {
+		mux.Handle("GET "+reg.Path+"/static/", http.StripPrefix(reg.Path+"/static/", gzipped.FileServer(gzipped.FS(staticFs))))
+	}
+
+	mux.HandleFunc(reg.Path+challenge.MakeChallengeUrlSuffix, func(w http.ResponseWriter, r *http.Request) {
+		data := challenge.RequestDataFromContext(r.Context())
+		err := ob.Instantiate("runtime", func(ctx context.Context, mod api.Module) (err error) {
+			key := challenge.GetChallengeKeyForRequest(state, reg, data.Expiration(reg.Duration), r)
+
+			in := _interface.MakeChallengeInput{
+				Key:        key[:],
+				Parameters: params.Settings,
+				Headers:    inline.MIMEHeader(r.Header),
+			}
+			in.Data, err = io.ReadAll(r.Body)
+			if err != nil {
+				return err
+			}
+
+			out, err := MakeChallengeCall(ctx, mod, in)
+			if err != nil {
+				return err
+			}
+
+			// set output headers
+			for k, v := range out.Headers {
+				w.Header()[k] = v
+			}
+			w.Header().Set("Content-Length", fmt.Sprintf("%d", len(out.Data)))
+			w.WriteHeader(out.Code)
+			_, _ = w.Write(out.Data)
+			return nil
+		})
+		if err != nil {
+			state.ErrorPage(w, r, http.StatusInternalServerError, err, "")
+			return
+		}
+	})
+
+	mux.HandleFunc(reg.Path+challenge.VerifyChallengeUrlSuffix, challenge.VerifyHandlerFunc(state, reg, nil, nil))
+
+	mux.HandleFunc("GET "+reg.Path+"/script.mjs", func(w http.ResponseWriter, r *http.Request) {
+		challenge.ServeChallengeScript(w, r, reg, params.Settings, path.Join(reg.Path, "static", params.Loader))
+	})
+
+	reg.Handler = mux
+
+	return nil
+}

lib/challenge/wasm/runner.go

@@ -94,11 +94,13 @@ func (r *Runner) Compile(key string, binary []byte) error {
 	return nil
 }
 
-func (r *Runner) Close() {
+func (r *Runner) Close() error {
 	for _, module := range r.modules {
-		module.Close(r.context)
+		if err := module.Close(r.context); err != nil {
+			return err
+		}
 	}
-	r.runtime.Close(r.context)
+	return r.runtime.Close(r.context)
 }
 
 var ErrModuleNotFound = errors.New("module not found")

lib/condition/condition.go

@@ -1,53 +0,0 @@
-package condition
-
-import (
-	"fmt"
-	"github.com/google/cel-go/cel"
-	"strings"
-)
-
-type Condition struct {
-	Expression *cel.Ast
-}
-
-const (
-	OperatorOr  = "||"
-	OperatorAnd = "&&"
-)
-
-func FromStrings(env *cel.Env, operator string, conditions ...string) (*cel.Ast, error) {
-	var asts []*cel.Ast
-	for _, c := range conditions {
-		ast, issues := env.Compile(c)
-		if issues != nil && issues.Err() != nil {
-			return nil, fmt.Errorf("condition %s: %s", issues.Err(), c)
-		}
-		asts = append(asts, ast)
-	}
-
-	return Merge(env, operator, asts...)
-}
-
-func Merge(env *cel.Env, operator string, conditions ...*cel.Ast) (*cel.Ast, error) {
-	if len(conditions) == 0 {
-		return nil, nil
-	} else if len(conditions) == 1 {
-		return conditions[0], nil
-	}
-	var asts []string
-	for _, c := range conditions {
-		ast, err := cel.AstToString(c)
-		if err != nil {
-			return nil, err
-		}
-		asts = append(asts, "("+ast+")")
-	}
-
-	condition := strings.Join(asts, " "+operator+" ")
-	ast, issues := env.Compile(condition)
-	if issues != nil && issues.Err() != nil {
-		return nil, issues.Err()
-	}
-
-	return ast, nil
-}

lib/conditions.go

@@ -1,28 +1,73 @@
 package lib
 
 import (
+	http_cel "codeberg.org/gone/http-cel"
 	"fmt"
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
+	"log/slog"
 	"net"
 )
 
 func (state *State) initConditions() (err error) {
-	state.RulesEnv, err = cel.NewEnv(
-		cel.DefaultUTCTimeZone(true),
-		cel.Variable("remoteAddress", cel.BytesType),
-		cel.Variable("host", cel.StringType),
-		cel.Variable("method", cel.StringType),
-		cel.Variable("userAgent", cel.StringType),
-		cel.Variable("path", cel.StringType),
-		cel.Variable("query", cel.MapType(cel.StringType, cel.StringType)),
-		// http.Header
-		cel.Variable("headers", cel.MapType(cel.StringType, cel.StringType)),
-		//TODO: dynamic type?
+	state.programEnv, err = http_cel.NewEnvironment(
+
+		cel.Variable("fp", cel.MapType(cel.StringType, cel.StringType)),
+		cel.Function("inDNSBL",
+			cel.Overload("inDNSBL_ip",
+				[]*cel.Type{cel.AnyType},
+				cel.BoolType,
+				cel.UnaryBinding(func(val ref.Val) ref.Val {
+					slog.Error("inDNSBL function has been deprecated, replace with dnsbl challenge")
+					return types.Bool(false)
+				}),
+			),
+		),
+
+		cel.Function("network",
+			cel.MemberOverload("netIP_network_string",
+				[]*cel.Type{cel.BytesType, cel.StringType},
+				cel.BoolType,
+				cel.BinaryBinding(func(lhs ref.Val, rhs ref.Val) ref.Val {
+					var ip net.IP
+					switch v := lhs.Value().(type) {
+					case []byte:
+						ip = v
+					case net.IP:
+						ip = v
+					}
+
+					if ip == nil {
+						panic(fmt.Errorf("invalid ip %v", lhs.Value()))
+					}
+
+					val, ok := rhs.Value().(string)
+					if !ok {
+						panic(fmt.Errorf("invalid network value %v", rhs.Value()))
+					}
+
+					network, ok := state.networks[val]
+					if !ok {
+						_, ipNet, err := net.ParseCIDR(val)
+						if err != nil {
+							panic("network not found")
+						}
+						return types.Bool(ipNet.Contains(ip))
+					} else {
+						ok, err := network.Contains(ip)
+						if err != nil {
+							panic(err)
+						}
+						return types.Bool(ok)
+					}
+				}),
+			),
+		),
+
 		cel.Function("inNetwork",
 			cel.Overload("inNetwork_string_ip",
-				[]*cel.Type{cel.StringType, cel.AnyType},
+				[]*cel.Type{cel.StringType, cel.BytesType},
 				cel.BoolType,
 				cel.BinaryBinding(func(lhs ref.Val, rhs ref.Val) ref.Val {
 					var ip net.IP
@@ -31,8 +76,6 @@ func (state *State) initConditions() (err error) {
 						ip = v
 					case net.IP:
 						ip = v
-					case string:
-						ip = net.ParseIP(v)
 					}
 
 					if ip == nil {
@@ -43,8 +86,9 @@ func (state *State) initConditions() (err error) {
 					if !ok {
 						panic(fmt.Errorf("invalid value %v", lhs.Value()))
 					}
+					slog.Debug(fmt.Sprintf("inNetwork function has been deprecated and will be removed in a future release, use remoteAddress.network(\"%s\") instead", val))
 
-					network, ok := state.Networks[val]
+					network, ok := state.networks[val]
 					if !ok {
 						_, ipNet, err := net.ParseCIDR(val)
 						if err != nil {

lib/http.go

@@ -1,402 +1,233 @@
 package lib
 
 import (
-	"bytes"
 	"codeberg.org/meta/gzipped/v2"
-	"context"
-	"crypto/rand"
-	"encoding/base64"
-	"encoding/hex"
-	"errors"
 	"fmt"
 	"git.gammaspectra.live/git/go-away/embed"
+	"git.gammaspectra.live/git/go-away/lib/action"
 	"git.gammaspectra.live/git/go-away/lib/challenge"
 	"git.gammaspectra.live/git/go-away/lib/policy"
 	"git.gammaspectra.live/git/go-away/utils"
-	"github.com/google/cel-go/common/types"
-	"html/template"
-	"io"
+	"golang.org/x/net/html"
 	"log/slog"
-	"maps"
 	"net/http"
-	"path"
-	"path/filepath"
-	"strconv"
+	"slices"
 	"strings"
 	"time"
 )
 
-var templates map[string]*template.Template
-
-var cacheBust string
-
-// DefaultValidity TODO: adjust
-const DefaultValidity = time.Hour * 24 * 7
-
-func init() {
-
-	buf := make([]byte, 16)
-	_, _ = rand.Read(buf)
-	cacheBust = base64.RawURLEncoding.EncodeToString(buf)
-
-	templates = make(map[string]*template.Template)
-
-	dir, err := embed.TemplatesFs.ReadDir("templates")
-	if err != nil {
-		panic(err)
-	}
-	for _, e := range dir {
-		if e.IsDir() {
-			continue
-		}
-		data, err := embed.TemplatesFs.ReadFile(filepath.Join("templates", e.Name()))
-		if err != nil {
-			panic(err)
-		}
-		err = initTemplate(e.Name(), string(data))
-		if err != nil {
-			panic(err)
-		}
-	}
-}
-
-func initTemplate(name, data string) error {
-	tpl := template.New(name)
-	_, err := tpl.Parse(data)
-	if err != nil {
-		return err
-	}
-	templates[name] = tpl
-	return nil
-}
-
-func (state *State) challengePage(w http.ResponseWriter, id string, status int, challenge string, params map[string]any) error {
-	input := make(map[string]any)
-	input["Id"] = id
-	input["Random"] = cacheBust
-	input["Challenge"] = challenge
-	input["Path"] = state.UrlPath
-	input["Theme"] = state.Settings.ChallengeTemplateTheme
-
-	maps.Copy(input, params)
-
-	if _, ok := input["Title"]; !ok {
-		input["Title"] = "Checking you are not a bot"
-	}
-
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-
-	buf := bytes.NewBuffer(make([]byte, 0, 8192))
-
-	err := templates["challenge-"+state.Settings.ChallengeTemplate+".gohtml"].Execute(buf, input)
-	if err != nil {
-		_ = state.errorPage(w, id, http.StatusInternalServerError, err, "")
-	} else {
-		w.WriteHeader(status)
-		_, _ = w.Write(buf.Bytes())
-	}
-	return nil
-}
-
-func (state *State) errorPage(w http.ResponseWriter, id string, status int, err error, redirect string) error {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-
-	buf := bytes.NewBuffer(make([]byte, 0, 8192))
-
-	err2 := templates["challenge-"+state.Settings.ChallengeTemplate+".gohtml"].Execute(buf, map[string]any{
-		"Id":          id,
-		"Random":      cacheBust,
-		"Error":       err.Error(),
-		"Path":        state.UrlPath,
-		"Theme":       state.Settings.ChallengeTemplateTheme,
-		"Title":       "Oh no! " + http.StatusText(status),
-		"HideSpinner": true,
-		"Challenge":   "",
-		"Redirect":    redirect,
-	})
-	if err2 != nil {
-		panic(err2)
-	} else {
-		w.WriteHeader(status)
-		_, _ = w.Write(buf.Bytes())
-	}
-	return nil
-}
-
-func (state *State) addTiming(w http.ResponseWriter, name, desc string, duration time.Duration) {
-	if state.Settings.Debug {
-		w.Header().Add("Server-Timing", fmt.Sprintf("%s;desc=%s;dur=%d", name, strconv.Quote(desc), duration.Milliseconds()))
-	}
-}
-
-func GetLoggerForRequest(r *http.Request, clientHeader string) *slog.Logger {
-	return slog.With(
-		"request_id", r.Header.Get("X-Away-Id"),
-		"remote_address", getRequestAddress(r, clientHeader),
+func GetLoggerForRequest(r *http.Request) *slog.Logger {
+	data := challenge.RequestDataFromContext(r.Context())
+	args := []any{
+		"request_id", data.Id.String(),
+		"remote_address", data.RemoteAddress.Addr().String(),
 		"user_agent", r.UserAgent(),
 		"host", r.Host,
 		"path", r.URL.Path,
 		"query", r.URL.RawQuery,
-	)
+	}
+
+	if fp := utils.GetTLSFingerprint(r); fp != nil {
+		if ja3n := fp.JA3N(); ja3n != nil {
+			args = append(args, "ja3n", ja3n.String())
+		}
+		if ja4 := fp.JA4(); ja4 != nil {
+			args = append(args, "ja4", ja4.String())
+		}
+	}
+	return slog.With(args...)
 }
 
-func (state *State) logger(r *http.Request) *slog.Logger {
-	return GetLoggerForRequest(r, state.Settings.ClientIpHeader)
+func (state *State) fetchMetaTags(host string, backend http.Handler, r *http.Request) []html.Node {
+	uri := *r.URL
+	q := uri.Query()
+	for k := range q {
+		if strings.HasPrefix(k, challenge.QueryArgPrefix) {
+			q.Del(k)
+		}
+	}
+	uri.RawQuery = q.Encode()
+
+	key := fmt.Sprintf("%s:%s", host, uri.String())
+
+	if v, ok := state.tagCache.Get(key); ok {
+		return v
+	}
+
+	result := utils.FetchTags(backend, &uri, "meta")
+	if result == nil {
+		return nil
+	}
+
+	entries := make([]html.Node, 0, len(result))
+
+	safeAttributes := []string{"name", "property", "content"}
+	for _, n := range result {
+		if n.Namespace != "" {
+			continue
+		}
+
+		var name string
+		for _, attr := range n.Attr {
+			if attr.Namespace != "" {
+				continue
+			}
+			if attr.Key == "name" {
+				name = attr.Val
+				break
+			}
+			if attr.Key == "property" && name == "" {
+				name = attr.Val
+			}
+		}
+
+		// prevent unwanted keys like CSRF and other internal entries to pass through as much as possible
+
+		var keep bool
+		if strings.HasPrefix("og:", name) || strings.HasPrefix("fb:", name) || strings.HasPrefix("twitter:", name) || strings.HasPrefix("profile:", name) {
+			// social / OpenGraph tags
+			keep = true
+		} else if name == "vcs" || strings.HasPrefix("vcs:", name) {
+			// source tags
+			keep = true
+		} else if name == "forge" || strings.HasPrefix("forge:", name) {
+			// forge tags
+			keep = true
+		} else {
+			switch name {
+			// standard content tags
+			case "application-name", "author", "description", "keywords", "robots", "thumbnail":
+				keep = true
+			case "go-import", "go-source":
+				// golang tags
+				keep = true
+			case "apple-itunes-app":
+			}
+		}
+
+		// prevent other arbitrary arguments
+		if keep {
+			newNode := html.Node{
+				Type: html.ElementNode,
+				Data: n.Data,
+			}
+			for _, attr := range n.Attr {
+				if attr.Namespace != "" {
+					continue
+				}
+				if slices.Contains(safeAttributes, attr.Key) {
+					newNode.Attr = append(newNode.Attr, attr)
+				}
+			}
+			if len(newNode.Attr) == 0 {
+				continue
+			}
+			entries = append(entries, newNode)
+		}
+	}
+
+	state.tagCache.Set(key, entries, time.Hour*6)
+	return entries
 }
 
 func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 	host := r.Host
 
-	data := RequestDataFromContext(r.Context())
+	data := challenge.RequestDataFromContext(r.Context())
 
-	backend, ok := state.Settings.Backends[host]
-	if !ok {
+	backend := state.GetBackend(host)
+	if backend == nil {
 		http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
 		return
 	}
 
-	lg := state.logger(r)
-
-	start := time.Now()
-
-	state.addTiming(w, "rule-env", "Setup the rule environment", time.Since(start))
-
-	var (
-		ruleEvalDuration time.Duration
-	)
-
-	serve := func() {
-		state.addTiming(w, "rule-eval", "Evaluate access rules", ruleEvalDuration)
-		backend.ServeHTTP(w, r)
-	}
-
-	fail := func(code int, err error) {
-		state.addTiming(w, "rule-eval", "Evaluate access rules", ruleEvalDuration)
-		_ = state.errorPage(w, r.Header.Get("X-Away-Id"), code, err, "")
-	}
-
-	setAwayState := func(rule RuleState) {
-		r.Header.Set("X-Away-Rule", rule.Name)
-		r.Header.Set("X-Away-Hash", rule.Hash)
-		r.Header.Set("X-Away-Action", string(rule.Action))
-		data.Headers(state, r.Header)
-	}
-
-	for _, rule := range state.Rules {
-		// skip rules that have host match
-		if rule.Host != nil && *rule.Host != host {
-			continue
+	getBackend := func() http.Handler {
+		if opt := data.GetOpt(challenge.RequestOptBackendHost, ""); opt != "" && opt != host {
+			b := state.GetBackend(host)
+			if b == nil {
+				http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+				// return empty backend
+				return http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
+			}
+			return b
 		}
-		start = time.Now()
-		out, _, err := rule.Program.Eval(data.ProgramEnv)
-		ruleEvalDuration += time.Since(start)
+		return backend
+	}
 
-		if err != nil {
-			fail(http.StatusInternalServerError, err)
-			lg.Error(err.Error(), "rule", rule.Name, "rule_hash", rule.Hash)
-			panic(err)
-			return
-		} else if out != nil && out.Type() == types.BoolType {
-			if out.Equal(types.True) == types.True {
-				switch rule.Action {
-				default:
-					panic(fmt.Errorf("unknown action %s", rule.Action))
-				case policy.RuleActionPASS:
-					lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash)
-					setAwayState(rule)
-					serve()
-					return
-				case policy.RuleActionCHALLENGE, policy.RuleActionCHECK:
-					for _, challengeId := range rule.Challenges {
-						if result := data.Challenges[challengeId]; !result.Ok() {
-							continue
-						} else {
-							if rule.Action == policy.RuleActionCHECK {
-								goto nextRule
-							}
+	lg := state.Logger(r)
 
-							// we passed the challenge!
-							lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", state.Challenges[challengeId].Name)
-							setAwayState(rule)
-							serve()
-							return
-						}
-					}
+	cleanupRequest := func(r *http.Request, fromChallenge bool) {
+		if fromChallenge {
+			r.Header.Del("Referer")
+		}
+		q := r.URL.Query()
 
-					// none matched, issue first challenge in priority
-					for _, challengeId := range rule.Challenges {
-						result := data.Challenges[challengeId]
-						if result.Ok() || result == challenge.VerifyResultSKIP {
-							// skip already ok'd challenges for some reason, and also skip skipped challenges
-							continue
-						}
-						c := state.Challenges[challengeId]
-						if c.ServeChallenge != nil {
-							result := c.ServeChallenge(w, r, state.GetChallengeKeyForRequest(c.Name, data.Expires, r), data.Expires)
-							switch result {
-							case challenge.ResultStop:
-								lg.Info("request challenged", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
-								return
-							case challenge.ResultContinue:
-								continue
-							case challenge.ResultPass:
-								if rule.Action == policy.RuleActionCHECK {
-									goto nextRule
-								}
-								state.logger(r).Warn("challenge passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
+		if ref := q.Get(challenge.QueryArgReferer); ref != "" {
+			r.Header.Set("Referer", ref)
+		}
 
-								// set pass if caller didn't set one
-								if !data.Challenges[c.Id].Ok() {
-									data.Challenges[c.Id] = challenge.VerifyResultPASS
-								}
-
-								// we pass the challenge early!
-								lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
-								setAwayState(rule)
-								serve()
-								return
-							}
-						} else {
-							panic("challenge not found")
-						}
-					}
-				case policy.RuleActionDENY:
-					lg.Info("request denied", "rule", rule.Name, "rule_hash", rule.Hash)
-					//TODO: config error code
-					fail(http.StatusForbidden, fmt.Errorf("access denied: denied by administrative rule %s/%s", r.Header.Get("X-Away-Id"), rule.Hash))
-					return
-				case policy.RuleActionBLOCK:
-					lg.Info("request blocked", "rule", rule.Name, "rule_hash", rule.Hash)
-					//TODO: config error code
-					//TODO: configure block
-					fail(http.StatusForbidden, fmt.Errorf("access denied: blocked by administrative rule %s/%s", r.Header.Get("X-Away-Id"), rule.Hash))
-					return
-				case policy.RuleActionPOISON:
-					lg.Info("request poisoned", "rule", rule.Name, "rule_hash", rule.Hash)
-
-					mime := "text/html"
-					switch path.Ext(r.URL.Path) {
-					case ".css":
-					case ".json", ".js", ".mjs":
-
-					}
-
-					encodings := strings.Split(r.Header.Get("Accept-Encoding"), ",")
-					for i, encoding := range encodings {
-						encodings[i] = strings.TrimSpace(strings.ToLower(encoding))
-					}
-
-					reader, encoding := state.getPoison(mime, encodings)
-					if reader == nil {
-						mime = "application/octet-stream"
-						reader, encoding = state.getPoison(mime, encodings)
-					}
-
-					if reader != nil {
-						defer reader.Close()
-					}
-
-					w.Header().Set("Cache-Control", "max-age=0, private, must-revalidate, no-transform")
-					w.Header().Set("Vary", "Accept-Encoding")
-					w.Header().Set("Content-Type", mime)
-					w.Header().Set("X-Content-Type-Options", "nosniff")
-					if encoding != "" {
-						w.Header().Set("Content-Encoding", encoding)
-					}
-					w.WriteHeader(http.StatusOK)
-					if flusher, ok := w.(http.Flusher); ok {
-						// trigger chunked encoding
-						flusher.Flush()
-					}
-					if r != nil {
-						_, _ = io.Copy(w, reader)
-					}
-					return
-				}
+		// delete query parameters that were set by go-away
+		for k := range q {
+			if strings.HasPrefix(k, challenge.QueryArgPrefix) {
+				q.Del(k)
 			}
 		}
+		r.URL.RawQuery = q.Encode()
 
-	nextRule:
+		data.RequestHeaders(r.Header)
+
+		// delete cookies set by go-away to prevent user tracking that way
+		cookies := r.Cookies()
+		r.Header.Del("Cookie")
+		for _, c := range cookies {
+			if !strings.HasPrefix(c.Name, utils.CookiePrefix) {
+				r.AddCookie(c)
+			}
+		}
 	}
 
-	serve()
-	return
+	for _, rule := range state.rules {
+		next, err := rule.Evaluate(lg, w, r, func() http.Handler {
+			cleanupRequest(r, true)
+			return getBackend()
+		})
+		if err != nil {
+			state.ErrorPage(w, r, http.StatusInternalServerError, err, "")
+			panic(err)
+			return
+		}
+
+		if !next {
+			return
+		}
+	}
+
+	state.RuleHit(r, "DEFAULT", lg)
+	data.State.ActionHit(r, policy.RuleActionPASS, lg)
+
+	// default pass
+	_, _ = action.Pass{}.Handle(lg, w, r, func() http.Handler {
+		r.Header.Set("X-Away-Rule", "DEFAULT")
+		r.Header.Set("X-Away-Action", "PASS")
+
+		cleanupRequest(r, false)
+		return getBackend()
+	})
 }
 
 func (state *State) setupRoutes() error {
 
 	state.Mux.HandleFunc("/", state.handleRequest)
 
-	state.Mux.Handle("GET "+state.UrlPath+"/assets/", http.StripPrefix(state.UrlPath, gzipped.FileServer(gzipped.FS(embed.AssetsFs))))
+	state.Mux.Handle("GET "+state.urlPath+"/assets/", http.StripPrefix(state.UrlPath()+"/assets/", gzipped.FileServer(gzipped.FS(embed.AssetsFs))))
 
-	for _, c := range state.Challenges {
-		if c.ServeStatic != nil {
-			state.Mux.Handle("GET "+c.Path+"/static/", c.ServeStatic)
-		}
+	for _, reg := range state.challenges {
 
-		if c.ServeScript != nil {
-			state.Mux.Handle("GET "+c.ServeScriptPath, c.ServeScript)
-		}
-
-		if c.ServeMakeChallenge != nil {
-			state.Mux.Handle(fmt.Sprintf("POST %s/make-challenge", c.Path), c.ServeMakeChallenge)
-		}
-
-		if c.ServeVerifyChallenge != nil {
-			state.Mux.Handle(fmt.Sprintf("GET %s/verify-challenge", c.Path), c.ServeVerifyChallenge)
-		} else if c.Verify != nil {
-			state.Mux.HandleFunc(fmt.Sprintf("GET %s/verify-challenge", c.Path), func(w http.ResponseWriter, r *http.Request) {
-				redirect, err := utils.EnsureNoOpenRedirect(r.FormValue("redirect"))
-				if redirect == "" {
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, "")
-					return
-				}
-
-				err = func() (err error) {
-
-					data := RequestDataFromContext(r.Context())
-
-					key := state.GetChallengeKeyForRequest(c.Name, data.Expires, r)
-					result := r.FormValue("result")
-
-					requestId, err := hex.DecodeString(r.FormValue("requestId"))
-					if err == nil {
-						// override
-						r.Header.Set("X-Away-Id", hex.EncodeToString(requestId))
-					}
-
-					start := time.Now()
-					ok, err := c.Verify(key, result, r)
-					state.addTiming(w, "challenge-verify", "Verify client challenge", time.Since(start))
-
-					if err != nil {
-						state.logger(r).Error(fmt.Errorf("challenge error: %w", err).Error(), "challenge", c.Name, "redirect", redirect)
-						return err
-					} else if !ok {
-						state.logger(r).Warn("challenge failed", "challenge", c.Name, "redirect", redirect)
-						utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-						_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", c.Name), redirect)
-						return nil
-					}
-					state.logger(r).Info("challenge passed", "challenge", c.Name, "redirect", redirect)
-
-					token, err := c.IssueChallengeToken(state.privateKey, key, []byte(result), data.Expires)
-					if err != nil {
-						utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-					} else {
-						utils.SetCookie(utils.CookiePrefix+c.Name, token, data.Expires, w)
-					}
-					data.Challenges[c.Id] = challenge.VerifyResultPASS
-
-					http.Redirect(w, r, redirect, http.StatusTemporaryRedirect)
-					return nil
-				}()
-				if err != nil {
-					utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, redirect)
-					return
-				}
-			})
+		if reg.Handler != nil {
+			state.Mux.Handle(reg.Path+"/", reg.Handler)
+		} else if reg.Verify != nil {
+			// default verify
+			state.Mux.HandleFunc(reg.Path+challenge.VerifyChallengeUrlSuffix, challenge.VerifyHandlerFunc(state, reg, nil, nil))
 		}
 	}
 
@@ -404,94 +235,13 @@ func (state *State) setupRoutes() error {
 }
 
 func (state *State) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	var data RequestData
-	// generate random id, todo: is this fast?
-	_, _ = rand.Read(data.Id[:])
-	data.Challenges = make(map[challenge.Id]challenge.VerifyResult, len(state.Challenges))
-	data.Expires = time.Now().UTC().Add(DefaultValidity).Round(DefaultValidity)
-	data.ProgramEnv = map[string]any{
-		"host":          r.Host,
-		"method":        r.Method,
-		"remoteAddress": getRequestAddress(r, state.Settings.ClientIpHeader),
-		"userAgent":     r.UserAgent(),
-		"path":          r.URL.Path,
-		"query": func() map[string]string {
-			result := make(map[string]string)
-			for k, v := range r.URL.Query() {
-				result[k] = strings.Join(v, ",")
-			}
-			return result
-		}(),
-		"headers": func() map[string]string {
-			result := make(map[string]string)
-			for k, v := range r.Header {
-				result[k] = strings.Join(v, ",")
-			}
-			return result
-		}(),
+	r, data := challenge.CreateRequestData(r, state)
+
+	data.EvaluateChallenges(w, r)
+
+	if state.Settings().MainName != "" {
+		w.Header().Add("Via", fmt.Sprintf("%s %s@%s", r.Proto, state.Settings().MainName, state.Settings().MainVersion))
 	}
 
-	for _, c := range state.Challenges {
-		key := state.GetChallengeKeyForRequest(c.Name, data.Expires, r)
-		result, err := c.VerifyChallengeToken(state.publicKey, key, r)
-		if err != nil && !errors.Is(err, http.ErrNoCookie) {
-			// clear invalid cookie
-			utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-		}
-
-		// prevent the challenge if not solved
-		if !result.Ok() && c.Program != nil {
-			out, _, err := c.Program.Eval(data.ProgramEnv)
-			// verify eligibility
-			if err != nil {
-				state.logger(r).Error(err.Error(), "challenge", c.Name)
-			} else if out != nil && out.Type() == types.BoolType {
-				if out.Equal(types.True) != types.True {
-					// skip challenge match!
-					result = challenge.VerifyResultSKIP
-					continue
-				}
-			}
-		}
-		data.Challenges[c.Id] = result
-	}
-
-	r.Header.Set("X-Away-Id", hex.EncodeToString(data.Id[:]))
-	w.Header().Set("X-Away-Id", hex.EncodeToString(data.Id[:]))
-
-	// send these to client so we consistently get the headers
-	//w.Header().Set("Accept-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
-	//w.Header().Set("Critical-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
-
-	r = r.WithContext(context.WithValue(r.Context(), "_goaway_data", &data))
-
 	state.Mux.ServeHTTP(w, r)
 }
-
-func RequestDataFromContext(ctx context.Context) *RequestData {
-	return ctx.Value("_goaway_data").(*RequestData)
-}
-
-type RequestData struct {
-	Id         [16]byte
-	ProgramEnv map[string]any
-	Expires    time.Time
-	Challenges map[challenge.Id]challenge.VerifyResult
-}
-
-func (d *RequestData) HasValidChallenge(id challenge.Id) bool {
-	return d.Challenges[id].Ok()
-}
-
-func (d *RequestData) Headers(state *State, headers http.Header) {
-	for id, result := range d.Challenges {
-		if result.Ok() {
-			c, ok := state.Challenges[id]
-			if !ok {
-				panic("challenge not found")
-			}
-
-			headers.Set(fmt.Sprintf("X-Away-Challenge-%s-Result", c.Name), result.String())
-		}
-	}
-}

lib/interface.go

@@ -0,0 +1,108 @@
+package lib
+
+import (
+	"crypto/ed25519"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/lib/settings"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/google/cel-go/cel"
+	"log/slog"
+	"net/http"
+)
+
+// Defines challenge.StateInterface
+
+var _ challenge.StateInterface
+
+func (state *State) ProgramEnv() *cel.Env {
+	return state.programEnv
+}
+
+func (state *State) Client() *http.Client {
+	return state.client
+}
+
+func (state *State) PrivateKey() ed25519.PrivateKey {
+	return state.privateKey
+}
+
+func (state *State) PublicKey() ed25519.PublicKey {
+	return state.publicKey
+}
+
+func (state *State) UrlPath() string {
+	return state.urlPath
+}
+
+func (state *State) ChallengeFailed(r *http.Request, reg *challenge.Registration, err error, redirect string, logger *slog.Logger) {
+	if logger == nil {
+		logger = state.Logger(r)
+	}
+	logger.Warn("challenge failed", "challenge", reg.Name, "err", err, "redirect", redirect)
+
+	metrics.Challenge(reg.Name, "fail")
+}
+
+func (state *State) ChallengePassed(r *http.Request, reg *challenge.Registration, redirect string, logger *slog.Logger) {
+	if logger == nil {
+		logger = state.Logger(r)
+	}
+	logger.Warn("challenge passed", "challenge", reg.Name, "redirect", redirect)
+
+	metrics.Challenge(reg.Name, "pass")
+}
+
+func (state *State) ChallengeIssued(r *http.Request, reg *challenge.Registration, redirect string, logger *slog.Logger) {
+	if logger == nil {
+		logger = state.Logger(r)
+	}
+	logger.Info("challenge issued", "challenge", reg.Name, "redirect", redirect)
+
+	metrics.Challenge(reg.Name, "issue")
+}
+
+func (state *State) ChallengeChecked(r *http.Request, reg *challenge.Registration, redirect string, logger *slog.Logger) {
+	metrics.Challenge(reg.Name, "check")
+}
+
+func (state *State) RuleHit(r *http.Request, name string, logger *slog.Logger) {
+	metrics.Rule(name, "hit")
+}
+
+func (state *State) RuleMiss(r *http.Request, name string, logger *slog.Logger) {
+	metrics.Rule(name, "miss")
+}
+
+func (state *State) ActionHit(r *http.Request, name policy.RuleAction, logger *slog.Logger) {
+	metrics.Action(name)
+}
+
+func (state *State) Logger(r *http.Request) *slog.Logger {
+	return GetLoggerForRequest(r)
+}
+
+func (state *State) GetChallenge(id challenge.Id) (*challenge.Registration, bool) {
+	reg, ok := state.challenges.Get(id)
+	return reg, ok
+}
+
+func (state *State) GetChallenges() challenge.Register {
+	return state.challenges
+}
+
+func (state *State) GetChallengeByName(name string) (*challenge.Registration, bool) {
+	reg, _, ok := state.challenges.GetByName(name)
+	return reg, ok
+}
+func (state *State) Settings() policy.StateSettings {
+	return state.settings
+}
+
+func (state *State) Options() settings.Settings {
+	return state.opt
+}
+
+func (state *State) GetBackend(host string) http.Handler {
+	return utils.SelectHTTPHandler(state.Settings().Backends, host)
+}

lib/metrics.go

@@ -0,0 +1,50 @@
+package lib
+
+import (
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+type stateMetrics struct {
+	rules      *prometheus.CounterVec
+	actions    *prometheus.CounterVec
+	challenges *prometheus.CounterVec
+}
+
+func newMetrics() *stateMetrics {
+	return &stateMetrics{
+		rules: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "go-away_rule_results",
+			Help: "The number of rule hits or misses",
+		}, []string{"rule", "result"}),
+		actions: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "go-away_action_results",
+			Help: "The number of each action issued",
+		}, []string{"action"}),
+		challenges: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "go-away_challenge_results",
+			Help: "The number of challenges issued, passed or explicitly failed",
+		}, []string{"challenge", "action"}),
+	}
+}
+
+func (metrics *stateMetrics) Rule(name, result string) {
+	metrics.rules.With(prometheus.Labels{"rule": name, "result": result}).Inc()
+}
+
+func (metrics *stateMetrics) Action(action policy.RuleAction) {
+	metrics.actions.With(prometheus.Labels{"action": string(action)}).Inc()
+}
+
+func (metrics *stateMetrics) Challenge(name, result string) {
+	metrics.challenges.With(prometheus.Labels{"challenge": name, "action": result}).Inc()
+}
+
+func (metrics *stateMetrics) Reset() {
+	metrics.rules.Reset()
+	metrics.actions.Reset()
+	metrics.challenges.Reset()
+}
+
+var metrics = newMetrics()

lib/poison.go

@@ -1,26 +0,0 @@
-package lib
-
-import (
-	"git.gammaspectra.live/git/go-away/embed"
-	"io"
-	"path"
-	"slices"
-	"strings"
-)
-
-var poisonEncodings = []string{"br", "zstd", "gzip"}
-
-func (state *State) getPoison(mime string, encodings []string) (r io.ReadCloser, encoding string) {
-	for _, encoding = range poisonEncodings {
-		if !slices.Contains(encodings, encoding) {
-			continue
-		}
-
-		p := path.Join("poison", strings.ReplaceAll(mime, "/", "_")+"."+encoding+".poison")
-		f, err := embed.PoisonFs.Open(p)
-		if err == nil {
-			return f, encoding
-		}
-	}
-	return nil, ""
-}

lib/policy/challenge.go

@@ -1,15 +1,15 @@
 package policy
 
+import (
+	"github.com/goccy/go-yaml/ast"
+	"time"
+)
+
 type Challenge struct {
 	Conditions []string `yaml:"conditions"`
-	Mode       string   `yaml:"mode"`
-	Asset      *string  `yaml:"asset,omitempty"`
-	Url        *string  `yaml:"url,omitempty"`
+	Runtime    string   `yaml:"runtime"`
 
-	Parameters map[string]string `json:"parameters,omitempty"`
-	Runtime    struct {
-		Mode        string  `yaml:"mode,omitempty"`
-		Asset       string  `yaml:"asset,omitempty"`
-		Probability float64 `yaml:"probability,omitempty"`
-	} `yaml:"runtime"`
+	Duration time.Duration `yaml:"duration"`
+
+	Parameters ast.Node `yaml:"parameters,omitempty"`
 }

lib/policy/network.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"git.gammaspectra.live/git/go-away/utils"
 	"github.com/itchyny/gojq"
 	"io"
 	"net"
@@ -13,16 +14,20 @@ import (
 )
 
 type Network struct {
+	// Fetches
 	Url  *string `yaml:"url,omitempty"`
 	File *string `yaml:"file,omitempty"`
+	ASN  *int    `yaml:"asn,omitempty"`
 
+	// Filtering
 	JqPath *string `yaml:"jq-path,omitempty"`
 	Regex  *string `yaml:"regex,omitempty"`
 
 	Prefixes []string `yaml:"prefixes,omitempty"`
 }
 
-func (n Network) FetchPrefixes(c *http.Client) (output []net.IPNet, err error) {
+func (n Network) FetchPrefixes(c *http.Client, whois *utils.RADb) (output []net.IPNet, err error) {
+
 	if len(n.Prefixes) > 0 {
 		for _, prefix := range n.Prefixes {
 			ipNet, err := parseCIDROrIP(prefix)
@@ -40,7 +45,7 @@ func (n Network) FetchPrefixes(c *http.Client) (output []net.IPNet, err error) {
 			return nil, err
 		}
 		defer response.Body.Close()
-		if response.StatusCode != 200 {
+		if response.StatusCode != http.StatusOK {
 			return nil, fmt.Errorf("unexpected status code: %d", response.StatusCode)
 		}
 		reader = response.Body
@@ -51,6 +56,12 @@ func (n Network) FetchPrefixes(c *http.Client) (output []net.IPNet, err error) {
 		}
 		defer file.Close()
 		reader = file
+	} else if n.ASN != nil {
+		result, err := whois.FetchASNets(*n.ASN)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch ASN %d: %v", *n.ASN, err)
+		}
+		return result, nil
 	} else {
 		if len(output) > 0 {
 			return output, nil
@@ -115,3 +126,30 @@ func (n Network) FetchPrefixes(c *http.Client) (output []net.IPNet, err error) {
 	}
 	return output, nil
 }
+
+func parseCIDROrIP(value string) (net.IPNet, error) {
+	_, ipNet, err := net.ParseCIDR(value)
+	if err != nil {
+		ip := net.ParseIP(value)
+		if ip == nil {
+			return net.IPNet{}, fmt.Errorf("failed to parse CIDR: %s", err)
+		}
+
+		if ip4 := ip.To4(); ip4 != nil {
+			return net.IPNet{
+				IP: ip4,
+				// single ip
+				Mask: net.CIDRMask(len(ip4)*8, len(ip4)*8),
+			}, nil
+		}
+		return net.IPNet{
+			IP: ip,
+			// single ip
+			Mask: net.CIDRMask(len(ip)*8, len(ip)*8),
+		}, nil
+	} else if ipNet != nil {
+		return *ipNet, nil
+	} else {
+		return net.IPNet{}, errors.New("invalid CIDR")
+	}
+}

lib/policy/policy.go

@@ -1,38 +1,13 @@
 package policy
 
 import (
-	"errors"
-	"fmt"
-	"net"
+	"bytes"
+	"github.com/goccy/go-yaml"
+	"io"
+	"os"
+	"path"
 )
 
-func parseCIDROrIP(value string) (net.IPNet, error) {
-	_, ipNet, err := net.ParseCIDR(value)
-	if err != nil {
-		ip := net.ParseIP(value)
-		if ip == nil {
-			return net.IPNet{}, fmt.Errorf("failed to parse CIDR: %s", err)
-		}
-
-		if ip4 := ip.To4(); ip4 != nil {
-			return net.IPNet{
-				IP: ip4,
-				// single ip
-				Mask: net.CIDRMask(len(ip4)*8, len(ip4)*8),
-			}, nil
-		}
-		return net.IPNet{
-			IP: ip,
-			// single ip
-			Mask: net.CIDRMask(len(ip)*8, len(ip)*8),
-		}, nil
-	} else if ipNet != nil {
-		return *ipNet, nil
-	} else {
-		return net.IPNet{}, errors.New("invalid CIDR")
-	}
-}
-
 type Policy struct {
 
 	// Networks map of networks and prefixes to be loaded
@@ -43,8 +18,81 @@ type Policy struct {
 	Challenges map[string]Challenge `yaml:"challenges"`
 
 	Rules []Rule `yaml:"rules"`
-
-	// Backends
-	// Deprecated
-	Backends map[string]string `json:"backends"`
+}
+
+func NewPolicy(r io.Reader, snippetsDirectories ...string) (*Policy, error) {
+	var p Policy
+	p.Networks = make(map[string][]Network)
+	p.Conditions = make(map[string][]string)
+	p.Challenges = make(map[string]Challenge)
+
+	if len(snippetsDirectories) == 0 {
+		err := yaml.NewDecoder(r).Decode(&p)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		var entries []string
+		for _, dir := range snippetsDirectories {
+			if dir == "" {
+				// skip nil directories
+				continue
+			}
+			dirFiles, err := os.ReadDir(dir)
+			if err != nil {
+				return nil, err
+			}
+			for _, file := range dirFiles {
+				if file.IsDir() {
+					continue
+				}
+				entries = append(entries, path.Join(dir, file.Name()))
+			}
+		}
+
+		err := yaml.NewDecoder(r, yaml.ReferenceFiles(entries...)).Decode(&p)
+		if err != nil {
+			return nil, err
+		}
+
+		// add specific entries from snippets
+		for _, entry := range entries {
+			var entryPolicy Policy
+			entryData, err := os.ReadFile(entry)
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.NewDecoder(bytes.NewReader(entryData), yaml.ReferenceFiles(entries...)).Decode(&entryPolicy)
+			if err != nil {
+				return nil, err
+			}
+
+			// add networks / conditions / challenges definitions if they don't exist already
+
+			for k, v := range entryPolicy.Networks {
+				// add network if policy entry does not exist
+				_, ok := p.Networks[k]
+				if !ok {
+					p.Networks[k] = v
+				}
+			}
+
+			for k, v := range entryPolicy.Conditions {
+				// add condition if policy entry does not exist
+				_, ok := p.Conditions[k]
+				if !ok {
+					p.Conditions[k] = v
+				}
+			}
+
+			for k, v := range entryPolicy.Challenges {
+				// add challenge if policy entry does not exist
+				_, ok := p.Challenges[k]
+				if !ok {
+					p.Challenges[k] = v
+				}
+			}
+		}
+	}
+	return &p, nil
 }

lib/policy/rule.go

@@ -1,22 +1,43 @@
 package policy
 
+import "github.com/goccy/go-yaml/ast"
+
 type RuleAction string
 
 const (
-	RuleActionPASS      RuleAction = "PASS"
-	RuleActionDENY      RuleAction = "DENY"
-	RuleActionBLOCK     RuleAction = "BLOCK"
+	// RuleActionNONE Does nothing. Useful for parent rules when children want to be specified
+	RuleActionNONE RuleAction = "NONE"
+	// RuleActionPASS Passes the connection immediately
+	RuleActionPASS RuleAction = "PASS"
+	// RuleActionDENY Denies the connection with a fancy page
+	RuleActionDENY RuleAction = "DENY"
+	// RuleActionBLOCK Denies the connection with a response code
+	RuleActionBLOCK RuleAction = "BLOCK"
+	// RuleActionCODE Returns a specified HTTP code
+	RuleActionCODE RuleAction = "CODE"
+
+	// RuleActionDROP Drops the connection without sending a reply
+	RuleActionDROP RuleAction = "DROP"
+
+	// RuleActionCHALLENGE Issues a challenge that when passed, passes the connection
 	RuleActionCHALLENGE RuleAction = "CHALLENGE"
-	RuleActionCHECK     RuleAction = "CHECK"
-	RuleActionPOISON    RuleAction = "POISON"
+	// RuleActionCHECK Issues a challenge that when passed, continues checking rules
+	RuleActionCHECK RuleAction = "CHECK"
+
+	// RuleActionPROXY Proxies request to a backend, with optional path replacements
+	RuleActionPROXY RuleAction = "PROXY"
+
+	// RuleActionCONTEXT Changes Request Context information or properties
+	RuleActionCONTEXT RuleAction = "CONTEXT"
 )
 
 type Rule struct {
 	Name       string   `yaml:"name"`
-	Host       *string  `yaml:"host"`
 	Conditions []string `yaml:"conditions"`
 
 	Action string `yaml:"action"`
 
-	Challenges []string `yaml:"challenges"`
+	Settings ast.Node `yaml:"settings"`
+
+	Children []Rule `yaml:"children"`
 }

lib/policy/state.go

@@ -0,0 +1,19 @@
+package policy
+
+import (
+	"git.gammaspectra.live/git/go-away/utils"
+	"net/http"
+)
+
+type StateSettings struct {
+	Cache           utils.Cache
+	Backends        map[string]http.Handler
+	PrivateKeySeed  []byte
+	MainName        string
+	MainVersion     string
+	BasePath        string
+	ClientIpHeader  string
+	BackendIpHeader string
+
+	ChallengeResponseCode int
+}

lib/rule.go

@@ -0,0 +1,150 @@
+package lib
+
+import (
+	http_cel "codeberg.org/gone/http-cel"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/action"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"log/slog"
+	"net/http"
+	"strings"
+)
+
+type RuleState struct {
+	Name string
+	Hash string
+
+	Condition cel.Program
+
+	Action  policy.RuleAction
+	Handler action.Handler
+
+	Children []RuleState
+}
+
+func NewRuleState(state challenge.StateInterface, r policy.Rule, replacer *strings.Replacer, parent *RuleState) (RuleState, error) {
+	fp := sha256.Sum256(state.PrivateKey())
+	hasher := sha256.New()
+	if parent != nil {
+		hasher.Write([]byte(parent.Name))
+		hasher.Write([]byte{0})
+		r.Name = fmt.Sprintf("%s/%s", parent.Name, r.Name)
+	}
+	hasher.Write([]byte(r.Name))
+	hasher.Write([]byte{0})
+	hasher.Write(fp[:])
+	sum := hasher.Sum(nil)
+
+	rule := RuleState{
+		Name:   r.Name,
+		Hash:   hex.EncodeToString(sum[:10]),
+		Action: policy.RuleAction(strings.ToUpper(r.Action)),
+	}
+
+	newHandler, ok := action.Register[rule.Action]
+	if !ok {
+		return RuleState{}, fmt.Errorf("unknown action %s", r.Action)
+	}
+
+	actionHandler, err := newHandler(state, rule.Name, rule.Hash, r.Settings)
+	if err != nil {
+		return RuleState{}, err
+	}
+	rule.Handler = actionHandler
+
+	if len(r.Conditions) > 0 {
+		// allow nesting
+		var conditions []string
+		for _, cond := range r.Conditions {
+			cond = replacer.Replace(cond)
+			conditions = append(conditions, cond)
+		}
+
+		ast, err := http_cel.NewAst(state.ProgramEnv(), http_cel.OperatorOr, conditions...)
+		if err != nil {
+			return RuleState{}, fmt.Errorf("error compiling conditions: %w", err)
+		}
+
+		program, err := http_cel.ProgramAst(state.ProgramEnv(), ast)
+		if err != nil {
+			return RuleState{}, fmt.Errorf("error compiling program: %w", err)
+		}
+		rule.Condition = program
+	}
+
+	if len(r.Children) > 0 {
+		for _, child := range r.Children {
+			childRule, err := NewRuleState(state, child, replacer, &rule)
+			if err != nil {
+				return RuleState{}, fmt.Errorf("child %s: %w", child.Name, err)
+			}
+			rule.Children = append(rule.Children, childRule)
+		}
+	}
+
+	return rule, nil
+}
+
+func (rule RuleState) Evaluate(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() http.Handler) (next bool, err error) {
+	data := challenge.RequestDataFromContext(r.Context())
+	var out ref.Val
+
+	lg := logger.With("rule", rule.Name, "rule_hash", rule.Hash, "action", string(rule.Action))
+	if rule.Condition != nil {
+		out, _, err = rule.Condition.Eval(data)
+	} else {
+		// default true
+		out = types.Bool(true)
+	}
+	if err != nil {
+		lg.Error(err.Error())
+		return false, fmt.Errorf("error: evaluating administrative rule %s/%s: %w", data.Id.String(), rule.Hash, err)
+	} else if out != nil && out.Type() == types.BoolType {
+		if out.Equal(types.True) == types.True {
+			data.State.RuleHit(r, rule.Name, logger)
+
+			data.State.ActionHit(r, rule.Action, logger)
+			next, err = rule.Handler.Handle(lg, w, r, func() http.Handler {
+				r.Header.Set("X-Away-Rule", rule.Name)
+				r.Header.Set("X-Away-Hash", rule.Hash)
+				r.Header.Set("X-Away-Action", string(rule.Action))
+
+				return done()
+			})
+			if err != nil {
+				lg.Error(err.Error())
+				return false, fmt.Errorf("error: executing administrative rule %s/%s: %w", data.Id.String(), rule.Hash, err)
+			}
+
+			if !next {
+				return next, nil
+			}
+
+			for _, child := range rule.Children {
+				next, err = child.Evaluate(logger, w, r, done)
+				if err != nil {
+					lg.Error(err.Error())
+					return false, fmt.Errorf("error: executing administrative rule %s/%s: %w", data.Id.String(), rule.Hash, err)
+				}
+
+				if !next {
+					return next, nil
+				}
+			}
+		} else {
+			data.State.RuleMiss(r, rule.Name, logger)
+		}
+	} else if out != nil {
+		err := fmt.Errorf("return type not Bool, got %s", out.Type().TypeName())
+		lg.Error(err.Error())
+		return false, fmt.Errorf("error: evaluating administrative rule %s/%s: %w", data.Id.String(), rule.Hash, err)
+	}
+
+	return true, nil
+}

lib/settings/backend.go

@@ -0,0 +1,106 @@
+package settings
+
+import (
+	"git.gammaspectra.live/git/go-away/utils"
+	"net/http"
+	"net/http/httputil"
+)
+
+type Backend struct {
+	// URL Target server backend path. Supports http/https/unix protocols.
+	URL string `yaml:"url"`
+
+	// Host Override the Host header and TLS SNI with this value if specified
+	Host string `yaml:"host"`
+
+	//ProxyProtocol uint8 `yaml:"proxy-protocol"`
+
+	// HTTP2Enabled Enable HTTP2 to backend
+	HTTP2Enabled bool `yaml:"http2-enabled"`
+
+	// TLSSkipVerify Disable TLS certificate verification, if any
+	TLSSkipVerify bool `yaml:"tls-skip-verify"`
+
+	// IpHeader HTTP header to set containing the IP header. Set - to forcefully ignore global defaults.
+	IpHeader string `yaml:"ip-header"`
+
+	// GoDNS Resolve URL using the Go DNS server
+	// Only relevant when running with CGO enabled
+	GoDNS bool `yaml:"go-dns"`
+}
+
+func (b Backend) Create() (*httputil.ReverseProxy, error) {
+	if b.IpHeader == "-" {
+		b.IpHeader = ""
+	}
+
+	proxy, err := utils.MakeReverseProxy(b.URL, b.GoDNS)
+	if err != nil {
+		return nil, err
+	}
+
+	transport := proxy.Transport.(*http.Transport)
+
+	if b.HTTP2Enabled {
+		transport.ForceAttemptHTTP2 = true
+	}
+
+	if b.TLSSkipVerify {
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	}
+
+	if b.Host != "" {
+		transport.TLSClientConfig.ServerName = b.Host
+	}
+
+	if b.IpHeader != "" || b.Host != "" {
+		director := proxy.Director
+		proxy.Director = func(req *http.Request) {
+			if b.IpHeader != "" {
+				if ip := utils.GetRemoteAddress(req.Context()); ip != nil {
+					req.Header.Set(b.IpHeader, ip.Addr().Unmap().String())
+				}
+			}
+			if b.Host != "" {
+				req.Host = b.Host
+			}
+			director(req)
+		}
+	}
+
+	/*if b.ProxyProtocol > 0 {
+		dialContext := transport.DialContext
+		if dialContext == nil {
+			dialContext = (&net.Dialer{}).DialContext
+		}
+		transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+			conn, err := dialContext(ctx, network, addr)
+			if err != nil {
+				return nil, err
+			}
+			addrPort := utils.GetRemoteAddress(ctx)
+			if addrPort == nil {
+				// pass as is
+				hdr := proxyproto.HeaderProxyFromAddrs(b.ProxyProtocol, conn.LocalAddr(), conn.RemoteAddr())
+				_, err = hdr.WriteTo(conn)
+				if err != nil {
+					conn.Close()
+					return nil, err
+				}
+			} else {
+				// set proper headers!
+				hdr := proxyproto.HeaderProxyFromAddrs(b.ProxyProtocol, net.TCPAddrFromAddrPort(*addrPort), conn.RemoteAddr())
+				_, err = hdr.WriteTo(conn)
+				if err != nil {
+					conn.Close()
+					return nil, err
+				}
+			}
+			return conn, nil
+		}
+	}*/
+
+	proxy.Transport = transport
+
+	return proxy, nil
+}

lib/settings/bind.go

@@ -0,0 +1,168 @@
+package settings
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/pires/go-proxyproto"
+	"golang.org/x/crypto/acme"
+	"golang.org/x/crypto/acme/autocert"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"sync/atomic"
+)
+
+type Bind struct {
+	Address    string `yaml:"address"`
+	Network    string `yaml:"network"`
+	SocketMode string `yaml:"socket-mode"`
+	Proxy      bool   `yaml:"proxy"`
+
+	Passthrough bool `yaml:"passthrough"`
+
+	// TLSAcmeAutoCert URL to ACME directory, or letsencrypt
+	TLSAcmeAutoCert string `yaml:"tls-acme-autocert"`
+
+	// TLSCertificate Alternate to TLSAcmeAutoCert
+	TLSCertificate string `yaml:"tls-certificate"`
+	// TLSPrivateKey Alternate to TLSAcmeAutoCert
+	TLSPrivateKey string `yaml:"tls-key"`
+}
+
+func (b *Bind) Listener() (net.Listener, string) {
+	return setupListener(b.Network, b.Address, b.SocketMode, b.Proxy)
+}
+
+func (b *Bind) Server(backends map[string]http.Handler, acmeCachePath string) (*http.Server, func(http.Handler), error) {
+
+	var tlsConfig *tls.Config
+
+	if b.TLSAcmeAutoCert != "" {
+		switch b.TLSAcmeAutoCert {
+		case "letsencrypt":
+			b.TLSAcmeAutoCert = acme.LetsEncryptURL
+		}
+
+		acmeManager := newACMEManager(b.TLSAcmeAutoCert, backends)
+		if acmeCachePath != "" {
+			err := os.MkdirAll(acmeCachePath, 0755)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to create acme cache directory: %w", err)
+			}
+			acmeManager.Cache = autocert.DirCache(acmeCachePath)
+		}
+		slog.Warn(
+			"acme-autocert enabled",
+			"directory", b.TLSAcmeAutoCert,
+		)
+		tlsConfig = acmeManager.TLSConfig()
+	} else if b.TLSCertificate != "" && b.TLSPrivateKey != "" {
+		tlsConfig = &tls.Config{}
+		var err error
+		tlsConfig.Certificates = make([]tls.Certificate, 1)
+		tlsConfig.Certificates[0], err = tls.LoadX509KeyPair(b.TLSCertificate, b.TLSPrivateKey)
+		if err != nil {
+			return nil, nil, err
+		}
+		slog.Warn(
+			"TLS enabled",
+			"certificate", b.TLSCertificate,
+		)
+	}
+
+	var serverHandler atomic.Pointer[http.Handler]
+	server := utils.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if handler := serverHandler.Load(); handler == nil {
+			http.Error(w, http.StatusText(http.StatusBadGateway), http.StatusBadGateway)
+		} else {
+			(*handler).ServeHTTP(w, r)
+		}
+	}), tlsConfig)
+
+	swap := func(handler http.Handler) {
+		serverHandler.Store(&handler)
+	}
+
+	if b.Passthrough {
+		// setup a passthrough handler temporarily
+		swap(http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backend := utils.SelectHTTPHandler(backends, r.Host)
+			if backend == nil {
+				http.Error(w, http.StatusText(http.StatusBadGateway), http.StatusBadGateway)
+			} else {
+				backend.ServeHTTP(w, r)
+			}
+		})))
+	}
+
+	return server, swap, nil
+
+}
+
+func setupListener(network, address, socketMode string, proxy bool) (net.Listener, string) {
+	if network == "proxy" {
+		network = "tcp"
+		proxy = true
+	}
+
+	formattedAddress := ""
+	switch network {
+	case "unix":
+		formattedAddress = "unix:" + address
+	case "tcp":
+		formattedAddress = "http://localhost" + address
+	default:
+		formattedAddress = fmt.Sprintf(`(%s) %s`, network, address)
+	}
+
+	listener, err := net.Listen(network, address)
+	if err != nil {
+		panic(fmt.Errorf("failed to bind to %s: %w", formattedAddress, err))
+	}
+
+	// additional permission handling for unix sockets
+	if network == "unix" {
+		mode, err := strconv.ParseUint(socketMode, 8, 0)
+		if err != nil {
+			listener.Close()
+			panic(fmt.Errorf("could not parse socket mode %s: %w", socketMode, err))
+		}
+
+		err = os.Chmod(address, os.FileMode(mode))
+		if err != nil {
+			listener.Close()
+			panic(fmt.Errorf("could not change socket mode: %w", err))
+		}
+	}
+
+	if proxy {
+		slog.Warn("listener PROXY enabled")
+		formattedAddress += " +PROXY"
+		listener = &proxyproto.Listener{
+			Listener: listener,
+		}
+	}
+
+	return listener, formattedAddress
+}
+
+func newACMEManager(clientDirectory string, backends map[string]http.Handler) *autocert.Manager {
+	manager := &autocert.Manager{
+		Prompt: autocert.AcceptTOS,
+		HostPolicy: autocert.HostPolicy(func(ctx context.Context, host string) error {
+			if utils.SelectHTTPHandler(backends, host) != nil {
+				return nil
+			}
+			return fmt.Errorf("acme/autocert: host %s not configured in backends", host)
+		}),
+		Client: &acme.Client{
+			HTTPClient:   http.DefaultClient,
+			DirectoryURL: clientDirectory,
+		},
+	}
+	return manager
+}

lib/settings/settings.go

@@ -0,0 +1,49 @@
+package settings
+
+import "maps"
+
+type Settings struct {
+	Bind Bind `yaml:"bind"`
+
+	Backends map[string]Backend `yaml:"backends"`
+
+	BindDebug   string `yaml:"bind-debug"`
+	BindMetrics string `yaml:"bind-metrics"`
+
+	Strings Strings `yaml:"strings"`
+
+	// Links to add to challenge/error pages like privacy/impressum.
+	Links []Link `yaml:"links"`
+
+	ChallengeTemplate string `yaml:"challenge-template"`
+
+	// ChallengeTemplateOverrides Key/Value overrides for the current chosen template
+	ChallengeTemplateOverrides map[string]string `yaml:"challenge-template-overrides"`
+}
+
+type Link struct {
+	Name string `yaml:"name"`
+	URL  string `yaml:"url"`
+}
+
+var DefaultSettings = Settings{
+	Strings:           DefaultStrings,
+	ChallengeTemplate: "anubis",
+	ChallengeTemplateOverrides: func() map[string]string {
+		m := make(map[string]string)
+		maps.Copy(m, map[string]string{
+			"Theme": "",
+			"Logo":  "",
+		})
+		return m
+	}(),
+
+	Bind: Bind{
+		Address:         ":8080",
+		Network:         "tcp",
+		SocketMode:      "0770",
+		Proxy:           false,
+		TLSAcmeAutoCert: "",
+	},
+	Backends: make(map[string]Backend),
+}

lib/settings/strings.go

@@ -0,0 +1,55 @@
+package settings
+
+import (
+	"html/template"
+	"maps"
+)
+
+type Strings map[string]string
+
+var DefaultStrings = make(Strings).set(map[string]string{
+	"title_challenge": "Checking you are not a bot",
+	"title_error":     "Oh no!",
+
+	"noscript_warning": "<p>Sadly, you may need to enable JavaScript to get past this challenge. This is required because AI companies have changed the social contract around how website hosting works.</p>",
+
+	"details_title": "Why am I seeing this?",
+	"details_text": `
+<p>
+	You are seeing this because the administrator of this website has set up <a href="https://git.gammaspectra.live/git/go-away">go-away</a> 
+	to protect the server against the scourge of <a href="https://thelibre.news/foss-infrastructure-is-under-attack-by-ai-companies/">AI companies aggressively scraping websites</a>.
+</p>
+<p>
+	Mass scraping can and does cause downtime for the websites, which makes their resources inaccessible for everyone.
+</p>
+<p>
+	Please note that some challenges requires the use of modern JavaScript features and some plugins may disable these.
+	Disable such plugins for this domain (for example, JShelter) if you encounter any issues.
+</p>
+`,
+	"details_contact_admin_with_request_id": "If you have any issues contact the site administrator and provide the following Request Id",
+
+	"button_refresh_page": "Refresh page",
+
+	"status_loading_challenge":   "Loading challenge",
+	"status_starting_challenge":  "Starting challenge",
+	"status_loading":             "Loading...",
+	"status_calculating":         "Calculating...",
+	"status_challenge_success":   "Challenge success!",
+	"status_challenge_done_took": "Done! Took",
+	"status_error":               "Error:",
+})
+
+func (s Strings) set(v map[string]string) Strings {
+	maps.Copy(s, v)
+	return s
+}
+
+func (s Strings) Get(value string) template.HTML {
+	v, ok := (s)[value]
+	if !ok {
+		// fallback
+		return template.HTML("string:" + value)
+	}
+	return template.HTML(v)
+}

lib/state.go

@@ -1,145 +1,97 @@
 package lib
 
 import (
-	"codeberg.org/meta/gzipped/v2"
-	"context"
+	http_cel "codeberg.org/gone/http-cel"
 	"crypto/ed25519"
 	"crypto/rand"
 	"crypto/sha256"
-	"crypto/subtle"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"git.gammaspectra.live/git/go-away/embed"
 	"git.gammaspectra.live/git/go-away/lib/challenge"
-	"git.gammaspectra.live/git/go-away/lib/challenge/wasm"
-	"git.gammaspectra.live/git/go-away/lib/challenge/wasm/interface"
-	"git.gammaspectra.live/git/go-away/lib/condition"
 	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/lib/settings"
 	"git.gammaspectra.live/git/go-away/utils"
-	"git.gammaspectra.live/git/go-away/utils/inline"
 	"github.com/google/cel-go/cel"
-	"github.com/tetratelabs/wazero/api"
 	"github.com/yl2chen/cidranger"
-	"html/template"
-	"io"
-	"io/fs"
+	"golang.org/x/net/html"
 	"log/slog"
+	"net"
 	"net/http"
 	"net/http/httputil"
-	"net/url"
 	"os"
 	"path"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"time"
 )
 
 type State struct {
-	Client   *http.Client
-	Settings StateSettings
-	UrlPath  string
-	Mux      *http.ServeMux
+	client  *http.Client
+	radb    *utils.RADb
+	urlPath string
 
-	Networks map[string]cidranger.Ranger
-
-	Wasm *wasm.Runner
-
-	Challenges map[challenge.Id]challenge.Challenge
-
-	RulesEnv *cel.Env
-
-	Rules []RuleState
+	programEnv *cel.Env
 
 	publicKey  ed25519.PublicKey
 	privateKey ed25519.PrivateKey
 
-	Poison map[string][]byte
+	opt      settings.Settings
+	settings policy.StateSettings
 
-	ChallengeSolve sync.Map
+	networks map[string]cidranger.Ranger
+
+	challenges challenge.Register
+
+	rules []RuleState
+
+	close chan struct{}
+
+	tagCache *utils.DecayMap[string, []html.Node]
+
+	Mux *http.ServeMux
 }
 
-func (state *State) AwaitChallenge(key []byte, ctx context.Context) challenge.VerifyResult {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	var result atomic.Int64
-
-	state.ChallengeSolve.Store(string(key), ChallengeCallback(func(receivedResult challenge.VerifyResult) {
-		result.Store(int64(receivedResult))
-		cancel()
-	}))
-
-	<-ctx.Done()
-
-	return challenge.VerifyResult(result.Load())
-}
-
-func (state *State) SolveChallenge(key []byte, result challenge.VerifyResult) {
-	if f, ok := state.ChallengeSolve.LoadAndDelete(string(key)); ok && f != nil {
-		if cb, ok := f.(ChallengeCallback); ok {
-			cb(result)
-		}
-	}
-}
-
-type ChallengeCallback func(result challenge.VerifyResult)
-
-type RuleState struct {
-	Name string
-	Hash string
-
-	Host *string
-
-	Program    cel.Program
-	Action     policy.RuleAction
-	Challenges []challenge.Id
-}
-
-type StateSettings struct {
-	Backends               map[string]http.Handler
-	PrivateKeySeed         []byte
-	Debug                  bool
-	PackageName            string
-	ChallengeTemplate      string
-	ChallengeTemplateTheme string
-	ClientIpHeader         string
-}
-
-func NewState(p policy.Policy, settings StateSettings) (state *State, err error) {
+func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSettings) (state *State, err error) {
 	state = new(State)
-	state.Settings = settings
-	state.Client = &http.Client{
+	state.close = make(chan struct{})
+	state.settings = settings
+	state.opt = opt
+	metrics.Reset()
+	state.client = &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}
-	state.UrlPath = "/.well-known/." + state.Settings.PackageName
+	state.radb, err = utils.NewRADb()
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize RADb client: %w", err)
+	}
+
+	state.urlPath = state.Settings().BasePath
 
 	// set a reasonable configuration for default http proxy if there is none
-	for _, backend := range state.Settings.Backends {
+	for _, backend := range state.Settings().Backends {
 		if proxy, ok := backend.(*httputil.ReverseProxy); ok {
 			if proxy.ErrorHandler == nil {
 				proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
-					state.logger(r).Error(err.Error())
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusBadGateway, err, "")
+					state.Logger(r).Error(err.Error())
+					state.ErrorPage(w, r, http.StatusBadGateway, err, "")
 				}
 			}
 		}
 	}
 
-	if len(state.Settings.PrivateKeySeed) > 0 {
-		if len(state.Settings.PrivateKeySeed) != ed25519.SeedSize {
-			return nil, fmt.Errorf("invalid private key seed length: %d", len(state.Settings.PrivateKeySeed))
+	if len(state.Settings().PrivateKeySeed) > 0 {
+		if len(state.Settings().PrivateKeySeed) != ed25519.SeedSize {
+			return nil, fmt.Errorf("invalid private key seed length: %d", len(state.Settings().PrivateKeySeed))
 		}
 
-		state.privateKey = ed25519.NewKeyFromSeed(state.Settings.PrivateKeySeed)
+		state.privateKey = ed25519.NewKeyFromSeed(state.Settings().PrivateKeySeed)
 		state.publicKey = state.privateKey.Public().(ed25519.PublicKey)
 
-		clear(state.Settings.PrivateKeySeed)
+		clear(state.settings.PrivateKeySeed)
 
 	} else {
 		state.publicKey, state.privateKey, err = ed25519.GenerateKey(rand.Reader)
@@ -148,36 +100,91 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 		}
 	}
 
-	privateKeyFingerprint := sha256.Sum256(state.privateKey)
+	if templates["challenge-"+state.Options().ChallengeTemplate+".gohtml"] == nil {
 
-	if state.Settings.ChallengeTemplate == "" {
-		state.Settings.ChallengeTemplate = "anubis"
-	}
-
-	if templates["challenge-"+state.Settings.ChallengeTemplate+".gohtml"] == nil {
-
-		if data, err := os.ReadFile(state.Settings.ChallengeTemplate); err == nil && len(data) > 0 {
-			name := path.Base(state.Settings.ChallengeTemplate)
+		if data, err := os.ReadFile(state.Options().ChallengeTemplate); err == nil && len(data) > 0 {
+			name := path.Base(state.Options().ChallengeTemplate)
 			err := initTemplate(name, string(data))
 			if err != nil {
-				return nil, fmt.Errorf("error loading template %s: %w", settings.ChallengeTemplate, err)
+				return nil, fmt.Errorf("error loading template %s: %w", state.Options().ChallengeTemplate, err)
 			}
-			state.Settings.ChallengeTemplate = name
+			state.opt.ChallengeTemplate = name
 		}
 
-		return nil, fmt.Errorf("no template defined for %s", settings.ChallengeTemplate)
+		return nil, fmt.Errorf("no template defined for %s", state.Options().ChallengeTemplate)
 	}
 
-	state.Networks = make(map[string]cidranger.Ranger)
+	state.networks = make(map[string]cidranger.Ranger)
+
+	networkCache := utils.CachePrefix(state.Settings().Cache, "networks/")
+
 	for k, network := range p.Networks {
+
 		ranger := cidranger.NewPCTrieRanger()
-		for _, e := range network {
-			if e.Url != nil {
-				slog.Debug("loading network url list", "network", k, "url", *e.Url)
-			}
-			prefixes, err := e.FetchPrefixes(state.Client)
+		for i, e := range network {
+			prefixes, err := func() ([]net.IPNet, error) {
+				var useCache bool
+
+				cacheKey := fmt.Sprintf("%s-%d-", k, i)
+				if e.Url != nil {
+					slog.Debug("loading network url list", "network", k, "url", *e.Url)
+					useCache = true
+					sum := sha256.Sum256([]byte(*e.Url))
+					cacheKey += hex.EncodeToString(sum[:4])
+				} else if e.ASN != nil {
+					slog.Debug("loading ASN", "network", k, "asn", *e.ASN)
+					useCache = true
+					cacheKey += strconv.FormatInt(int64(*e.ASN), 10)
+				}
+
+				var cached []net.IPNet
+				if useCache && networkCache != nil {
+					//TODO: add randomness
+					cachedData, err := networkCache.Get(cacheKey, time.Hour*24)
+					var l []string
+					_ = json.Unmarshal(cachedData, &l)
+					for _, n := range l {
+						_, ipNet, err := net.ParseCIDR(n)
+						if err == nil {
+							cached = append(cached, *ipNet)
+						}
+					}
+					if err == nil {
+						// use
+						return cached, nil
+
+					}
+				}
+
+				prefixes, err := e.FetchPrefixes(state.client, state.radb)
+				if err != nil {
+					if len(cached) > 0 {
+						// use cached meanwhile
+						return cached, err
+					}
+					return nil, err
+				}
+				if useCache && networkCache != nil {
+					var l []string
+					for _, n := range prefixes {
+						l = append(l, n.String())
+					}
+					cachedData, err := json.Marshal(l)
+					if err == nil {
+						_ = networkCache.Set(cacheKey, cachedData)
+					}
+				}
+				return prefixes, nil
+			}()
 			if err != nil {
-				return nil, fmt.Errorf("networks %s: error fetching prefixes: %v", k, err)
+				if e.Url != nil {
+					slog.Error("error loading network list", "network", k, "url", *e.Url, "error", err)
+				} else if e.ASN != nil {
+					slog.Error("error loading ASN", "network", k, "asn", *e.ASN, "error", err)
+				} else {
+					slog.Error("error loading list", "network", k, "error", err)
+				}
+				continue
 			}
 			for _, prefix := range prefixes {
 				err = ranger.Insert(cidranger.NewBasicRangerEntry(prefix))
@@ -189,11 +196,9 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 
 		slog.Warn("loaded network prefixes", "network", k, "count", ranger.Len())
 
-		state.Networks[k] = ranger
+		state.networks[k] = ranger
 	}
 
-	state.Wasm = wasm.NewRunner(true)
-
 	err = state.initConditions()
 	if err != nil {
 		return nil, err
@@ -201,7 +206,7 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 
 	var replacements []string
 	for k, entries := range p.Conditions {
-		ast, err := condition.FromStrings(state.RulesEnv, condition.OperatorOr, entries...)
+		ast, err := http_cel.NewAst(state.programEnv, http_cel.OperatorOr, entries...)
 		if err != nil {
 			return nil, fmt.Errorf("conditions %s: error compiling conditions: %v", k, err)
 		}
@@ -216,543 +221,25 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 	}
 	conditionReplacer := strings.NewReplacer(replacements...)
 
-	state.Challenges = make(map[challenge.Id]challenge.Challenge)
+	state.challenges = make(challenge.Register)
 
-	idCounter := challenge.Id(1)
-
-	for challengeName, p := range p.Challenges {
-
-		// allow nesting
-		var conditions []string
-		for _, cond := range p.Conditions {
-			cond = conditionReplacer.Replace(cond)
-			conditions = append(conditions, cond)
+	//TODO: move this to self-contained challenge files
+	for challengeName, pol := range p.Challenges {
+		_, _, err := state.challenges.Create(state, challengeName, pol, conditionReplacer)
+		if err != nil {
+			return nil, fmt.Errorf("challenge %s: %w", challengeName, err)
 		}
-
-		var program cel.Program
-		if len(conditions) > 0 {
-			ast, err := condition.FromStrings(state.RulesEnv, condition.OperatorOr, conditions...)
-			if err != nil {
-				return nil, fmt.Errorf("challenge %s: error compiling conditions: %v", challengeName, err)
-			}
-			program, err = state.RulesEnv.Program(ast)
-			if err != nil {
-				return nil, fmt.Errorf("challenge %s: error compiling program: %v", challengeName, err)
-			}
-		}
-
-		c := challenge.Challenge{
-			Id:                idCounter,
-			Program:           program,
-			Name:              challengeName,
-			Path:              fmt.Sprintf("%s/challenge/%s", state.UrlPath, challengeName),
-			VerifyProbability: p.Runtime.Probability,
-		}
-		idCounter++
-
-		if c.VerifyProbability <= 0 {
-			//10% default
-			c.VerifyProbability = 0.1
-		} else if c.VerifyProbability > 1.0 {
-			c.VerifyProbability = 1.0
-		}
-
-		assetPath := c.Path + "/static/"
-		subFs, err := fs.Sub(embed.ChallengeFs, fmt.Sprintf("challenge/%s/static", challengeName))
-		if err == nil {
-			c.ServeStatic = http.StripPrefix(
-				assetPath,
-				gzipped.FileServer(gzipped.FS(subFs)),
-			)
-		}
-
-		switch p.Mode {
-		default:
-			return nil, fmt.Errorf("unknown challenge mode: %s", p.Mode)
-		case "http":
-			if p.Url == nil {
-				return nil, fmt.Errorf("challenge %s: missing url", challengeName)
-			}
-			method := p.Parameters["http-method"]
-			if method == "" {
-				method = "GET"
-			}
-
-			httpCode, _ := strconv.Atoi(p.Parameters["http-code"])
-			if httpCode == 0 {
-				httpCode = http.StatusOK
-			}
-
-			expectedCookie := p.Parameters["http-cookie"]
-
-			c.Verify = func(key []byte, result string, r *http.Request) (bool, error) {
-				var cookieValue string
-				if expectedCookie != "" {
-					if cookie, err := r.Cookie(expectedCookie); err != nil || cookie == nil {
-						// skip check if we don't have cookie or it's expired
-						return false, nil
-					} else {
-						cookieValue = cookie.Value
-					}
-				}
-				// bind hash of cookie contents
-				sum := sha256.New()
-				sum.Write([]byte(cookieValue))
-				sum.Write([]byte{0})
-				sum.Write(key)
-				sum.Write([]byte{0})
-				sum.Write(state.publicKey)
-
-				if subtle.ConstantTimeCompare(sum.Sum(nil), []byte(result)) == 1 {
-					return true, nil
-				}
-				return false, nil
-			}
-
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-
-				data := RequestDataFromContext(r.Context())
-
-				if result := data.Challenges[c.Id]; result.Ok() {
-					return challenge.ResultPass
-				}
-
-				var cookieValue string
-				if expectedCookie != "" {
-					if cookie, err := r.Cookie(expectedCookie); err != nil || cookie == nil {
-						// skip check if we don't have cookie or it's expired
-						return challenge.ResultContinue
-					} else {
-						cookieValue = cookie.Value
-					}
-				}
-
-				request, err := http.NewRequest(method, *p.Url, nil)
-				if err != nil {
-					return challenge.ResultContinue
-				}
-
-				request.Header = r.Header
-				response, err := state.Client.Do(request)
-				if err != nil {
-					return challenge.ResultContinue
-				}
-				defer response.Body.Close()
-				defer io.Copy(io.Discard, response.Body)
-
-				if response.StatusCode != httpCode {
-					utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-					// continue other challenges!
-
-					//TODO: negatively cache failure
-
-					return challenge.ResultContinue
-				} else {
-					// bind hash of cookie contents
-					sum := sha256.New()
-					sum.Write([]byte(cookieValue))
-					sum.Write([]byte{0})
-					sum.Write(key)
-					sum.Write([]byte{0})
-					sum.Write(state.publicKey)
-					token, err := c.IssueChallengeToken(state.privateKey, key, sum.Sum(nil), expiry)
-					if err != nil {
-						utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-					} else {
-						utils.SetCookie(utils.CookiePrefix+challengeName, token, expiry, w)
-					}
-
-					data.Challenges[c.Id] = challenge.VerifyResultPASS
-
-					// we passed it!
-					return challenge.ResultPass
-				}
-			}
-
-		case "cookie":
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-
-				token, err := c.IssueChallengeToken(state.privateKey, key, nil, expiry)
-				if err != nil {
-					utils.ClearCookie(utils.CookiePrefix+challengeName, w)
-				} else {
-					utils.SetCookie(utils.CookiePrefix+challengeName, token, expiry, w)
-				}
-				// self redirect!
-				//TODO: add redirect loop detect parameter
-				http.Redirect(w, r, r.URL.String(), http.StatusTemporaryRedirect)
-				return challenge.ResultStop
-			}
-		case "meta-refresh":
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-				redirectUri := new(url.URL)
-				redirectUri.Path = c.Path + "/verify-challenge"
-
-				values := make(url.Values)
-				values.Set("result", hex.EncodeToString(key))
-				values.Set("redirect", r.URL.String())
-				values.Set("requestId", r.Header.Get("X-Away-Id"))
-
-				redirectUri.RawQuery = values.Encode()
-
-				_ = state.challengePage(w, r.Header.Get("X-Away-Id"), http.StatusTeapot, "", map[string]any{
-					"Meta": map[string]string{
-						"refresh": "0; url=" + redirectUri.String(),
-					},
-				})
-
-				return challenge.ResultStop
-			}
-		case "header-refresh":
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-				redirectUri := new(url.URL)
-				redirectUri.Path = c.Path + "/verify-challenge"
-
-				values := make(url.Values)
-				values.Set("result", hex.EncodeToString(key))
-				values.Set("redirect", r.URL.String())
-				values.Set("requestId", r.Header.Get("X-Away-Id"))
-
-				redirectUri.RawQuery = values.Encode()
-
-				// self redirect!
-				w.Header().Set("Refresh", "0; url="+redirectUri.String())
-
-				_ = state.challengePage(w, r.Header.Get("X-Away-Id"), http.StatusTeapot, "", nil)
-
-				return challenge.ResultStop
-			}
-		case "preload-link":
-			deadline, _ := time.ParseDuration(p.Parameters["preload-early-hint-deadline"])
-			if deadline == 0 {
-				deadline = time.Second * 3
-			}
-
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-				// this only works on HTTP/2 and HTTP/3
-
-				if r.ProtoMajor < 2 {
-					// this can happen if we are an upgraded request from HTTP/1.1 to HTTP/2 in H2C
-					if _, ok := w.(http.Pusher); !ok {
-						return challenge.ResultContinue
-					}
-				}
-
-				data := RequestDataFromContext(r.Context())
-				redirectUri := new(url.URL)
-				redirectUri.Scheme = getRequestScheme(r)
-				redirectUri.Host = r.Host
-				redirectUri.Path = c.Path + "/verify-challenge"
-
-				values := make(url.Values)
-				values.Set("result", hex.EncodeToString(key))
-				values.Set("requestId", r.Header.Get("X-Away-Id"))
-
-				redirectUri.RawQuery = values.Encode()
-
-				w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"preload\"; as=\"style\"; fetchpriority=high", redirectUri.String()))
-				defer func() {
-					// remove old header so it won't show on response!
-					w.Header().Del("Link")
-				}()
-				w.WriteHeader(http.StatusEarlyHints)
-
-				ctx, cancel := context.WithTimeout(r.Context(), deadline)
-				defer cancel()
-				if result := state.AwaitChallenge(key, ctx); result.Ok() {
-					data.Challenges[c.Id] = challenge.VerifyResultPASS
-
-					// this should serve!
-					return challenge.ResultPass
-				}
-
-				data.Challenges[c.Id] = challenge.VerifyResultFAIL
-				// we failed, continue
-				return challenge.ResultContinue
-			}
-		case "resource-load":
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-				redirectUri := new(url.URL)
-				redirectUri.Path = c.Path + "/verify-challenge"
-
-				values := make(url.Values)
-				values.Set("result", hex.EncodeToString(key))
-				values.Set("requestId", r.Header.Get("X-Away-Id"))
-
-				redirectUri.RawQuery = values.Encode()
-
-				// self redirect!
-				w.Header().Set("Refresh", "2; url="+r.URL.String())
-
-				_ = state.challengePage(w, r.Header.Get("X-Away-Id"), http.StatusTeapot, "", map[string]any{
-					"Tags": []template.HTML{
-						template.HTML(fmt.Sprintf("<link href=\"%s\" rel=\"stylesheet\" crossorigin=\"use-credentials\">", redirectUri.String())),
-					},
-				})
-
-				return challenge.ResultStop
-			}
-		case "js":
-			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
-				_ = state.challengePage(w, r.Header.Get("X-Away-Id"), http.StatusTeapot, challengeName, nil)
-
-				return challenge.ResultStop
-			}
-			c.ServeScriptPath = c.Path + "/challenge.mjs"
-			c.ServeScript = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				params, _ := json.Marshal(p.Parameters)
-
-				//TODO: move this to http.go as a template
-				w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
-				w.Header().Set("Content-Type", "text/javascript; charset=utf-8")
-				w.WriteHeader(http.StatusOK)
-
-				err := templates["challenge.mjs"].Execute(w, map[string]any{
-					"Path":       c.Path,
-					"Parameters": string(params),
-					"Random":     cacheBust,
-					"Challenge":  challengeName,
-					"ChallengeScript": func() string {
-						if p.Asset != nil {
-							return assetPath + *p.Asset
-						} else if p.Url != nil {
-							return *p.Url
-						} else {
-							panic("not implemented")
-						}
-					}(),
-				})
-				if err != nil {
-					//TODO: log
-				}
-			})
-		}
-
-		// how to runtime
-		switch p.Runtime.Mode {
-		default:
-			return nil, fmt.Errorf("unknown challenge runtime mode: %s", p.Runtime.Mode)
-		case "":
-		case "http":
-		case "key":
-			mimeType := p.Parameters["key-mime"]
-			if mimeType == "" {
-				mimeType = "text/html; charset=utf-8"
-			}
-
-			httpCode, _ := strconv.Atoi(p.Parameters["key-code"])
-			if httpCode == 0 {
-				httpCode = http.StatusTemporaryRedirect
-			}
-
-			var content []byte
-			if data, ok := p.Parameters["key-content"]; ok {
-				content = []byte(data)
-			}
-
-			c.Verify = func(key []byte, result string, r *http.Request) (bool, error) {
-				resultBytes, err := hex.DecodeString(result)
-				if err != nil {
-					return false, err
-				}
-
-				if subtle.ConstantTimeCompare(resultBytes, key) != 1 {
-					return false, nil
-				}
-				return true, nil
-			}
-
-			c.ServeVerifyChallenge = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
-				redirect, err := utils.EnsureNoOpenRedirect(r.FormValue("redirect"))
-				if err != nil {
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, "")
-					return
-				}
-
-				err = func() (err error) {
-
-					data := RequestDataFromContext(r.Context())
-
-					key := state.GetChallengeKeyForRequest(challengeName, data.Expires, r)
-					result := r.FormValue("result")
-
-					requestId, err := hex.DecodeString(r.FormValue("requestId"))
-					if err == nil {
-						r.Header.Set("X-Away-Id", hex.EncodeToString(requestId))
-					}
-
-					if ok, err := c.Verify(key, result, r); err != nil {
-						return err
-					} else if !ok {
-						state.logger(r).Warn("challenge failed", "challenge", challengeName, "redirect", redirect)
-						utils.ClearCookie(utils.CookiePrefix+challengeName, w)
-
-						state.SolveChallenge(key, challenge.VerifyResultFAIL)
-
-						_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", challengeName), redirect)
-						return nil
-					}
-
-					state.logger(r).Warn("challenge passed", "challenge", challengeName, "redirect", redirect)
-
-					token, err := c.IssueChallengeToken(state.privateKey, key, []byte(result), data.Expires)
-					if err != nil {
-						utils.ClearCookie(utils.CookiePrefix+challengeName, w)
-					} else {
-						utils.SetCookie(utils.CookiePrefix+challengeName, token, data.Expires, w)
-					}
-					data.Challenges[c.Id] = challenge.VerifyResultPASS
-
-					state.SolveChallenge(key, challenge.VerifyResultPASS)
-
-					switch httpCode {
-					case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
-						http.Redirect(w, r, redirect, httpCode)
-					default:
-						w.Header().Set("Content-Type", mimeType)
-						w.WriteHeader(httpCode)
-						if content != nil {
-							_, _ = w.Write(content)
-						}
-					}
-
-					return nil
-				}()
-				if err != nil {
-					utils.ClearCookie(utils.CookiePrefix+challengeName, w)
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, redirect)
-					return
-				}
-			})
-
-		case "wasm":
-			wasmData, err := embed.ChallengeFs.ReadFile(fmt.Sprintf("challenge/%s/runtime/%s", challengeName, p.Runtime.Asset))
-			if err != nil {
-				return nil, fmt.Errorf("c %s: could not load runtime: %w", challengeName, err)
-			}
-			err = state.Wasm.Compile(challengeName, wasmData)
-			if err != nil {
-				return nil, fmt.Errorf("c %s: compiling runtime: %w", challengeName, err)
-			}
-
-			c.ServeMakeChallenge = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				err := state.Wasm.Instantiate(challengeName, func(ctx context.Context, mod api.Module) (err error) {
-
-					data := RequestDataFromContext(r.Context())
-
-					in := _interface.MakeChallengeInput{
-						Key:        state.GetChallengeKeyForRequest(challengeName, data.Expires, r),
-						Parameters: p.Parameters,
-						Headers:    inline.MIMEHeader(r.Header),
-					}
-					in.Data, err = io.ReadAll(r.Body)
-					if err != nil {
-						return err
-					}
-
-					out, err := wasm.MakeChallengeCall(ctx, mod, in)
-					if err != nil {
-						return err
-					}
-
-					// set output headers
-					for k, v := range out.Headers {
-						w.Header()[k] = v
-					}
-					w.Header().Set("Content-Length", fmt.Sprintf("%d", len(out.Data)))
-					w.WriteHeader(out.Code)
-					_, _ = w.Write(out.Data)
-					return nil
-				})
-				if err != nil {
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, "")
-					return
-				}
-			})
-
-			c.Verify = func(key []byte, result string, r *http.Request) (ok bool, err error) {
-				err = state.Wasm.Instantiate(challengeName, func(ctx context.Context, mod api.Module) (err error) {
-					in := _interface.VerifyChallengeInput{
-						Key:        key,
-						Parameters: p.Parameters,
-						Result:     []byte(result),
-					}
-
-					out, err := wasm.VerifyChallengeCall(ctx, mod, in)
-					if err != nil {
-						return err
-					}
-
-					if out == _interface.VerifyChallengeOutputError {
-						return errors.New("error checking challenge")
-					}
-					ok = out == _interface.VerifyChallengeOutputOK
-					return nil
-				})
-				if err != nil {
-					return false, err
-				}
-				return ok, nil
-			}
-		}
-
-		state.Challenges[c.Id] = c
 	}
 
-	for _, rule := range p.Rules {
-		hasher := sha256.New()
-		hasher.Write([]byte(rule.Name))
-		hasher.Write([]byte{0})
-		if rule.Host != nil {
-			hasher.Write([]byte(*rule.Host))
-		}
-		hasher.Write([]byte{0})
-		hasher.Write(privateKeyFingerprint[:])
-		sum := hasher.Sum(nil)
-
-		challenges := make([]challenge.Id, 0, len(rule.Challenges))
-
-		for _, challengeName := range rule.Challenges {
-			c, ok := state.GetChallengeByName(challengeName)
-			if !ok {
-				return nil, fmt.Errorf("challenge %s not found", challengeName)
-			}
-			challenges = append(challenges, c.Id)
-		}
-
-		r := RuleState{
-			Name:       rule.Name,
-			Hash:       hex.EncodeToString(sum[:8]),
-			Host:       rule.Host,
-			Action:     policy.RuleAction(strings.ToUpper(rule.Action)),
-			Challenges: challenges,
-		}
-
-		if (r.Action == policy.RuleActionCHALLENGE || r.Action == policy.RuleActionCHECK) && len(r.Challenges) == 0 {
-			return nil, fmt.Errorf("no challenges found in rule %s", rule.Name)
-		}
-
-		// allow nesting
-		var conditions []string
-		for _, cond := range rule.Conditions {
-			cond = conditionReplacer.Replace(cond)
-			conditions = append(conditions, cond)
-		}
-
-		ast, err := condition.FromStrings(state.RulesEnv, condition.OperatorOr, conditions...)
+	for _, r := range p.Rules {
+		rule, err := NewRuleState(state, r, conditionReplacer, nil)
 		if err != nil {
-			return nil, fmt.Errorf("rules %s: error compiling conditions: %v", rule.Name, err)
+			return nil, fmt.Errorf("rule %s: %w", r.Name, err)
 		}
-		program, err := state.RulesEnv.Program(ast)
-		if err != nil {
-			return nil, fmt.Errorf("rules %s: error compiling program: %v", rule.Name, err)
-		}
-		r.Program = program
 
-		slog.Warn("loaded rule", "rule", r.Name, "hash", r.Hash, "action", rule.Action)
+		slog.Warn("loaded rule", "rule", rule.Name, "hash", rule.Hash, "action", rule.Action, "children", len(rule.Children))
 
-		state.Rules = append(state.Rules, r)
+		state.rules = append(state.rules, rule)
 	}
 
 	state.Mux = http.NewServeMux()
@@ -761,14 +248,30 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 		return nil, err
 	}
 
+	state.tagCache = utils.NewDecayMap[string, []html.Node]()
+
+	go func() {
+		ticker := time.NewTicker(time.Minute * 37)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				state.tagCache.Decay()
+			case <-state.close:
+				return
+			}
+		}
+	}()
+
 	return state, nil
 }
 
-func (state *State) GetChallengeByName(name string) (challenge.Challenge, bool) {
-	for _, c := range state.Challenges {
-		if c.Name == name {
-			return c, true
-		}
+func (state *State) Close() error {
+	select {
+	case <-state.close:
+		return errors.New("already closed")
+	default:
+		close(state.close)
 	}
-	return challenge.Challenge{}, false
+	return nil
 }

lib/template.go

@@ -0,0 +1,153 @@
+package lib
+
+import (
+	"bytes"
+	"git.gammaspectra.live/git/go-away/embed"
+	"git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/utils"
+	"html/template"
+	"maps"
+	"net/http"
+)
+
+var templates map[string]*template.Template
+
+func init() {
+
+	templates = make(map[string]*template.Template)
+
+	dir, err := embed.TemplatesFs.ReadDir(".")
+	if err != nil {
+		panic(err)
+	}
+	for _, e := range dir {
+		if e.IsDir() {
+			continue
+		}
+		data, err := embed.TemplatesFs.ReadFile(e.Name())
+		if err != nil {
+			panic(err)
+		}
+		err = initTemplate(e.Name(), string(data))
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func initTemplate(name, data string) error {
+	tpl := template.New(name).Funcs(template.FuncMap{
+		"attr": func(s string) template.HTMLAttr {
+			return template.HTMLAttr(s)
+		},
+		"safe": func(s string) template.HTML {
+			return template.HTML(s)
+		},
+	})
+	_, err := tpl.Parse(data)
+	if err != nil {
+		return err
+	}
+	templates[name] = tpl
+	return nil
+}
+
+func (state *State) ChallengePage(w http.ResponseWriter, r *http.Request, status int, reg *challenge.Registration, params map[string]any) {
+	data := challenge.RequestDataFromContext(r.Context())
+	input := make(map[string]any)
+	input["Id"] = data.Id.String()
+	input["Random"] = utils.CacheBust()
+
+	input["Path"] = state.UrlPath()
+	input["Links"] = state.Options().Links
+	input["Strings"] = state.Options().Strings
+	for k, v := range state.Options().ChallengeTemplateOverrides {
+		input[k] = v
+	}
+
+	if reg != nil {
+		input["Challenge"] = reg.Name
+	}
+
+	maps.Copy(input, params)
+
+	if _, ok := input["Title"]; !ok {
+		input["Title"] = state.Options().Strings.Get("title_challenge")
+	}
+
+	if data.GetOptBool(challenge.RequestOptCacheMetaTags, false) {
+		backend, host := data.BackendHost()
+		if tags := state.fetchMetaTags(host, backend, r); len(tags) > 0 {
+			tagMap, _ := input["Meta"].([]map[string]string)
+
+			for _, tag := range tags {
+				tagAttrs := make(map[string]string, len(tag.Attr))
+				for _, v := range tag.Attr {
+					tagAttrs[v.Key] = v.Val
+				}
+				tagMap = append(tagMap, tagAttrs)
+			}
+			input["Meta"] = tagMap
+		}
+	}
+
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+
+	buf := bytes.NewBuffer(make([]byte, 0, 8192))
+
+	err := templates["challenge-"+state.Options().ChallengeTemplate+".gohtml"].Execute(buf, input)
+	if err != nil {
+		state.ErrorPage(w, r, http.StatusInternalServerError, err, "")
+	} else {
+		w.WriteHeader(status)
+		_, _ = w.Write(buf.Bytes())
+	}
+}
+
+func (state *State) ErrorPage(w http.ResponseWriter, r *http.Request, status int, err error, redirect string) {
+	data := challenge.RequestDataFromContext(r.Context())
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+
+	buf := bytes.NewBuffer(make([]byte, 0, 8192))
+
+	input := map[string]any{
+		"Id":        data.Id.String(),
+		"Random":    utils.CacheBust(),
+		"Error":     err.Error(),
+		"Path":      state.UrlPath(),
+		"Theme":     "",
+		"Title":     template.HTML(string(state.Options().Strings.Get("title_error")) + " " + http.StatusText(status)),
+		"Challenge": "",
+		"Redirect":  redirect,
+		"Links":     state.Options().Links,
+		"Strings":   state.Options().Strings,
+	}
+	for k, v := range state.Options().ChallengeTemplateOverrides {
+		input[k] = v
+	}
+
+	if data.GetOptBool(challenge.RequestOptCacheMetaTags, false) {
+		backend, host := data.BackendHost()
+		if tags := state.fetchMetaTags(host, backend, r); len(tags) > 0 {
+			tagMap, _ := input["Meta"].([]map[string]string)
+
+			for _, tag := range tags {
+				tagAttrs := make(map[string]string, len(tag.Attr))
+				for _, v := range tag.Attr {
+					tagAttrs[v.Key] = v.Val
+				}
+				tagMap = append(tagMap, tagAttrs)
+			}
+			input["Meta"] = tagMap
+		}
+	}
+
+	err2 := templates["challenge-"+state.Options().ChallengeTemplate+".gohtml"].Execute(buf, input)
+	if err2 != nil {
+		// nested errors!
+		panic(err2)
+	} else {
+		w.WriteHeader(status)
+		_, _ = w.Write(buf.Bytes())
+	}
+}

tests/action_test.go

@@ -0,0 +1,280 @@
+package tests
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/utils"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func testAction(t *testing.T, pol policy.Policy, expected int, url string) (*http.Response, error) {
+	settings := setupDefaultSettings(t)
+	var r *http.Response
+	err := MakeGoAwayState(pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		request, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return err
+		}
+		request.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		response, err := do(request)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != expected {
+			return fmt.Errorf("expected status code %d, got %d", expected, response.StatusCode)
+		}
+		r = response
+
+		return nil
+	})
+	return r, err
+}
+
+func TestActionPass(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: pass
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	_, err = testAction(t, *pol, http.StatusOK, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestActionNone(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: none
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	_, err = testAction(t, *pol, http.StatusOK, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestActionDrop(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: drop
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusForbidden, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) != 0 {
+		t.Fatal(fmt.Errorf("expected empty response, got %s", string(data)))
+	}
+}
+
+func TestActionDeny(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: deny
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusForbidden, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) == 0 {
+		t.Fatal(errors.New("expected non-empty response, got none"))
+	}
+}
+
+func TestActionBlock(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: block
+    settings:
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusForbidden, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) == 0 {
+		t.Fatal(errors.New("expected non-empty response, got none"))
+	}
+}
+
+func TestActionCode(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: code
+    settings:
+      http-code: 418
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	_, err = testAction(t, *pol, http.StatusTeapot, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestActionContextResponseHeaders(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test
+    conditions: ["true"]
+    action: context
+    settings:
+      response-headers:
+        X-World-Domination: yes
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	response, err := testAction(t, *pol, http.StatusOK, "/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response.Header.Get("X-World-Domination") != "yes" {
+		t.Fatal(fmt.Errorf("expected header set, got %s", response.Header.Get("X-World-Domination")))
+	}
+}
+
+func TestActionContextSetMetaTags(t *testing.T) {
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+rules:
+  - name: test-context
+    conditions: ["true"]
+    action: context
+    settings:
+      context-set:
+        proxy-meta-tags: yes
+
+  - name: test
+    conditions: ["true"]
+    action: deny
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	uri, err := url.Parse("/test")
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+	q := uri.Query()
+	q.Set("mime-type", "text/html")
+	q.Set("content", base64.RawURLEncoding.EncodeToString([]byte(`
+<!DOCTYPE html>
+<html>
+<head>
+	<meta name="description" content="test">
+</head>
+</html>
+`)))
+
+	uri.RawQuery = q.Encode()
+
+	response, err := testAction(t, *pol, http.StatusForbidden, uri.String())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tags := utils.FetchTagsFromReader(response.Body, "meta")
+
+	if str := func() string {
+		for _, t := range tags {
+			var is bool
+			var val string
+			for _, a := range t.Attr {
+				if a.Key == "name" && a.Val == "description" {
+					is = true
+				}
+				if a.Key == "content" {
+					val = a.Val
+				}
+			}
+			if is {
+				return val
+			}
+		}
+		return "NONE"
+	}(); str != "test" {
+		t.Fatal(fmt.Errorf("expected meta tag with 'test', got %s", str))
+	}
+}

tests/away.go

@@ -0,0 +1,34 @@
+package tests
+
+import (
+	"git.gammaspectra.live/git/go-away/lib"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"git.gammaspectra.live/git/go-away/lib/settings"
+	"net/http"
+	"net/http/httptest"
+)
+
+var DefaultSettings = policy.StateSettings{
+	Cache: nil,
+	Backends: map[string]http.Handler{
+		"*": MakeTestBackend(),
+	},
+	MainName:              "go-away/tests",
+	MainVersion:           "testing",
+	BasePath:              "/.go-away",
+	ChallengeResponseCode: http.StatusTeapot,
+	ClientIpHeader:        "X-Forwarded-For",
+}
+
+func MakeGoAwayState(pol policy.Policy, stateSettings policy.StateSettings, f func(do func(r *http.Request, errs ...error) (*http.Response, error)) error) error {
+	state, err := lib.NewState(pol, settings.DefaultSettings, stateSettings)
+	if err != nil {
+		return err
+	}
+
+	return f(func(r *http.Request, errs ...error) (*http.Response, error) {
+		recorder := httptest.NewRecorder()
+		state.ServeHTTP(recorder, r)
+		return recorder.Result(), nil
+	})
+}

tests/backend.go

@@ -0,0 +1,57 @@
+package tests
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"strconv"
+)
+
+func MakeTestBackend() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		responseCode := http.StatusOK
+		var err error
+		if opt := q.Get("http-code"); opt != "" {
+			rc, err := strconv.ParseInt(opt, 10, 64)
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			responseCode = int(rc)
+		}
+		type ResponseJson struct {
+			Method string `json:"method"`
+			Path   string `json:"path"`
+			Query  string `json:"query"`
+		}
+
+		if opt := q.Get("mime-type"); opt != "" {
+			w.Header().Set("Content-Type", opt)
+		} else {
+			w.Header().Set("Content-Type", "application/json")
+		}
+
+		var data []byte
+		if opt := q.Get("content"); opt != "" {
+			data, err = base64.RawURLEncoding.DecodeString(opt)
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+		} else {
+			data, err = json.Marshal(ResponseJson{
+				Method: r.Method,
+				Path:   r.URL.Path,
+				Query:  r.URL.RawQuery,
+			})
+			if err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+		}
+
+		w.WriteHeader(responseCode)
+		_, _ = w.Write(data)
+	})
+}

tests/challenge_test.go

@@ -0,0 +1,362 @@
+package tests
+
+import (
+	"encoding/hex"
+	"fmt"
+	challenge2 "git.gammaspectra.live/git/go-away/lib/challenge"
+	"git.gammaspectra.live/git/go-away/lib/policy"
+	"golang.org/x/net/html"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func setupDefaultSettings(t *testing.T) policy.StateSettings {
+	settings := DefaultSettings
+	slog.SetDefault(slog.New(initLogger(t)))
+
+	return settings
+}
+
+func TestChallengeCookie(t *testing.T) {
+	settings := setupDefaultSettings(t)
+
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+challenges:
+  "challenge-cookie":
+    runtime: "cookie"
+
+rules:
+  - name: catch-all
+    conditions: ["true"]
+    action: challenge
+    settings:
+      challenges: ["challenge-cookie"]
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	var expectedCode = http.StatusTemporaryRedirect
+
+	err = MakeGoAwayState(*pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		challenge, err := http.NewRequest(http.MethodGet, "/test", nil)
+		challenge.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		challengeResponse, err := do(challenge)
+		if err != nil {
+			return err
+		}
+		defer challengeResponse.Body.Close()
+		if challengeResponse.StatusCode != expectedCode {
+			return fmt.Errorf("expected challenge status code %d, got %d", expectedCode, challengeResponse.StatusCode)
+		} else if cookies := challengeResponse.Cookies(); len(cookies) == 0 {
+			return fmt.Errorf("expected set cookies to be non-empty, got none")
+		} else if challengeResponse.Header.Get("Location") == "" {
+			return fmt.Errorf("expected header 'Location' to be non-empty, got none")
+		}
+
+		solveLocation := challengeResponse.Header.Get("Location")
+
+		if !strings.HasPrefix(solveLocation, "/test") {
+			return fmt.Errorf("expected next location to start with '/test', got %s", solveLocation)
+		}
+
+		// test pass
+		pass, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		pass.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+		for _, c := range challengeResponse.Cookies() {
+			pass.AddCookie(c)
+		}
+
+		response, err := do(pass)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected pass status code %d, got %d", http.StatusOK, response.StatusCode)
+		}
+
+		// test failure
+		fail, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		fail.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err = do(fail)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusForbidden {
+			return fmt.Errorf("expected fail status code %d, got %d", http.StatusForbidden, response.StatusCode)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestChallengeHeaderRefresh(t *testing.T) {
+	settings := setupDefaultSettings(t)
+
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+challenges:
+  "challenge-header-refresh":
+    runtime: "refresh"
+    parameters:
+      refresh-via: "header"
+
+rules:
+  - name: catch-all
+    conditions: ["true"]
+    action: challenge
+    settings:
+      challenges: ["challenge-header-refresh"]
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	var expectedCode = settings.ChallengeResponseCode
+
+	err = MakeGoAwayState(*pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		challenge, err := http.NewRequest(http.MethodGet, "/test", nil)
+		challenge.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		challengeResponse, err := do(challenge)
+		if err != nil {
+			return err
+		}
+		defer challengeResponse.Body.Close()
+		if challengeResponse.StatusCode != expectedCode {
+			return fmt.Errorf("expected challenge status code %d, got %d", expectedCode, challengeResponse.StatusCode)
+		} else if challengeResponse.Header.Get("Refresh") == "" {
+			return fmt.Errorf("expected header 'Refresh' to be non-empty, got none")
+		}
+
+		solveLocation, err := url.QueryUnescape(strings.Split(challengeResponse.Header.Get("Refresh"), "; url=")[1])
+		if err != nil {
+			return err
+		}
+
+		// test solve
+		solve, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		solve.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err := do(solve)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusTemporaryRedirect {
+			return fmt.Errorf("expected solve status code %d, got %d", http.StatusTemporaryRedirect, response.StatusCode)
+		} else if cookies := response.Cookies(); len(cookies) == 0 {
+			return fmt.Errorf("expected set cookies to be non-empty, got none")
+		} else if response.Header.Get("Location") == "" {
+			return fmt.Errorf("expected header 'Location' to be non-empty, got none")
+		} else if !strings.HasPrefix(response.Header.Get("Location"), "/test") {
+			return fmt.Errorf("expected next location to start with '/test', got %s", response.Header.Get("Location"))
+		}
+
+		// test pass
+		pass, err := http.NewRequest(http.MethodGet, response.Header.Get("Location"), nil)
+		pass.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+		for _, c := range response.Cookies() {
+			pass.AddCookie(c)
+		}
+
+		response, err = do(pass)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected pass status code %d, got %d", http.StatusOK, response.StatusCode)
+		}
+
+		// test failure
+		uri, err := url.Parse(solveLocation)
+		q := uri.Query()
+		q.Set(challenge2.QueryArgToken, hex.EncodeToString(make([]byte, challenge2.KeySize)))
+		uri.RawQuery = q.Encode()
+
+		fail, err := http.NewRequest(http.MethodGet, uri.String(), nil)
+		fail.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err = do(fail)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusBadRequest {
+			return fmt.Errorf("expected fail status code %d, got %d", http.StatusBadRequest, response.StatusCode)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestChallengeMetaRefresh(t *testing.T) {
+	settings := setupDefaultSettings(t)
+
+	pol, err := policy.NewPolicy(strings.NewReader(
+		`
+challenges:
+  "challenge-meta-refresh":
+    runtime: "refresh"
+    parameters:
+      refresh-via: "meta"
+
+rules:
+  - name: catch-all
+    conditions: ["true"]
+    action: challenge
+    settings:
+      challenges: ["challenge-meta-refresh"]
+
+`,
+	))
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to create policy: %w", err))
+	}
+
+	var expectedCode = settings.ChallengeResponseCode
+
+	err = MakeGoAwayState(*pol, settings, func(do func(r *http.Request, errs ...error) (*http.Response, error)) error {
+		challenge, err := http.NewRequest(http.MethodGet, "/test", nil)
+		challenge.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		challengeResponse, err := do(challenge)
+		if err != nil {
+			return err
+		}
+		defer challengeResponse.Body.Close()
+		if challengeResponse.StatusCode != expectedCode {
+			return fmt.Errorf("expected challenge status code %d, got %d", expectedCode, challengeResponse.StatusCode)
+		} else if challengeResponse.Header.Get("Refresh") != "" {
+			return fmt.Errorf("expected header 'Refresh' to be empty, got \"%s\"", challengeResponse.Header.Get("Refresh"))
+		}
+
+		node, err := html.ParseWithOptions(challengeResponse.Body, html.ParseOptionEnableScripting(false))
+		if err != nil {
+			return nil
+		}
+
+		var refresh string
+		for n := range node.Descendants() {
+			if n.Type == html.ElementNode && n.Data == "meta" {
+				var is bool
+				var val string
+				for _, a := range n.Attr {
+					if a.Key == "http-equiv" && a.Val == "refresh" {
+						is = true
+					}
+					if a.Key == "content" {
+						val = a.Val
+					}
+				}
+				if is {
+					refresh = val
+					break
+				}
+			}
+		}
+
+		solveLocation, err := url.QueryUnescape(strings.Split(refresh, "; url=")[1])
+		if err != nil {
+			return err
+		}
+
+		// test solve
+		solve, err := http.NewRequest(http.MethodGet, solveLocation, nil)
+		solve.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err := do(solve)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusTemporaryRedirect {
+			return fmt.Errorf("expected solve status code %d, got %d", http.StatusTemporaryRedirect, response.StatusCode)
+		} else if cookies := response.Cookies(); len(cookies) == 0 {
+			return fmt.Errorf("expected set cookies to be non-empty, got none")
+		} else if response.Header.Get("Location") == "" {
+			return fmt.Errorf("expected header 'Location' to be non-empty, got none")
+		} else if !strings.HasPrefix(response.Header.Get("Location"), "/test") {
+			return fmt.Errorf("expected next location to start with '/test', got %s", response.Header.Get("Location"))
+		}
+
+		// test pass
+		pass, err := http.NewRequest(http.MethodGet, response.Header.Get("Location"), nil)
+		pass.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+		for _, c := range response.Cookies() {
+			pass.AddCookie(c)
+		}
+
+		response, err = do(pass)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected pass status code %d, got %d", http.StatusOK, response.StatusCode)
+		}
+
+		// test failure
+		uri, err := url.Parse(solveLocation)
+		q := uri.Query()
+		q.Set(challenge2.QueryArgToken, hex.EncodeToString(make([]byte, challenge2.KeySize)))
+		uri.RawQuery = q.Encode()
+
+		fail, err := http.NewRequest(http.MethodGet, uri.String(), nil)
+		fail.Header.Set(settings.ClientIpHeader, "127.0.0.1")
+		if err != nil {
+			return err
+		}
+
+		response, err = do(fail)
+		if err != nil {
+			return err
+		}
+		defer response.Body.Close()
+		if response.StatusCode != http.StatusBadRequest {
+			return fmt.Errorf("expected fail status code %d, got %d", http.StatusBadRequest, response.StatusCode)
+		}
+
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

tests/logger_test.go

@@ -0,0 +1,57 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"testing"
+)
+
+type logger struct {
+	t     *testing.T
+	attrs []slog.Attr
+}
+
+func (l logger) Enabled(ctx context.Context, level slog.Level) bool {
+	return true
+}
+
+func (l logger) Handle(ctx context.Context, record slog.Record) error {
+	str := fmt.Sprintf("[%s] %s", record.Level, record.Message)
+
+	if record.NumAttrs() > 0 || len(l.attrs) > 0 {
+		str += ": "
+	}
+	for _, attr := range l.attrs {
+		str += fmt.Sprintf("%s=%s ", attr.Key, attr.Value.String())
+	}
+	record.Attrs(func(attr slog.Attr) bool {
+		str += fmt.Sprintf("%s=%s ", attr.Key, attr.Value.String())
+		return true
+	})
+
+	if record.Level == slog.LevelError {
+		l.t.Error(str)
+	} else {
+		l.t.Log(str)
+	}
+	return nil
+}
+
+func (l logger) WithAttrs(attrs []slog.Attr) slog.Handler {
+	newAttrs := make([]slog.Attr, 0, len(attrs)+len(l.attrs))
+	newAttrs = append(newAttrs, l.attrs...)
+	newAttrs = append(newAttrs, attrs...)
+	return logger{
+		t:     l.t,
+		attrs: newAttrs,
+	}
+}
+
+func (l logger) WithGroup(name string) slog.Handler {
+	return l
+}
+
+func initLogger(t *testing.T) slog.Handler {
+	return logger{t: t}
+}

utils/cache.go

@@ -0,0 +1,86 @@
+package utils
+
+import (
+	"errors"
+	"os"
+	"path"
+	"time"
+)
+
+type Cache interface {
+	Get(key string, maxAge time.Duration) ([]byte, error)
+
+	Set(key string, value []byte) error
+}
+
+func CachePrefix(c Cache, prefix string) Cache {
+	if c == nil {
+		return nil
+	}
+	return prefixCache{
+		c:      c,
+		prefix: prefix,
+	}
+}
+
+func CacheDirectory(directory string) (Cache, error) {
+	if stat, err := os.Stat(directory); err != nil {
+		return nil, err
+	} else if !stat.IsDir() {
+		return nil, errors.New("not a directory")
+	}
+	return dirCache(directory), nil
+}
+
+type prefixCache struct {
+	c      Cache
+	prefix string
+}
+
+func (c prefixCache) Get(key string, maxAge time.Duration) ([]byte, error) {
+	return c.c.Get(c.prefix+key, maxAge)
+}
+
+func (c prefixCache) Set(key string, value []byte) error {
+	return c.c.Set(c.prefix+key, value)
+}
+
+type dirCache string
+
+var ErrExpired = errors.New("key expired")
+
+func (d dirCache) Get(key string, maxAge time.Duration) ([]byte, error) {
+	fname := path.Join(string(d), key)
+	stat, err := os.Stat(fname)
+	if err != nil {
+		return nil, err
+	}
+	if stat.IsDir() {
+		return nil, errors.New("key is directory")
+	}
+	data, err := os.ReadFile(fname)
+	if err != nil {
+		return nil, err
+	}
+
+	if stat.ModTime().Before(time.Now().Add(-maxAge)) {
+		return data, ErrExpired
+	} else {
+		return data, nil
+	}
+}
+
+func (d dirCache) Set(key string, value []byte) error {
+	fname := path.Join(string(d), key)
+	fs, err := os.Create(fname)
+	if err != nil {
+		return err
+	}
+	defer fs.Close()
+	_, err = fs.Write(value)
+	fs.Sync()
+	fs.Close()
+
+	_ = os.Chtimes(fname, time.Time{}, time.Now())
+	return err
+}

utils/cookie.go

@@ -1,27 +1,41 @@
 package utils
 
 import (
+	"net"
 	"net/http"
 	"time"
 )
 
 var CookiePrefix = ".go-away-"
 
-func SetCookie(name, value string, expiry time.Time, w http.ResponseWriter) {
+// getValidHost Gets a valid host for an http.Cookie Domain field
+// TODO: bug: does not work with IPv6, see https://github.com/golang/go/issues/65521
+func getValidHost(host string) string {
+	ipStr, _, err := net.SplitHostPort(host)
+	if err != nil {
+		return host
+	}
+	return ipStr
+}
+
+func SetCookie(name, value string, expiry time.Time, w http.ResponseWriter, r *http.Request) {
 	http.SetCookie(w, &http.Cookie{
 		Name:     name,
 		Value:    value,
 		Expires:  expiry,
 		SameSite: http.SameSiteLaxMode,
 		Path:     "/",
+		Domain:   getValidHost(r.Host),
 	})
 }
-func ClearCookie(name string, w http.ResponseWriter) {
+
+func ClearCookie(name string, w http.ResponseWriter, r *http.Request) {
 	http.SetCookie(w, &http.Cookie{
 		Name:     name,
 		Value:    "",
 		Expires:  time.Now().Add(-1 * time.Hour),
 		MaxAge:   -1,
 		SameSite: http.SameSiteLaxMode,
+		Domain:   getValidHost(r.Host),
 	})
 }

utils/decaymap.go

@@ -0,0 +1,73 @@
+package utils
+
+import (
+	"sync"
+	"time"
+)
+
+func zilch[T any]() T {
+	var zero T
+	return zero
+}
+
+type DecayMap[K comparable, V any] struct {
+	data map[K]DecayMapEntry[V]
+	lock sync.RWMutex
+}
+
+type DecayMapEntry[V any] struct {
+	Value  V
+	expiry time.Time
+}
+
+func NewDecayMap[K comparable, V any]() *DecayMap[K, V] {
+	return &DecayMap[K, V]{
+		data: make(map[K]DecayMapEntry[V]),
+	}
+}
+
+func (m *DecayMap[K, V]) Get(key K) (V, bool) {
+	m.lock.RLock()
+	value, ok := m.data[key]
+	m.lock.RUnlock()
+
+	if !ok {
+		return zilch[V](), false
+	}
+
+	if time.Now().After(value.expiry) {
+		m.lock.Lock()
+		// Since previously reading m.data[key], the value may have been updated.
+		// Delete the entry only if the expiry time is still the same.
+		if m.data[key].expiry == value.expiry {
+			delete(m.data, key)
+		}
+		m.lock.Unlock()
+
+		return zilch[V](), false
+	}
+
+	return value.Value, true
+}
+
+func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	m.data[key] = DecayMapEntry[V]{
+		Value:  value,
+		expiry: time.Now().Add(ttl),
+	}
+}
+
+func (m *DecayMap[K, V]) Decay() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	now := time.Now()
+	for key, entry := range m.data {
+		if now.After(entry.expiry) {
+			delete(m.data, key)
+		}
+	}
+}

utils/dnsbl.go

@@ -0,0 +1,75 @@
+package utils
+
+import (
+	"context"
+	"net"
+	"strconv"
+)
+
+type DNSBL struct {
+	target   string
+	resolver *net.Resolver
+}
+
+func NewDNSBL(target string, resolver *net.Resolver) *DNSBL {
+	if resolver == nil {
+		resolver = net.DefaultResolver
+	}
+	return &DNSBL{
+		target:   target,
+		resolver: resolver,
+	}
+}
+
+var nibbleTable = [16]byte{
+	'0', '1', '2', '3',
+	'4', '5', '6', '7',
+	'8', '9', 'a', 'b',
+	'c', 'd', 'e', 'f',
+}
+
+type DNSBLResponse uint8
+
+func (r DNSBLResponse) Bad() bool {
+	return r != ResponseGood && r != ResponseUnknown
+}
+
+const (
+	ResponseGood    = DNSBLResponse(0)
+	ResponseUnknown = DNSBLResponse(255)
+)
+
+func (bl DNSBL) Lookup(ctx context.Context, ip net.IP) (DNSBLResponse, error) {
+	var target []byte
+	if ip4 := ip.To4(); ip4 != nil {
+		// max length preallocate
+		target = make([]byte, 0, len(bl.target)+1+len(ip4)*4)
+
+		for i := len(ip4) - 1; i >= 0; i-- {
+			target = strconv.AppendUint(target, uint64(ip4[i]), 10)
+			target = append(target, '.')
+		}
+	} else {
+		// IPv6
+		// max length preallocate
+		target = make([]byte, 0, len(bl.target)+1+len(ip)*4)
+
+		for i := len(ip) - 1; i >= 0; i-- {
+			target = append(target, nibbleTable[ip[i]&0xf], '.', ip[i]>>4, '.')
+		}
+	}
+
+	target = append(target, bl.target...)
+
+	ips, err := bl.resolver.LookupIP(ctx, "ip4", string(target))
+	if err != nil {
+		return ResponseUnknown, err
+	}
+
+	for _, ip := range ips {
+		ip4 := ip.To4()
+		return DNSBLResponse(ip4[len(ip4)-1]), nil
+	}
+
+	return ResponseUnknown, nil
+}

utils/exp/go.mod

@@ -1,5 +0,0 @@
-module git.gammaspectra.live/git/go-away/utils/exp
-
-go 1.22.0
-
-toolchain go1.22.12

utils/exp/slices/func.go

@@ -1,7 +0,0 @@
-package slices
-
-import "slices"
-
-func EqualFunc[S1 ~[]E1, S2 ~[]E2, E1, E2 any](s1 S1, s2 S2, eq func(E1, E2) bool) bool {
-	return slices.EqualFunc(s1, s2, eq)
-}

utils/fingerprint.go

@@ -0,0 +1,335 @@
+package utils
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/http"
+	"slices"
+	"strconv"
+	"strings"
+	"sync/atomic"
+)
+
+func applyTLSFingerprinter(server *http.Server) {
+	if server.TLSConfig == nil {
+		return
+	}
+	server.TLSConfig = server.TLSConfig.Clone()
+
+	getConfigForClient := server.TLSConfig.GetConfigForClient
+
+	if getConfigForClient == nil {
+		getConfigForClient = func(info *tls.ClientHelloInfo) (*tls.Config, error) {
+			return nil, nil
+		}
+	}
+
+	server.TLSConfig.GetConfigForClient = func(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
+		ja3n, ja4 := buildTLSFingerprint(clientHello)
+		ptr := clientHello.Context().Value(tlsFingerprintKey{})
+		if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
+			fpPtr.ja3n.Store(&ja3n)
+			fpPtr.ja4.Store(&ja4)
+		}
+		return getConfigForClient(clientHello)
+	}
+	server.ConnContext = func(ctx context.Context, c net.Conn) context.Context {
+		return context.WithValue(ctx, tlsFingerprintKey{}, &TLSFingerprint{})
+	}
+}
+
+type tlsFingerprintKey struct{}
+type TLSFingerprint struct {
+	ja3n atomic.Pointer[TLSFingerprintJA3N]
+	ja4  atomic.Pointer[TLSFingerprintJA4]
+}
+
+type TLSFingerprintJA3N [md5.Size]byte
+
+func (f TLSFingerprintJA3N) String() string {
+	return hex.EncodeToString(f[:])
+}
+
+type TLSFingerprintJA4 struct {
+	A [10]byte
+	B [6]byte
+	C [6]byte
+}
+
+func (f TLSFingerprintJA4) String() string {
+	return strings.Join([]string{
+		string(f.A[:]),
+		hex.EncodeToString(f.B[:]),
+		hex.EncodeToString(f.C[:]),
+	}, "_")
+}
+
+func (f *TLSFingerprint) JA3N() *TLSFingerprintJA3N {
+	return f.ja3n.Load()
+}
+
+func (f *TLSFingerprint) JA4() *TLSFingerprintJA4 {
+	return f.ja4.Load()
+}
+
+const greaseMask = 0x0F0F
+const greaseValue = 0x0a0a
+
+// TLS extension numbers
+const (
+	extensionServerName              uint16 = 0
+	extensionStatusRequest           uint16 = 5
+	extensionSupportedCurves         uint16 = 10 // supported_groups in TLS 1.3, see RFC 8446, Section 4.2.7
+	extensionSupportedPoints         uint16 = 11
+	extensionSignatureAlgorithms     uint16 = 13
+	extensionALPN                    uint16 = 16
+	extensionSCT                     uint16 = 18
+	extensionExtendedMasterSecret    uint16 = 23
+	extensionSessionTicket           uint16 = 35
+	extensionPreSharedKey            uint16 = 41
+	extensionEarlyData               uint16 = 42
+	extensionSupportedVersions       uint16 = 43
+	extensionCookie                  uint16 = 44
+	extensionPSKModes                uint16 = 45
+	extensionCertificateAuthorities  uint16 = 47
+	extensionSignatureAlgorithmsCert uint16 = 50
+	extensionKeyShare                uint16 = 51
+	extensionQUICTransportParameters uint16 = 57
+	extensionRenegotiationInfo       uint16 = 0xff01
+	extensionECHOuterExtensions      uint16 = 0xfd00
+	extensionEncryptedClientHello    uint16 = 0xfe0d
+)
+
+func tlsFingerprintJA3(hello *tls.ClientHelloInfo, sortExtensions bool) []byte {
+	buf := make([]byte, 0, 256)
+
+	{
+		var sslVersion uint16
+		var hasGrease bool
+		for _, v := range hello.SupportedVersions {
+			if v&greaseMask != greaseValue {
+				if v > sslVersion {
+					sslVersion = v
+				}
+			} else {
+				hasGrease = true
+			}
+		}
+
+		// maximum TLS 1.2 as specified on JA3, as TLS 1.3 is put in SupportedVersions
+		if slices.Contains(hello.Extensions, extensionSupportedVersions) && hasGrease && sslVersion > tls.VersionTLS12 {
+			sslVersion = tls.VersionTLS12
+		}
+
+		buf = strconv.AppendUint(buf, uint64(sslVersion), 10)
+		buf = append(buf, ',')
+	}
+
+	n := 0
+	for _, cipher := range hello.CipherSuites {
+		//if !slices.Contains(greaseValues[:], cipher) {
+		if cipher&greaseMask != greaseValue {
+			buf = strconv.AppendUint(buf, uint64(cipher), 10)
+			buf = append(buf, '-')
+			n = 1
+		}
+	}
+
+	buf = buf[:len(buf)-n]
+	buf = append(buf, ',')
+	n = 0
+
+	extensions := hello.Extensions
+	if sortExtensions {
+		extensions = slices.Clone(extensions)
+		slices.Sort(extensions)
+	}
+
+	for _, extension := range extensions {
+		if extension&greaseMask != greaseValue {
+			buf = strconv.AppendUint(buf, uint64(extension), 10)
+			buf = append(buf, '-')
+			n = 1
+		}
+	}
+
+	buf = buf[:len(buf)-n]
+	buf = append(buf, ',')
+	n = 0
+
+	for _, curve := range hello.SupportedCurves {
+		if curve&greaseMask != greaseValue {
+			buf = strconv.AppendUint(buf, uint64(curve), 10)
+			buf = append(buf, '-')
+			n = 1
+		}
+	}
+
+	buf = buf[:len(buf)-n]
+	buf = append(buf, ',')
+	n = 0
+
+	for _, point := range hello.SupportedPoints {
+		buf = strconv.AppendUint(buf, uint64(point), 10)
+		buf = append(buf, '-')
+		n = 1
+	}
+
+	buf = buf[:len(buf)-n]
+
+	sum := md5.Sum(buf)
+	return sum[:]
+}
+
+func tlsFingerprintJA4(hello *tls.ClientHelloInfo) (ja4 TLSFingerprintJA4) {
+	buf := make([]byte, 0, 10)
+
+	// TODO: t = TLS, q = QUIC
+	buf = append(buf, 't')
+
+	{
+		var sslVersion uint16
+		for _, v := range hello.SupportedVersions {
+			if v&greaseMask != greaseValue {
+				if v > sslVersion {
+					sslVersion = v
+				}
+			}
+		}
+
+		switch sslVersion {
+		case tls.VersionSSL30:
+			buf = append(buf, 's', '3')
+		case tls.VersionTLS10:
+			buf = append(buf, '1', '0')
+		case tls.VersionTLS11:
+			buf = append(buf, '1', '1')
+		case tls.VersionTLS12:
+			buf = append(buf, '1', '2')
+		case tls.VersionTLS13:
+			buf = append(buf, '1', '3')
+		default:
+			sslVersion -= 0x0201
+			buf = strconv.AppendUint(buf, uint64(sslVersion>>8), 10)
+			buf = strconv.AppendUint(buf, uint64(sslVersion&0xff), 10)
+		}
+
+	}
+
+	if slices.Contains(hello.Extensions, extensionServerName) && hello.ServerName != "" {
+		buf = append(buf, 'd')
+	} else {
+		buf = append(buf, 'i')
+	}
+
+	ciphers := make([]uint16, 0, len(hello.CipherSuites))
+	for _, cipher := range hello.CipherSuites {
+		if cipher&greaseMask != greaseValue {
+			ciphers = append(ciphers, cipher)
+		}
+	}
+
+	extensionCount := 0
+	extensions := make([]uint16, 0, len(hello.Extensions))
+	for _, extension := range hello.Extensions {
+		if extension&greaseMask != greaseValue {
+			extensionCount++
+			if extension != extensionALPN && extension != extensionServerName {
+				extensions = append(extensions, extension)
+			}
+		}
+	}
+
+	schemes := make([]tls.SignatureScheme, 0, len(hello.SignatureSchemes))
+
+	for _, scheme := range hello.SignatureSchemes {
+		if scheme&greaseMask != greaseValue {
+			schemes = append(schemes, scheme)
+		}
+	}
+
+	//TODO: maybe little endian
+	slices.Sort(ciphers)
+	slices.Sort(extensions)
+	//slices.Sort(schemes)
+
+	if len(ciphers) < 10 {
+		buf = append(buf, '0')
+		buf = strconv.AppendUint(buf, uint64(len(ciphers)), 10)
+	} else if len(ciphers) > 99 {
+		buf = append(buf, '9', '9')
+	} else {
+		buf = strconv.AppendUint(buf, uint64(len(ciphers)), 10)
+	}
+
+	if extensionCount < 10 {
+		buf = append(buf, '0')
+		buf = strconv.AppendUint(buf, uint64(extensionCount), 10)
+	} else if extensionCount > 99 {
+		buf = append(buf, '9', '9')
+	} else {
+		buf = strconv.AppendUint(buf, uint64(extensionCount), 10)
+	}
+
+	if len(hello.SupportedProtos) > 0 && len(hello.SupportedProtos[0]) > 1 {
+		buf = append(buf, hello.SupportedProtos[0][0], hello.SupportedProtos[0][len(hello.SupportedProtos[0])-1])
+	} else {
+		buf = append(buf, '0', '0')
+	}
+
+	copy(ja4.A[:], buf)
+
+	ja4.B = ja4SHA256(uint16SliceToHex(ciphers))
+
+	extBuf := uint16SliceToHex(extensions)
+
+	if len(schemes) > 0 {
+		extBuf = append(extBuf, '_')
+		extBuf = append(extBuf, uint16SliceToHex(schemes)...)
+	}
+
+	ja4.C = ja4SHA256(extBuf)
+
+	return ja4
+}
+
+func uint16SliceToHex[T ~uint16](in []T) (out []byte) {
+	if len(in) == 0 {
+		return out
+	}
+	out = slices.Grow(out, hex.EncodedLen(len(in)*2)+len(in))
+
+	for _, n := range in {
+		out = append(out, fmt.Sprintf("%04x", uint16(n))...)
+		out = append(out, ',')
+	}
+	out = out[:len(out)-1]
+
+	return out
+}
+
+func ja4SHA256(buf []byte) [6]byte {
+	if len(buf) == 0 {
+		return [6]byte{0, 0, 0, 0, 0, 0}
+	}
+	sum := sha256.Sum256(buf)
+
+	return [6]byte(sum[:6])
+}
+
+func buildTLSFingerprint(hello *tls.ClientHelloInfo) (ja3n TLSFingerprintJA3N, ja4 TLSFingerprintJA4) {
+	return TLSFingerprintJA3N(tlsFingerprintJA3(hello, true)), tlsFingerprintJA4(hello)
+}
+
+func GetTLSFingerprint(r *http.Request) *TLSFingerprint {
+	ptr := r.Context().Value(tlsFingerprintKey{})
+	if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
+		return fpPtr
+	}
+	return nil
+}

utils/http.go

@@ -2,15 +2,55 @@ package utils
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"net"
 	"net/http"
 	"net/http/httputil"
+	"net/netip"
 	"net/url"
 	"strings"
 )
 
+func NewServer(handler http.Handler, tlsConfig *tls.Config) *http.Server {
+	if tlsConfig == nil {
+		proto := new(http.Protocols)
+		proto.SetHTTP1(true)
+		proto.SetUnencryptedHTTP2(true)
+		h1s := &http.Server{
+			Handler:   handler,
+			Protocols: proto,
+		}
+
+		return h1s
+	} else {
+		server := &http.Server{
+			TLSConfig: tlsConfig,
+			Handler:   handler,
+		}
+		applyTLSFingerprinter(server)
+		return server
+	}
+}
+
+func SelectHTTPHandler(backends map[string]http.Handler, host string) http.Handler {
+	backend, ok := backends[host]
+	if !ok {
+		// do wildcard match
+		wildcard := "*." + strings.Join(strings.Split(host, ".")[1:], ".")
+		backend, ok = backends[wildcard]
+
+		if !ok {
+			// return fallback
+			backend = backends["*"]
+		}
+	}
+	return backend
+}
+
 func EnsureNoOpenRedirect(redirect string) (string, error) {
 	uri, err := url.Parse(redirect)
 	if err != nil {
@@ -29,13 +69,14 @@ func EnsureNoOpenRedirect(redirect string) (string, error) {
 	return uri.String(), nil
 }
 
-func MakeReverseProxy(target string) (*httputil.ReverseProxy, error) {
+func MakeReverseProxy(target string, goDns bool) (*httputil.ReverseProxy, error) {
 	u, err := url.Parse(target)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse target URL: %w", err)
 	}
 
 	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{}
 
 	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
 	if u.Scheme == "unix" {
@@ -49,10 +90,83 @@ func MakeReverseProxy(target string) (*httputil.ReverseProxy, error) {
 		}
 		// tell transport how to handle the unix url scheme
 		transport.RegisterProtocol("unix", UnixRoundTripper{Transport: transport})
+	} else if goDns {
+		dialer := &net.Dialer{
+			Resolver: &net.Resolver{
+				PreferGo: true,
+			},
+		}
+		transport.DialContext = dialer.DialContext
 	}
 
 	rp := httputil.NewSingleHostReverseProxy(u)
+
 	rp.Transport = transport
 
 	return rp, nil
 }
+
+func GetRequestScheme(r *http.Request) string {
+	if proto := r.Header.Get("X-Forwarded-Proto"); proto == "http" || proto == "https" {
+		return proto
+	}
+
+	if r.TLS != nil {
+		return "https"
+	}
+
+	return "http"
+}
+
+func GetRequestAddress(r *http.Request, clientHeader string) netip.AddrPort {
+	strVal := r.RemoteAddr
+
+	if clientHeader != "" {
+		strVal = r.Header.Get(clientHeader)
+	}
+	if strVal != "" {
+		// handle X-Forwarded-For
+		strVal = strings.Split(strVal, ",")[0]
+	}
+
+	// fallback
+	if strVal == "" {
+		strVal = r.RemoteAddr
+	}
+
+	addrPort, err := netip.ParseAddrPort(strVal)
+	if err != nil {
+		addr, err2 := netip.ParseAddr(strVal)
+		if err2 != nil {
+			return netip.AddrPort{}
+		}
+		addrPort = netip.AddrPortFrom(addr, 0)
+	}
+	return addrPort
+}
+
+type remoteAddress struct{}
+
+func SetRemoteAddress(r *http.Request, addrPort netip.AddrPort) *http.Request {
+	return r.WithContext(context.WithValue(r.Context(), remoteAddress{}, addrPort))
+}
+func GetRemoteAddress(ctx context.Context) *netip.AddrPort {
+	ip, ok := ctx.Value(remoteAddress{}).(netip.AddrPort)
+	if !ok {
+		return nil
+	}
+	return &ip
+}
+
+func CacheBust() string {
+	return cacheBust
+}
+
+var cacheBust string
+
+func init() {
+
+	buf := make([]byte, 16)
+	_, _ = rand.Read(buf)
+	cacheBust = base64.RawURLEncoding.EncodeToString(buf)
+}

utils/radb.go

@@ -0,0 +1,169 @@
+package utils
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type RADb struct {
+	target string
+	dialer net.Dialer
+}
+
+const RADBServer = "whois.radb.net:43"
+
+func NewRADb() (*RADb, error) {
+
+	host, port, err := net.SplitHostPort(RADBServer)
+	if err != nil {
+		return nil, err
+	}
+
+	return &RADb{
+		target: fmt.Sprintf("%s:%s", host, port),
+		dialer: net.Dialer{
+			Timeout: 5 * time.Second,
+		},
+	}, nil
+}
+
+var whoisRouteRegex = regexp.MustCompile("(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+)")
+
+func (db *RADb) query(fn func(n int, record []byte) error, queries ...string) error {
+
+	conn, err := db.dialer.Dial("tcp", db.target)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if len(queries) > 1 {
+		// enable persistent conn
+		_ = conn.SetDeadline(time.Now().Add(time.Second * 5))
+		_, err = conn.Write([]byte("!!\n"))
+		if err != nil {
+			return err
+		}
+	}
+
+	scanner := bufio.NewScanner(conn)
+	scanner.Split(bufio.ScanLines)
+	// 16 MiB lines
+	const bufferSize = 1024 * 1024 * 16
+	scanner.Buffer(make([]byte, 0, bufferSize), bufferSize)
+
+	for _, q := range queries {
+
+		_ = conn.SetDeadline(time.Now().Add(time.Second * 5))
+		_, err = conn.Write([]byte(strings.TrimSpace(q) + "\n"))
+		if err != nil {
+			return err
+		}
+
+		n := 0
+
+		for scanner.Scan() {
+			buf := bytes.Trim(scanner.Bytes(), "\r\n")
+			if bytes.HasPrefix(buf, []byte("%")) || bytes.Equal(buf, []byte("C")) {
+				// end of record
+				break
+			}
+			err = fn(n, buf)
+			if err != nil {
+				return err
+			}
+			n++
+		}
+
+		if scanner.Err() != nil {
+			return scanner.Err()
+		}
+	}
+
+	if len(queries) > 1 {
+		// exit
+		_ = conn.SetDeadline(time.Now().Add(time.Second * 5))
+		_, err = conn.Write([]byte("q\n"))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (db *RADb) FetchIPInfo(ip net.IP) (result []string, err error) {
+	var ipNet net.IPNet
+	if ip4 := ip.To4(); ip4 != nil {
+		ipNet = net.IPNet{
+			IP: ip4,
+			// single ip
+			Mask: net.CIDRMask(len(ip4)*8, len(ip4)*8),
+		}
+	} else {
+		ipNet = net.IPNet{
+			IP: ip,
+			// single ip
+			Mask: net.CIDRMask(len(ip)*8, len(ip)*8),
+		}
+	}
+
+	err = db.query(func(n int, record []byte) error {
+		result = append(result, string(record))
+		return nil
+	}, fmt.Sprintf("!r%s,l", ipNet.String()))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (db *RADb) FetchASNets(asn int) (result []net.IPNet, err error) {
+
+	ix := whoisRouteRegex.SubexpIndex("prefix")
+	if ix == -1 {
+		panic("invalid regex prefix")
+	}
+
+	var data []byte
+
+	err = db.query(func(n int, record []byte) error {
+		if n == 0 {
+			// do not append ASN number reply
+			return nil
+		}
+		// pad data
+		if n == 1 {
+			data = append(data, ' ')
+		}
+		data = append(data, record...)
+		return nil
+	},
+		// See https://www.radb.net/query/help
+		// fetch IPv4 routes
+		fmt.Sprintf("!gas%d", asn),
+		// fetch IPv6 routes
+		fmt.Sprintf("!6as%d", asn),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	matches := whoisRouteRegex.FindAllSubmatch(data, -1)
+	for _, match := range matches {
+		_, ipNet, err := net.ParseCIDR(string(match[ix]))
+		if err != nil {
+			return nil, fmt.Errorf("invalid CIDR %s: %w", string(match[ix]), err)
+		}
+		result = append(result, *ipNet)
+	}
+
+	return result, nil
+}

utils/tagfetcher.go

@@ -0,0 +1,59 @@
+package utils
+
+import (
+	"golang.org/x/net/html"
+	"io"
+	"mime"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+)
+
+func FetchTags(backend http.Handler, uri *url.URL, kind string) (result []html.Node) {
+	writer := httptest.NewRecorder()
+	backend.ServeHTTP(writer, &http.Request{
+		Method: http.MethodGet,
+		URL:    uri,
+		Header: http.Header{
+			"User-Agent": []string{"Mozilla 5.0 (compatible; go-away/1.0 fetch-tags) TwitterBot/1.0"},
+			"Accept":     []string{"text/html,application/xhtml+xml"},
+		},
+		Close: true,
+	})
+	response := writer.Result()
+	if response == nil {
+		return nil
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return nil
+	}
+
+	if contentType, _, _ := mime.ParseMediaType(response.Header.Get("Content-Type")); contentType != "text/html" && contentType != "application/xhtml+xml" {
+		return nil
+	}
+
+	return FetchTagsFromReader(response.Body, kind)
+}
+
+func FetchTagsFromReader(r io.Reader, kind string) (result []html.Node) {
+	//TODO: handle non UTF-8 documents
+	node, err := html.ParseWithOptions(r, html.ParseOptionEnableScripting(false))
+	if err != nil {
+		return nil
+	}
+
+	for n := range node.Descendants() {
+		if n.Type == html.ElementNode && n.Data == kind {
+			result = append(result, html.Node{
+				Type:      n.Type,
+				DataAtom:  n.DataAtom,
+				Data:      n.Data,
+				Namespace: n.Namespace,
+				Attr:      n.Attr,
+			})
+		}
+	}
+
+	return result
+}