Point generic challenge to generic.yml

.drone.jsonnet

@@ -51,7 +51,7 @@ local Build(go, alpine, os, arch) = {
     ]
 };
 
-local Publish(go, alpine, os, arch, platforms) = {
+local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
     kind: "pipeline",
     type: "docker",
     name: "publish-" + go + "-alpine" + alpine,
@@ -59,19 +59,18 @@ local Publish(go, alpine, os, arch, platforms) = {
         os: os,
         arch: arch,
     },
-    trigger: {
-        event: ["promote"],
-        target: ["production"],
-    },
+    trigger: trigger,
     steps: [
         {
             name: "docker",
             image: "plugins/buildx",
             privileged: true,
+            environment: {
+                DOCKER_BUILDKIT: "1"
+            },
             settings: {
                   registry: "git.gammaspectra.live",
                   repo: "git.gammaspectra.live/git/go-away",
-                  squash: true,
                   compress: true,
                   platform: platforms,
                   builder_driver: "docker-container",
@@ -79,7 +78,6 @@ local Publish(go, alpine, os, arch, platforms) = {
                     from_builder: "golang:" + go +"-alpine" + alpine,
                     from: "alpine:" + alpine,
                   },
-                  auto_tag: true,
                   auto_tag_suffix: "alpine" + alpine,
                   username: {
                     from_secret: "git_username",
@@ -87,7 +85,7 @@ local Publish(go, alpine, os, arch, platforms) = {
                   password: {
                     from_secret: "git_password",
                   },
-            }
+            } + extra,
         },
     ]
 };
@@ -95,11 +93,16 @@ local Publish(go, alpine, os, arch, platforms) = {
 #
 
 [
-    Build("1.22", "3.20", "linux", "amd64"),
-    Build("1.22", "3.20", "linux", "arm64"),
+    Build("1.24", "3.20", "linux", "amd64"),
+    Build("1.24", "3.20", "linux", "arm64"),
     Build("1.24", "3.21", "linux", "amd64"),
     Build("1.24", "3.21", "linux", "arm64"),
 
-    Publish("1.24", "3.21", "linux", "amd64", ["linux/amd64", "linux/arm64"]),
-    Publish("1.22", "3.20", "linux", "amd64", ["linux/amd64", "linux/arm64"]),
+    # latest
+    Publish("1.24", "3.21", "linux", "amd64", {event: ["push"], branch: ["master"], }, ["linux/amd64", "linux/arm64"], {tags: ["latest"],}) + {name: "publish-latest"},
+
+    # modern
+    Publish("1.24", "3.21", "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, ["linux/amd64", "linux/arm64"], {auto_tag: true,}),
+    # legacy
+    Publish("1.24", "3.20", "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, ["linux/amd64", "linux/arm64"], {auto_tag: true,}),
 ]

.drone.yml

@@ -4,7 +4,7 @@ environment:
   GOARCH: amd64
   GOOS: linux
 kind: pipeline
-name: build-1.22-alpine3.20-amd64
+name: build-1.24-alpine3.20-amd64
 platform:
   arch: amd64
   os: linux
@@ -15,7 +15,7 @@ steps:
   - mkdir .bin
   - go build -v -o ./.bin/go-away ./cmd/go-away
   - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
-  image: golang:1.22-alpine3.20
+  image: golang:1.24-alpine3.20
   name: build
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
@@ -44,7 +44,7 @@ environment:
   GOARCH: arm64
   GOOS: linux
 kind: pipeline
-name: build-1.22-alpine3.20-arm64
+name: build-1.24-alpine3.20-arm64
 platform:
   arch: arm64
   os: linux
@@ -55,7 +55,7 @@ steps:
   - mkdir .bin
   - go build -v -o ./.bin/go-away ./cmd/go-away
   - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
-  image: golang:1.22-alpine3.20
+  image: golang:1.24-alpine3.20
   name: build
 - commands:
   - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
@@ -160,12 +160,50 @@ steps:
 type: docker
 ---
 kind: pipeline
+name: publish-latest
+platform:
+  arch: amd64
+  os: linux
+steps:
+- environment:
+    DOCKER_BUILDKIT: "1"
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    builder_driver: docker-container
+    compress: true
+    password:
+      from_secret: git_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    registry: git.gammaspectra.live
+    repo: git.gammaspectra.live/git/go-away
+    tags:
+    - latest
+    username:
+      from_secret: git_username
+trigger:
+  branch:
+  - master
+  event:
+  - push
+type: docker
+---
+kind: pipeline
 name: publish-1.24-alpine3.21
 platform:
   arch: amd64
   os: linux
 steps:
-- image: plugins/buildx
+- environment:
+    DOCKER_BUILDKIT: "1"
+  image: plugins/buildx
   name: docker
   privileged: true
   settings:
@@ -183,23 +221,25 @@ steps:
     - linux/arm64
     registry: git.gammaspectra.live
     repo: git.gammaspectra.live/git/go-away
-    squash: true
     username:
       from_secret: git_username
 trigger:
   event:
   - promote
+  - tag
   target:
   - production
 type: docker
 ---
 kind: pipeline
-name: publish-1.22-alpine3.20
+name: publish-1.24-alpine3.20
 platform:
   arch: amd64
   os: linux
 steps:
-- image: plugins/buildx
+- environment:
+    DOCKER_BUILDKIT: "1"
+  image: plugins/buildx
   name: docker
   privileged: true
   settings:
@@ -207,7 +247,7 @@ steps:
     auto_tag_suffix: alpine3.20
     build_args:
       from: alpine:3.20
-      from_builder: golang:1.22-alpine3.20
+      from_builder: golang:1.24-alpine3.20
     builder_driver: docker-container
     compress: true
     password:
@@ -217,17 +257,17 @@ steps:
     - linux/arm64
     registry: git.gammaspectra.live
     repo: git.gammaspectra.live/git/go-away
-    squash: true
     username:
       from_secret: git_username
 trigger:
   event:
   - promote
+  - tag
   target:
   - production
 type: docker
 ---
 kind: signature
-hmac: b24b30bb7ac591a385173daeb0edbcd9119918ee9e38be90d232f83b8b767115
+hmac: 8583621811fa483a6594352a8f9eeca7d66f6509f8e36bd2299b9e0723ed1451
 
 ...

Dockerfile

@@ -1,13 +1,13 @@
 ARG from_builder=golang:1.24-alpine3.21
 ARG from=alpine:3.21
 
-ARG TARGETPLATFORM
-ARG TARGETARCH
-ARG TARGETOS
 ARG BUILDPLATFORM
 
 FROM --platform=$BUILDPLATFORM ${from_builder} AS build
 
+ARG TARGETARCH
+ARG TARGETOS
+
 RUN apk update && apk add --no-cache \
     bash \
     git \
@@ -41,16 +41,23 @@ ENV GOAWAY_CHALLENGE_TEMPLATE="anubis"
 ENV GOAWAY_CHALLENGE_TEMPLATE_THEME=""
 ENV GOAWAY_SLOG_LEVEL="WARN"
 ENV GOAWAY_CLIENT_IP_HEADER=""
+ENV GOAWAY_BACKEND_IP_HEADER=""
 ENV GOAWAY_JWT_PRIVATE_KEY_SEED=""
 ENV GOAWAY_BACKEND=""
+ENV GOAWAY_DNSBL="dnsbl.dronebl.org"
+ENV GOAWAY_ACME_AUTOCERT=""
+ENV GOAWAY_CACHE="/cache"
 
 EXPOSE 8080/tcp
 EXPOSE 8080/udp
 
 ENV JWT_PRIVATE_KEY_SEED="${GOAWAY_JWT_PRIVATE_KEY_SEED}"
 
-ENTRYPOINT  /bin/go-away --bind ${GOAWAY_BIND} --bind-network ${GOAWAY_BIND_NETWORK} --socket-mode ${GOAWAY_SOCKET_MODE} \
-            --policy ${GOAWAY_POLICY} --client-ip-header ${GOAWAY_CLIENT_IP_HEADER} \
-            --challenge-template ${GOAWAY_CHALLENGE_TEMPLATE} --challenge-template-theme ${GOAWAY_CHALLENGE_TEMPLATE_THEME} \
-            --slog-level ${GOAWAY_SLOG_LEVEL} \
-            --backend ${GOAWAY_BACKEND}
+ENTRYPOINT  /bin/go-away --bind "${GOAWAY_BIND}" --bind-network "${GOAWAY_BIND_NETWORK}" --socket-mode "${GOAWAY_SOCKET_MODE}" \
+            --policy ${GOAWAY_POLICY} --client-ip-header "${GOAWAY_CLIENT_IP_HEADER}" --backend-ip-header "${GOAWAY_BACKEND_IP_HEADER}" \
+            --cache "${GOAWAY_CACHE}" \
+            --dnsbl "${GOAWAY_DNSBL}" \
+            --challenge-template "${GOAWAY_CHALLENGE_TEMPLATE}" --challenge-template-theme "${GOAWAY_CHALLENGE_TEMPLATE_THEME}" \
+            --slog-level "${GOAWAY_SLOG_LEVEL}" \
+            --acme-autocert "${GOAWAY_ACME_AUTOCERT}" \
+            --backend "${GOAWAY_BACKEND}"

README.md

@@ -0,0 +1,509 @@
+# go-away
+
+Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots.
+
+[![Build Status](https://ci.gammaspectra.live/api/badges/git/go-away/status.svg)](https://ci.gammaspectra.live/git/go-away)
+[![Go Reference](https://pkg.go.dev/badge/git.gammaspectra.live/git/go-away.svg)](https://pkg.go.dev/git.gammaspectra.live/git/go-away)
+
+This documentation is a work in progress. For now, see policy examples under [examples/](examples/).
+
+This Go package can be used as a command on `git.gammaspectra.live/git/go-away/cmd/go-away` or a library under `git.gammaspectra.live/git/go-away/lib`
+
+
+## Support
+
+If you have some suggestion or issue, feel free to open a [New Issue](https://git.gammaspectra.live/git/go-away/issues/new) on the repository. 
+
+[Pull Requests](https://git.gammaspectra.live/git/go-away/pulls) are encouraged and desired.
+
+For real-time chat and other support join IRC on [##go-away](ircs://irc.libera.chat/##go-away) on Libera.Chat. The channel may not be monitored at all times, feel free to ping the operators there.
+
+
+## Features
+
+### Rich rule matching
+
+[Common Expression Language (CEL)](https://cel.dev/overview/cel-overview) is used to allow arbitrary selection of client properties, not only limited to regex. Boolean operators are supported.
+
+Templates can be defined in the Policy to allow reuse of such conditions on rule matching. Challenges can also be gated behind conditions.
+
+See the [CEL Language Definition](https://github.com/google/cel-spec/blob/master/doc/langdef.md) for the syntax.
+
+Rules and conditions are served with this environment:
+
+```
+remoteAddress (net.IP) - Connecting client remote address from headers or properties
+host (string) - HTTP Host
+method (string) - HTTP Method/Verb
+userAgent (string) - HTTP User-Agent header
+path (string) - HTTP request Path
+query (map[string]string) - HTTP request Query arguments
+headers (map[string]string) - HTTP request headers
+   
+Only available when TLS is enabled
+   fpJA3N (string) JA3N TLS Fingerprint
+   fpJA4 (string) JA4 TLS Fingerprint
+```
+
+Additionally, these functions are available:
+```
+Check whether a given IP is listed on the underlying defined network or CIDR
+    inNetwork(networkName string, address net.IP) bool
+    inNetwork(networkCIDR string, address net.IP) bool
+
+Check whether a given IP is listed on the provided DNSBL
+    inDNSBL(address net.IP) bool
+```
+
+### Template support
+
+Internal or external templates can be loaded to customize the look of the challenge or error page. Additionally, themes can be configured to change the look of these quickly.
+
+These templates are included by default:
+
+* `anubis`: An anubis-like themed challenge.
+* `forgejo`: Uses the Forgejo template and assets from your own instance. Supports specifying themes like `forgejo-light` and `forgejo-dark`.
+
+External templates for your site can be loaded specifying a full path to the `.gohtml` file. See [embed/templates/](embed/templates/) for examples to follow.
+
+### Extended rule actions
+
+In addition to the common PASS / CHALLENGE / DENY rules, we offer CHECK and POISON.
+
+CHECK allows the client to be challenged but continue matching rules after these.
+
+POISON sends defined responses to bad clients that will annoy them.
+
+### Multiple challenge matching
+
+Several challenges can be offered as options for rules. This allows users that have passed other challenges before to not be affected.
+
+For example:
+```yaml
+  - name: standard-browser
+    action: challenge
+    challenges: [http-cookie-check, self-preload-link, self-meta-refresh, self-resource-load, js-pow-sha256]
+    conditions:
+      - '($is-generic-browser)'
+```
+
+This rule has the user be checked against a backend, then attempts pass a few browser challenges.
+
+In this case the processing would stop at `self-meta-refresh` due to the behavior of earlier challenges.
+
+Any of these listed challenges being passed in the past will allow the client through, including non-offered `self-resource-load` and `js-pow-sha256`.
+
+### Non-Javascript challenges
+
+Several challenges that do not require JavaScript are offered, some targeting the HTTP stack and others a general browser behavior, or consulting with a backend service.
+
+These can be used for light checking of requests that eliminate most of the low effort scraping.
+
+See [Challenges](#challenges) below for a list of them.
+
+### Custom proof-of-work JS / WASM challenges
+
+A WASM interface for server-side proof generation and checking is offered. We provide `js-pow-sha256` as an example of one.
+
+An internal test has shown you can implement Captchas or other browser fingerprinting tests within this interface.
+
+If you are interested in creating your own, see the [Development](#development) section below.
+
+### Upstream PROXY support
+
+Support for [HAProxy PROXY protocol](https://github.com/haproxy/haproxy/blob/master/doc/proxy-protocol.txt) can be enabled.
+
+This allows sending the client IP without altering the connection or HTTP headers.
+
+Supported by HAProxy, [Caddy](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#proxy_protocol), [nginx](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_protocol) and others.
+
+### Automatic TLS support and HTTP/2 support
+
+You can enable automatic certificate generation and TLS for the site via any ACME directory, which enables HTTP/2.
+
+Without TLS, HTTP/2 cleartext is supported, but you will need to configure the upstream proxy to send this protocol (`h2c://` on Caddy for example).
+
+
+### TLS Fingerprinting
+
+When running with TLS via autocert, TLS Fingerprinting of the incoming client is done.
+
+This can be targeted on conditions or other application logic.
+
+Read more about [JA3](https://medium.com/salesforce-engineering/tls-fingerprinting-with-ja3-and-ja3s-247362855967) and [JA4](https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/README.md).
+
+
+### DNSBL
+
+You can configure a [DNSBL (Domain Name System blocklist)](https://en.wikipedia.org/wiki/Domain_Name_System_blocklist) to be queried on rules and conditions.
+
+This allows you to serve harder or different challenges to higher risk clients, or block them from specific sections.
+
+Only rules that match DNSBL will cause a query to be sent, meaning the bulk of requests will not be sent to this service upstream.
+
+Results will be temporarily cached
+
+By default, [DroneBL](https://dronebl.org/) is used.
+
+### Network range loading
+
+Network ranges can be loaded via fetched JSON / TXT / HTML pages, or via lists. You can filter these using _jq_ or a regex.
+
+Example for _jq_:
+```yaml
+  aws-cloud:
+    - url: https://ip-ranges.amazonaws.com/ip-ranges.json
+      jq-path: '(.prefixes[] | select(has("ip_prefix")) | .ip_prefix), (.prefixes[] | select(has("ipv6_prefix")) | .ipv6_prefix)'
+```
+
+Example for _regex_:
+```yaml
+  cloudflare:
+    - url: https://www.cloudflare.com/ips-v4
+      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
+    - url: https://www.cloudflare.com/ips-v6
+      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"
+```
+
+
+### Sharing of signing seed across instances
+
+You can share the signing secret across multiple of your instances if you'd like to deploy multiple across the world.
+
+That way signed secrets will be verifiable across all the instances.
+
+By default, a random temporary key is generated every run.
+
+### Multiple backend support
+
+Multiple backends are supported, and rules specific on backend can be defined, and conditions and rules can match this as well.
+
+This allows one instance to run multiple domains or subdomains.
+
+### Package path
+
+You can modify the path where challenges are served and package name, if you don't want its presence to be easily discoverable.
+
+No source code editing or forking necessary!
+
+## Why?
+In the past few years this small git instance has been hit by waves and waves of scraping.
+This was usually fought back by random useragent blocks for bots that did not follow [robots.txt](/robots.txt), until the past half year, where low-effort mass scraping was used more prominently.
+
+Recently these networks go from using residential IP blocks to sending requests at several hundred rps.
+
+If the server gets sluggish, more requests pile up. Even when denied they scrape for weeks later. Effectively spray and pray scraping, process later.
+
+At some point about 300Mbit/s of incoming requests (not including the responses) was hitting the server. And all at nonsense URLs
+
+If AI is so smart, why not just git clone the repositories?
+
+
+Xe (anubis creator) has written about similar frustrations in several blogposts:
+
+* [Amazon's AI crawler is making my git server unstable](https://xeiaso.net/notes/2025/amazon-crawler/) [01/17/2025]
+* [Anubis works](https://xeiaso.net/notes/2025/anubis-works/) [04/12/2025]
+
+Drew DeVault (sourcehut) has posted several articles regarding the same issues:
+* [Please stop externalizing your costs directly into my face](https://drewdevault.com/2025/03/17/2025-03-17-Stop-externalizing-your-costs-on-me.html) [17/03/2025]
+  * (fun tidbit: I'm the one quoted as having the feedback discussion interrupted to deal with bots!)
+
+Others were also suffering at the same time [[1]](https://donotsta.re/notice/AreSNZlRlJv73AW7tI) [[2]](https://community.ipfire.org/t/suricata-ruleset-to-prevent-ai-scraping/11974) [[3]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[4]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[5]](https://blog.nytsoi.net/2025/03/01/obliterated-by-ai).
+
+---
+Initially I deployed Anubis, and yeah, it does work!
+
+This tool started as a way to replace [Anubis](https://anubis.techaro.lol/) as it was not found as featureful as desired.
+
+go-away may not be as straight to configure as Anubis but this was chosen to reduce impact on legitimate users, and offers many more options to dynamically target new waves.
+
+### Can't scrapers adapt?
+
+Yes, they can. At the moment their spray-and-pray approach is cheap for them.
+
+If they have to start adding an active browser in their scraping, that makes their collection expensive and slow. 
+
+This would more or less eliminate the high rate low effort passive scraping and replace it with an active model. 
+
+go-anubis offers a highly configurable set of challenges and rules that you can adapt to new ways.
+
+## Example policies
+
+### Forgejo
+
+The policy file at [examples/forgejo.yml](examples/forgejo.yml) provides a ready template to be used on your own Forgejo instance.
+
+Important notes:
+* Edit the `homesite` rule, as it's targeted to common users or orgs on the instance. A better regex might be possible in the future.
+* Edit the `http-cookie-check` challenge, as this will fetch the listed backend with the given session cookie to check for user login.
+* Adjust the desired blocked networks or others. A template list of network ranges is provided, feel free to remove these if not needed.
+* Check the conditions and base rules to change your challenges offered and other ordering.
+* By default Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot are allowed by useragent and network ranges.
+
+### Generic
+
+The policy file at [examples/generic.yml](examples/generic.yml) provides a baseline to place on any site, that can be modified to fit your needs.
+
+Important notes:
+* Edit the `homesite` rule, as it's targeted to pages you always want to have available, like landing pages.
+* Edit the `is-static-asset` condition or the `allow-static-resources` rule to allow static file access as necessary.
+* If you have an API, add a PASS rule targeting it.
+* Check the conditions and base rules to change your challenges offered and other ordering.
+* Add or modify rules to target specific pages on your site as desired.
+* By default Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot are allowed by useragent and network ranges.
+
+
+## Setup
+
+It is recommended to have another reverse proxy above (for example [Caddy](https://caddyserver.com/), nginx, HAProxy) to handle HTTPs or similar.
+
+go-away for now only accepts plaintext connections, although it can take _HTTP/2_ / _h2c_ connections if desired over the same port.
+
+### Binary / Go
+
+Requires Go 1.24+. Builds statically without CGo usage.
+
+```shell
+git clone https://git.gammaspectra.live/git/go-away.git && cd go-away
+
+CGO_ENABLED=0 go build -pgo=auto -v -trimpath -o ./go-away ./cmd/go-away
+
+# Run on port 8080, forwarding matching requests on git.example.com to http://forgejo:3000
+./go-away --bind :8080 \
+--backend git.example.com=http://forgejo:3000 \
+--policy examples/forgejo.yml \
+--challenge-template forgejo --challenge-template-theme forgejo-dark
+
+```
+
+### Dockerfile
+
+Available under [Dockerfile](Dockerfile). See the _docker compose_ below for the environment variables.
+
+### docker compose
+
+Example follows a hypothetical Forgejo server running on `http://forgejo:3000` serving `git.example.com`
+
+```yaml
+networks:
+  forgejo:
+    external: false
+    
+volumes:
+  goaway_cache:
+    
+services:
+  go-away:
+    image: git.gammaspectra.live/git/go-away:latest
+    restart: always
+    ports:
+      - "3000:8080"
+    networks:
+      - forgejo
+    depends_on:
+      - forgejo
+    volumes:
+      - "goaway_cache:/cache"
+      - "./examples/forgejo.yml:/policy.yml:ro"
+    environment:
+      #GOAWAY_BIND: ":8080"
+      # Supported tcp, unix, and proxy (for enabling PROXY module for request unwrapping)
+      #GOAWAY_BIND_NETWORK: "tcp"
+      #GOAWAY_SOCKET_MODE: "0770"
+      
+      # set to letsencrypt or other directory URL to enable HTTPS. Above ports will be TLS only.
+      # enables request JA3N / JA4 client TLS fingerprinting
+      # TLS fingerprints are served on X-TLS-Fingerprint-JA3N and X-TLS-Fingerprint-JA4 headers
+      # TLS fingerprints can be matched against on CEL conditions
+      #GOAWAY_ACME_AUTOCERT: ""
+      
+      # Cache path for several services like certificates and caching network ranges
+      # Can be semi-ephemeral, recommended to be mapped to a permanent volume
+      #GOAWAY_CACHE="/cache"
+      
+      # default is WARN, set to INFO to also see challenge successes and others
+      #GOAWAY_SLOG_LEVEL: "INFO"
+      
+      # this value is used to sign cookies and challenges. by default a new one is generated each time
+      # set to generate to create one, then set the same value across all your instances
+      #GOAWAY_JWT_PRIVATE_KEY_SEED: ""
+      
+      # HTTP header that the client ip will be fetched from
+      # Defaults to the connection ip itself, if set here make sure your upstream proxy sets this properly
+      # Usually X-Forwarded-For is a good pick
+      # Not necessary with GOAWAY_BIND_NETWORK: proxy
+      GOAWAY_CLIENT_IP_HEADER: "X-Real-Ip"
+      
+      # HTTP header that go-away will set the obtained ip will be set to
+      # If left empty, the header on GOAWAY_CLIENT_IP_HEADER will be left as-is
+      #GOAWAY_BACKEND_IP_HEADER: ""
+      
+      GOAWAY_POLICY: "/policy.yml"
+      
+      # Template, and theme for the template to pick. defaults to an anubis-like one
+      # An file path can be specified. See embed/templates for a few examples
+      GOAWAY_CHALLENGE_TEMPLATE: forgejo
+      GOAWAY_CHALLENGE_TEMPLATE_THEME: forgejo-dark
+      
+      # specify a DNSBL for usage in conditions. Defaults to DroneBL 
+      # GOAWAY_DNSBL: "dnsbl.dronebl.org"
+      
+      GOAWAY_BACKEND: "git.example.com=http://forgejo:3000"
+      
+    # additional backends can be specified via more command arguments  
+    # command: ["--backend", "ci.example.com=http://ci:3000"]
+
+  forgejo:
+    # etc.
+
+```
+
+## Challenges
+
+#### http
+
+Verify incoming requests against a specified backend to allow the user through. Cookies and some other headers are passed.
+
+For example, this allows verifying the user cookies against the backend to have the user skip all other challenges.
+
+Example on Forgejo, checks that current user is authenticated:
+```yaml
+  http-cookie-check:
+    mode: http
+    url: http://forgejo:3000/user/stopwatches
+    # url: http://forgejo:3000/repo/search
+    # url: http://forgejo:3000/notifications/new
+    parameters:
+      http-method: GET
+      http-cookie: i_like_gitea
+      http-code: 200
+```
+
+#### preload-link
+
+Requires HTTP/2+ response parsing and logic, silent challenge (does not display a challenge page).
+
+Browsers that support [103 Early Hints](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/103) are indicated to fetch a CSS resource via [Link](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link) preload that solves the challenge.
+
+The server waits until solved or defined timeout, then continues on other challenges if failed.
+
+Example:
+```yaml
+  self-preload-link:
+    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
+    mode: "preload-link"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      preload-early-hint-deadline: 3s
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+```
+
+#### header-refresh
+
+Requires HTTP response parsing and logic, displays challenge site instantly.
+
+Have the browser solve the challenge by following the URL listed on HTTP [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh) instantly.
+
+
+#### meta-refresh
+
+Requires HTTP and HTML response parsing and logic, displays challenge site instantly.
+
+Have the browser solve the challenge by following the URL listed on HTML `<meta http-equiv=refresh>` tag instantly. Equivalent to above.
+
+#### resource-load
+
+Requires HTTP and HTML response parsing and logic, displays challenge site.
+
+Servers a challenge page with a linked resource that is loaded by the browser, which solves the challenge. Page refreshes a few seconds later via [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh).
+
+Example:
+```yaml
+  self-resource-load:
+    mode: "resource-load"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+```
+
+#### cookie
+
+Requires HTTP parsing and a Cookie Jar, silent challenge (does not display a challenge page unless failed).
+
+Serves the client with a Set-Cookie that solves the challenge, and redirects it back to the same page. Browser must present the cookie to load.
+
+Several tools implement this, but usually not mass scrapers.
+
+#### js-pow-sha256
+
+Requires JavaScript and workers, displays challenge site.
+
+Has the user solve a Proof of Work using SHA256 hashes, with configurable difficulty.
+
+Example:
+```yaml
+  js-pow-sha256:
+    # Asset must be under challenges/{name}/static/{asset}
+    # Other files here will be available under that path
+    mode: js
+    asset: load.mjs
+    parameters:
+      # difficulty is number of bits that must be set to 0 from start
+      # Anubis challenge difficulty 5 becomes 5 * 8 = 20
+      difficulty: 20
+    runtime:
+      mode: wasm
+      # Verify must be under challenges/{name}/runtime/{asset}
+      asset: runtime.wasm
+      probability: 0.02
+```
+
+
+
+
+## Development
+
+### Compiling WASM runtime challenge modules
+
+Custom WASM runtime modules follow the WASI `wasip1` preview syscall API.
+
+It is recommended using TinyGo to compile / refresh modules, and some function helpers are provided.
+
+If you want to use a different language or compiler, enable `wasip1` and the following interface must be exported:
+
+```
+// Allocation is a combination of pointer location in WASM memory and size of it
+type Allocation uint64
+
+func (p Allocation) Pointer() uint32 {
+	return uint32(p >> 32)
+}
+func (p Allocation) Size() uint32 {
+	return uint32(p)
+}
+
+
+// MakeChallenge MakeChallengeInput / MakeChallengeOutput are valid JSON.
+// See lib/challenge/interface.go for a definition
+func MakeChallenge(in Allocation[MakeChallengeInput]) Allocation[MakeChallengeOutput]
+
+// VerifyChallenge VerifyChallengeInput is valid JSON.
+// See lib/challenge/interface.go for a definition
+func VerifyChallenge(in Allocation[VerifyChallengeInput]) VerifyChallengeOutput
+
+func malloc(size uint32) uintptr
+func free(size uintptr)
+
+```
+
+Modules will be recreated for each call, so there is no state leftover

cmd/go-away/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/tls"
 	"encoding/hex"
 	"errors"
 	"flag"
@@ -11,6 +12,9 @@ import (
 	"git.gammaspectra.live/git/go-away/lib"
 	"git.gammaspectra.live/git/go-away/lib/policy"
 	"git.gammaspectra.live/git/go-away/utils"
+	"github.com/pires/go-proxyproto"
+	"golang.org/x/crypto/acme"
+	"golang.org/x/crypto/acme/autocert"
 	"gopkg.in/yaml.v3"
 	"log"
 	"log/slog"
@@ -18,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path"
 	"runtime/debug"
 	"strconv"
 	"strings"
@@ -25,7 +30,12 @@ import (
 	"time"
 )
 
-func setupListener(network, address, socketMode string) (net.Listener, string) {
+func setupListener(network, address, socketMode string, proxy bool) (net.Listener, string) {
+	if network == "proxy" {
+		network = "tcp"
+		proxy = true
+	}
+
 	formattedAddress := ""
 	switch network {
 	case "unix":
@@ -56,6 +66,14 @@ func setupListener(network, address, socketMode string) (net.Listener, string) {
 		}
 	}
 
+	if proxy {
+		slog.Warn("listener PROXY enabled")
+		formattedAddress += " +PROXY"
+		listener = &proxyproto.Listener{
+			Listener: listener,
+		}
+	}
+
 	return listener, formattedAddress
 }
 
@@ -79,16 +97,46 @@ func (v *MultiVar) Set(value string) error {
 	return nil
 }
 
+func newACMEManager(clientDirectory string, backends map[string]http.Handler) *autocert.Manager {
+
+	var domains []string
+	for d := range backends {
+		parts := strings.Split(d, ":")
+		d = parts[0]
+		if net.ParseIP(d) != nil {
+			continue
+		}
+		domains = append(domains, d)
+	}
+
+	manager := &autocert.Manager{
+		Prompt:     autocert.AcceptTOS,
+		HostPolicy: autocert.HostWhitelist(domains...),
+		Client: &acme.Client{
+			HTTPClient:   http.DefaultClient,
+			DirectoryURL: clientDirectory,
+		},
+	}
+	return manager
+}
+
 func main() {
-	bind := flag.String("bind", ":8080", "network address to bind HTTP to")
+	bind := flag.String("bind", ":8080", "network address to bind HTTP/HTTP(s) to")
 	bindNetwork := flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
+	bindProxy := flag.Bool("bind-proxy", false, "use PROXY protocol in front of the listener")
 	socketMode := flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
 
 	slogLevel := flag.String("slog-level", "WARN", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
 	debugMode := flag.Bool("debug", false, "debug mode with logs and server timings")
-	passThrough := flag.Bool("passthrough", true, "passthrough mode sends all requests to matching backends until state is loaded")
+	passThrough := flag.Bool("passthrough", false, "passthrough mode sends all requests to matching backends until state is loaded")
+	acmeAutocert := flag.String("acme-autocert", "", "enables HTTP(s) mode and uses the provided ACME server URL or available service (available: letsencrypt)")
 
 	clientIpHeader := flag.String("client-ip-header", "", "Client HTTP header to fetch their IP address from (X-Real-Ip, X-Client-Ip, X-Forwarded-For, Cf-Connecting-Ip, etc.)")
+	backendIpHeader := flag.String("backend-ip-header", "", "Backend HTTP header to set the client IP address from, if empty defaults to leaving Client header alone (X-Real-Ip, X-Client-Ip, X-Forwarded-For, Cf-Connecting-Ip, etc.)")
+
+	dnsbl := flag.String("dnsbl", "dnsbl.dronebl.org", "blocklist for DNSBL (default DroneBL)")
+
+	cachePath := flag.String("cache", path.Join(os.TempDir(), "go_away_cache"), "path to temporary cache directory")
 
 	policyFile := flag.String("policy", "", "path to policy YAML file")
 	challengeTemplate := flag.String("challenge-template", "anubis", "name or path of the challenge template to use (anubis, forgejo)")
@@ -186,6 +234,36 @@ func main() {
 		createdBackends[k] = backend
 	}
 
+	if *cachePath != "" {
+		err = os.MkdirAll(*cachePath, 0755)
+		if err != nil {
+			log.Fatal(fmt.Errorf("failed to create cache directory: %w", err))
+		}
+	}
+
+	var tlsConfig *tls.Config
+
+	if *acmeAutocert != "" {
+		switch *acmeAutocert {
+		case "letsencrypt":
+			*acmeAutocert = acme.LetsEncryptURL
+		}
+
+		acmeManager := newACMEManager(*acmeAutocert, createdBackends)
+		if *cachePath != "" {
+			err = os.MkdirAll(path.Join(*cachePath, "acme"), 0755)
+			if err != nil {
+				log.Fatal(fmt.Errorf("failed to create acme cache directory: %w", err))
+			}
+			acmeManager.Cache = autocert.DirCache(path.Join(*cachePath, "acme"))
+		}
+		slog.Warn(
+			"acme-autocert enabled",
+			"directory", *acmeAutocert,
+		)
+		tlsConfig = acmeManager.TLSConfig()
+	}
+
 	var wg sync.WaitGroup
 
 	passThroughCtx, cancelFunc := context.WithCancel(context.Background())
@@ -196,19 +274,17 @@ func main() {
 		go func() {
 			defer wg.Done()
 
-			server := http.Server{
-				Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-					backend, ok := createdBackends[r.Host]
-					if !ok {
-						http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
-						return
-					}
+			server := utils.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				backend, ok := createdBackends[r.Host]
+				if !ok {
+					http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+					return
+				}
 
-					backend.ServeHTTP(w, r)
-				}),
-			}
+				backend.ServeHTTP(w, r)
+			}), tlsConfig)
 
-			listener, listenUrl := setupListener(*bindNetwork, *bind, *socketMode)
+			listener, listenUrl := setupListener(*bindNetwork, *bind, *socketMode, *bindProxy)
 			slog.Warn(
 				"listening passthrough",
 				"url", listenUrl,
@@ -218,8 +294,15 @@ func main() {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
-					log.Fatal(err)
+
+				if tlsConfig != nil {
+					if err := server.ServeTLS(listener, "", ""); !errors.Is(err, http.ErrServerClosed) {
+						log.Fatal(err)
+					}
+				} else {
+					if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
+						log.Fatal(err)
+					}
 				}
 			}()
 
@@ -228,14 +311,14 @@ func main() {
 			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 			defer cancel()
 
-			server.Close()
 			if err := server.Shutdown(ctx); err != nil {
 				log.Fatal(err)
 			}
+			_ = server.Close()
 		}()
 	}
 
-	state, err := lib.NewState(p, lib.StateSettings{
+	settings := lib.StateSettings{
 		Backends:               createdBackends,
 		Debug:                  *debugMode,
 		PackageName:            *packageName,
@@ -243,7 +326,14 @@ func main() {
 		ChallengeTemplateTheme: *challengeTemplateTheme,
 		PrivateKeySeed:         seed,
 		ClientIpHeader:         *clientIpHeader,
-	})
+		BackendIpHeader:        *backendIpHeader,
+	}
+
+	if *dnsbl != "" {
+		settings.DNSBL = utils.NewDNSBL(*dnsbl, net.DefaultResolver)
+	}
+
+	state, err := lib.NewState(p, settings)
 
 	if err != nil {
 		log.Fatal(fmt.Errorf("failed to create state: %w", err))
@@ -253,17 +343,23 @@ func main() {
 	cancelFunc()
 	wg.Wait()
 
-	listener, listenUrl := setupListener(*bindNetwork, *bind, *socketMode)
+	listener, listenUrl := setupListener(*bindNetwork, *bind, *socketMode, *bindProxy)
 	slog.Warn(
 		"listening",
 		"url", listenUrl,
 	)
 
-	server := http.Server{
-		Handler: state,
+	server := utils.NewServer(state, tlsConfig)
+
+	if tlsConfig != nil {
+		if err := server.ServeTLS(listener, "", ""); !errors.Is(err, http.ErrServerClosed) {
+			log.Fatal(err)
+		}
+	} else {
+
+		if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
+			log.Fatal(err)
+		}
 	}
 
-	if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
-		log.Fatal(err)
-	}
 }

examples/forgejo.yml

@@ -1,5 +1,5 @@
 # Example cmdline (forward requests from upstream to port :8080)
-# $ go-away --bind :8080 --backend git.example.com:http://forgejo:3000 --policy examples/forgejo.yml --challenge-template forgejo --challenge-template-theme forgejo-dark
+# $ go-away --bind :8080 --backend git.example.com=http://forgejo:3000 --policy examples/forgejo.yml --challenge-template forgejo --challenge-template-theme forgejo-dark
 
 
 
@@ -93,7 +93,6 @@ networks:
       regex: "\\n(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+) "
 
 
-# todo: define interface
 challenges:
   js-pow-sha256:
     # Asset must be under challenges/{name}/static/{asset}
@@ -112,7 +111,23 @@ challenges:
   self-cookie:
     mode: "cookie"
 
-  # Challenges with a redirect via header (non-JS, requires HTTP parsing and logic)
+
+  # Challenges with a redirect via Link header with rel=preload and early hints (non-JS, requires HTTP parsing, fetching and logic)
+  # Works on HTTP/2 and above!
+  self-preload-link:
+    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
+    mode: "preload-link"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      preload-early-hint-deadline: 3s
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+
+  # Challenges with a redirect via Refresh header (non-JS, requires HTTP parsing and logic)
   self-header-refresh:
     mode: "header-refresh"
     runtime:
@@ -120,7 +135,7 @@ challenges:
       mode: "key"
       probability: 0.1
 
-  # Challenges with a redirect via meta (non-JS, requires HTML parsing and logic)
+  # Challenges with a redirect via Refresh meta (non-JS, requires HTML parsing and logic)
   self-meta-refresh:
     mode: "meta-refresh"
     runtime:
@@ -151,7 +166,6 @@ challenges:
       http-method: GET
       http-cookie: i_like_gitea
       http-code: 200
-      # todo: archive value of session within token to bind it
 
 conditions:
   # Conditions will get replaced on rules AST when found as ($condition-name)
@@ -186,12 +200,14 @@ conditions:
     # Golang proxy and initial fetch
     - 'userAgent.startsWith("GoModuleMirror/")'
     - 'userAgent.startsWith("Go-http-client/") && "go-get" in query && query["go-get"] == "1"'
+    - '"Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"'
   is-git-path:
     - 'path.matches("^/[^/]+/[^/]+/(git-upload-pack|git-receive-pack|HEAD|info/refs|info/lfs|objects)")'
 
   is-generic-robot-ua:
     - 'userAgent.contains("compatible;") && !userAgent.contains("Trident/")'
     - 'userAgent.matches("\\+https?://")'
+    - 'userAgent.contains("@")'
     - 'userAgent.matches("[bB]ot/[0-9]")'
 
   is-tool-ua:
@@ -215,7 +231,7 @@ conditions:
     # Old IE browsers
     - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
     # Old Linux browsers
-    - 'userAgent.contains("Linux i686")'
+    - 'userAgent.contains("Linux i[63]86") || userAgent.contains("FreeBSD i[63]86")'
     # Old Windows browsers
     - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
     # Old mobile browsers
@@ -238,24 +254,18 @@ conditions:
     # user activity tab
     - 'path.matches("^/[^/]+$") && "tab" in query && query.tab == "activity"'
 
-# Rules and conditions are served this environment
-#   remoteAddress (net.IP) - Connecting client remote address from headers or properties
-#   host (string) - HTTP Host
-#   method (string) - HTTP Method/Verb
-#   userAgent (string) - HTTP User-Agent header
-#   path (string) - HTTP request Path
-#   query (map[string]string) - HTTP request Query arguments
-#   headers (map[string]string) - HTTP request headers
-#
-# Additionally these functions are available
-#   inNetwork(networkName string, address net.IP) bool
-#   inNetwork(networkCIDR string, address net.IP) bool
+
 rules:
   - name: allow-well-known-resources
     conditions:
       - '($is-well-known-asset)'
     action: pass
 
+  - name: allow-static-resources
+    conditions:
+      - '($is-static-asset)'
+    action: pass
+
   - name: undesired-networks
     conditions:
       - 'inNetwork("huawei-cloud", remoteAddress) || inNetwork("alibaba-cloud", remoteAddress) || inNetwork("zenlayer-inc", remoteAddress)'
@@ -299,21 +309,22 @@ rules:
   - name: suspicious-crawlers/1
     conditions: ['($is-suspicious-crawler)']
     action: check
-    challenges: [self-header-refresh]
+    challenges: [self-preload-link]
   - name: suspicious-crawlers/2
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [self-header-refresh]
+  - name: suspicious-crawlers/3
     conditions: ['($is-suspicious-crawler)']
     action: check
     challenges: [self-resource-load]
 
-  - name: allow-static-resources
-    conditions:
-      - '($is-static-asset)'
-    action: pass
-
   - name: always-pow-challenge
     conditions:
-      # login paths
-      - 'path.startsWith("/user/sign_up") || path.startsWith("/user/login") || path.startsWith("/user/oauth2/")'
+      # login and sign up paths
+      - 'path.startsWith("/user/sign_up")'
+      - 'path.startsWith("/user/login") || path.startsWith("/user/oauth2/")'
+      - 'path.startsWith("/user/activate")'
       # repo / org / mirror creation paths
       - 'path == "/repo/create" || path == "/repo/migrate" || path ==  "/org/create"'
       # user profile info edit paths
@@ -371,7 +382,7 @@ rules:
   - name: desired-crawlers
     conditions:
       - 'userAgent.contains("+https://kagi.com/bot") && inNetwork("kagibot", remoteAddress)'
-      - '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-InspectionTool")) && inNetwork("googlebot", remoteAddress)'
+      - '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-InspectionTool") || userAgent.contains("Googlebot")) && inNetwork("googlebot", remoteAddress)'
       - 'userAgent.contains("+http://www.bing.com/bingbot.htm") && inNetwork("bingbot", remoteAddress)'
       - 'userAgent.contains("+http://duckduckgo.com/duckduckbot.html") && inNetwork("duckduckbot", remoteAddress)'
       - 'userAgent.contains("+https://help.qwant.com/bot/") && inNetwork("qwantbot", remoteAddress)'
@@ -387,16 +398,10 @@ rules:
       - 'path.matches("(?i)^/(WeebDataHoarder|P2Pool|mirror|git|S\\.O\\.N\\.G|FM10K|Sillycom|pwgen2155|kaitou|metonym)/[^/]+$")'
     action: pass
 
-  - name: suspicious-fetchers
-    action: challenge
-    challenges: [js-pow-sha256, http-cookie-check]
-    conditions:
-      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
-
   # check a sequence of challenges
   - name: heavy-operations/0
     action: check
-    challenges: [self-header-refresh, js-pow-sha256, http-cookie-check]
+    challenges: [self-preload-link, self-header-refresh, js-pow-sha256, http-cookie-check]
     conditions: ['($is-heavy-resource)']
   - name: heavy-operations/1
     action: check
@@ -409,10 +414,23 @@ rules:
     conditions:
       - 'path.matches("^/[^/]+/[^/]+/raw/branch/")'
       - 'path.matches("^/[^/]+/[^/]+/archive/")'
-      - 'path.matches("^/[^/]+/[^/]+/media/")'
       - 'path.matches("^/[^/]+/[^/]+/releases/download/")'
+      - 'path.matches("^/[^/]+/[^/]+/media/") && ($is-generic-browser)'
     action: pass
 
+  # check DNSBL and serve harder challenges
+  - name: undesired-dnsbl
+    conditions:
+      - 'inDNSBL(remoteAddress)'
+    action: check
+    challenges: [js-pow-sha256, http-cookie-check]
+
+  - name: suspicious-fetchers
+    action: check
+    challenges: [js-pow-sha256]
+    conditions:
+      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
+
   # Allow PUT/DELETE/PATCH/POST requests in general
   - name: non-get-request
     action: pass
@@ -420,16 +438,16 @@ rules:
       - '!(method == "HEAD" || method == "GET")'
 
 
-
   - name: standard-tools
     action: challenge
-    challenges: [self-meta-refresh]
+    challenges: [self-cookie]
     conditions:
       - '($is-generic-robot-ua)'
       - '($is-tool-ua)'
+      - '!($is-generic-browser)'
 
   - name: standard-browser
     action: challenge
-    challenges: [http-cookie-check, self-meta-refresh, self-resource-load, js-pow-sha256]
+    challenges: [http-cookie-check, self-preload-link, self-meta-refresh, self-resource-load, js-pow-sha256]
     conditions:
       - '($is-generic-browser)'

examples/generic.yml

@@ -0,0 +1,274 @@
+# Example cmdline (forward requests from upstream to port :8080)
+# $ go-away --bind :8080 --backend site.example.com=http://site:3000 --policy examples/generic.yml --challenge-template anubis
+
+
+
+# Define networks to be used later below
+networks:
+
+  googlebot:
+    - url: https://developers.google.com/static/search/apis/ipranges/googlebot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  bingbot:
+    - url: https://www.bing.com/toolbox/bingbot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  qwantbot:
+    - url: https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  duckduckbot:
+    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
+      regex: "<li>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</li>"
+  yandexbot:
+    # todo: detected as bot
+    # - url: https://yandex.com/ips
+    #  regex: "<span>(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+)[ \\\\t]*</span><br/>"
+    - prefixes:
+        - "5.45.192.0/18"
+        - "5.255.192.0/18"
+        - "37.9.64.0/18"
+        - "37.140.128.0/18"
+        - "77.88.0.0/18"
+        - "84.252.160.0/19"
+        - "87.250.224.0/19"
+        - "90.156.176.0/22"
+        - "93.158.128.0/18"
+        - "95.108.128.0/17"
+        - "141.8.128.0/18"
+        - "178.154.128.0/18"
+        - "185.32.187.0/24"
+        - "2a02:6b8::/29"
+  kagibot:
+    - url: https://kagi.com/bot
+      regex: "\\n(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+) "
+
+
+challenges:
+  js-pow-sha256:
+    # Asset must be under challenges/{name}/static/{asset}
+    # Other files here will be available under that path
+    mode: js
+    asset: load.mjs
+    parameters:
+      difficulty: 15
+    runtime:
+      mode: wasm
+      # Verify must be under challenges/{name}/runtime/{asset}
+      asset: runtime.wasm
+      probability: 0.02
+
+  # Challenges with a cookie, self redirect (non-JS, requires HTTP parsing)
+  self-cookie:
+    mode: "cookie"
+
+
+  # Challenges with a redirect via Link header with rel=preload and early hints (non-JS, requires HTTP parsing, fetching and logic)
+  # Works on HTTP/2 and above!
+  self-preload-link:
+    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
+    mode: "preload-link"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      preload-early-hint-deadline: 3s
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+
+  # Challenges with a redirect via Refresh header (non-JS, requires HTTP parsing and logic)
+  self-header-refresh:
+    mode: "header-refresh"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+
+  # Challenges with a redirect via Refresh meta (non-JS, requires HTML parsing and logic)
+  self-meta-refresh:
+    mode: "meta-refresh"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+
+  # Challenges with loading a random CSS or image document (non-JS, requires HTML parsing and logic)
+  self-resource-load:
+    mode: "resource-load"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+
+conditions:
+  # Conditions will get replaced on rules AST when found as ($condition-name)
+  # Checks to detect a headless chromium via headers only
+  is-headless-chromium:
+    - 'userAgent.contains("HeadlessChrome") || userAgent.contains("HeadlessChromium")'
+    - '"Sec-Ch-Ua" in headers && (headers["Sec-Ch-Ua"].contains("HeadlessChrome") || headers["Sec-Ch-Ua"].contains("HeadlessChromium"))'
+    #- '(userAgent.contains("Chrome/") || userAgent.contains("Chromium/")) && (!("Accept-Language" in headers) || !("Accept-Encoding" in headers))'
+
+  is-generic-browser:
+    - 'userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")'
+
+  is-well-known-asset:
+    - 'path == "/robots.txt"'
+    - 'path.startsWith("/.well-known")'
+
+  is-static-asset:
+    - 'path == "/favicon.ico"'
+    - 'path == "/apple-touch-icon.png"'
+    - 'path == "/apple-touch-icon-precomposed.png"'
+    - 'path.matches("\\.(manifest|ttf|woff|woff2|jpg|jpeg|gif|png|webp|avif|svg|mp4|webm|css|js|mjs|wasm)$")'
+
+
+  is-generic-robot-ua:
+    - 'userAgent.contains("compatible;") && !userAgent.contains("Trident/")'
+    - 'userAgent.matches("\\+https?://")'
+    - 'userAgent.contains("@")'
+    - 'userAgent.matches("[bB]ot/[0-9]")'
+
+  is-tool-ua:
+    - 'userAgent.startsWith("python-requests/")'
+    - 'userAgent.startsWith("Python-urllib/")'
+    - 'userAgent.startsWith("python-httpx/")'
+    - 'userAgent.contains("aoihttp/")'
+    - 'userAgent.startsWith("http.rb/")'
+    - 'userAgent.startsWith("curl/")'
+    - 'userAgent.startsWith("Wget/")'
+    - 'userAgent.startsWith("libcurl/")'
+    - 'userAgent.startsWith("okhttp/")'
+    - 'userAgent.startsWith("Java/")'
+    - 'userAgent.startsWith("Apache-HttpClient//")'
+    - 'userAgent.startsWith("Go-http-client/")'
+    - 'userAgent.startsWith("node-fetch/")'
+    - 'userAgent.startsWith("reqwest/")'
+
+  is-suspicious-crawler:
+    - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
+    # Old IE browsers
+    - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
+    # Old Linux browsers
+    - 'userAgent.contains("Linux i[63]86") || userAgent.contains("FreeBSD i[63]86")'
+    # Old Windows browsers
+    - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
+    # Old mobile browsers
+    - 'userAgent.matches("Android [1-5]\\.") || userAgent.matches("(iPad|iPhone) OS [1-9]_")'
+    # Old generic browsers
+    - 'userAgent.startsWith("Opera/")'
+    #- 'userAgent.matches("Gecko/(201[0-9]|200[0-9])")'
+    - 'userAgent.matches("^Mozilla/[1-4]")'
+
+
+
+rules:
+  - name: allow-well-known-resources
+    conditions:
+      - '($is-well-known-asset)'
+    action: pass
+
+  - name: allow-static-resources
+    conditions:
+      - '($is-static-asset)'
+    action: pass
+
+  - name: undesired-crawlers
+    conditions:
+      - '($is-headless-chromium)'
+      - 'userAgent.startsWith("Lightpanda/")'
+      - 'userAgent.startsWith("masscan/")'
+      # Typo'd opera botnet
+      - 'userAgent.matches("^Opera/[0-9.]+\\.\\(")'
+      # AI bullshit stuff, they do not respect robots.txt even while they read it
+      # TikTok Bytedance AI training
+      - 'userAgent.contains("Bytedance") || userAgent.contains("Bytespider")'
+      # Meta AI training; The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.
+      - 'userAgent.contains("meta-externalagent/") || userAgent.contains("meta-externalfetcher/") || userAgent.contains("FacebookBot")'
+      # Anthropic AI training and usage
+      - 'userAgent.contains("ClaudeBot") || userAgent.contains("Claude-User")|| userAgent.contains("Claude-SearchBot")'
+      # Common Crawl AI crawlers
+      - 'userAgent.contains("CCBot")'
+      # ChatGPT AI crawlers https://platform.openai.com/docs/bots
+      - 'userAgent.contains("GPTBot") || userAgent.contains("OAI-SearchBot") || userAgent.contains("ChatGPT-User")'
+      # Other AI crawlers
+      - 'userAgent.contains("Amazonbot") || userAgent.contains("Google-Extended") || userAgent.contains("PanguBot") || userAgent.contains("AI2Bot") || userAgent.contains("Diffbot") || userAgent.contains("cohere-training-data-crawler") || userAgent.contains("Applebot-Extended")'
+      # SEO / Ads and marketing
+      - 'userAgent.contains("BLEXBot")'
+    action: deny
+
+  - name: unknown-crawlers
+    conditions:
+      # No user agent set
+      - 'userAgent == ""'
+    action: deny
+
+  # check a sequence of challenges
+  - name: suspicious-crawlers/0
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [js-pow-sha256]
+  - name: suspicious-crawlers/1
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [self-preload-link]
+  - name: suspicious-crawlers/2
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [self-header-refresh]
+  - name: suspicious-crawlers/3
+    conditions: ['($is-suspicious-crawler)']
+    action: check
+    challenges: [self-resource-load]
+
+  - name: desired-crawlers
+    conditions:
+      - 'userAgent.contains("+https://kagi.com/bot") && inNetwork("kagibot", remoteAddress)'
+      - '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-InspectionTool") || userAgent.contains("Googlebot")) && inNetwork("googlebot", remoteAddress)'
+      - 'userAgent.contains("+http://www.bing.com/bingbot.htm") && inNetwork("bingbot", remoteAddress)'
+      - 'userAgent.contains("+http://duckduckgo.com/duckduckbot.html") && inNetwork("duckduckbot", remoteAddress)'
+      - 'userAgent.contains("+https://help.qwant.com/bot/") && inNetwork("qwantbot", remoteAddress)'
+      - 'userAgent.contains("+http://yandex.com/bots") && inNetwork("yandexbot", remoteAddress)'
+    action: pass
+
+  - name: homesite
+    conditions:
+      - 'path == "/"'
+    action: pass
+
+  # check DNSBL and serve harder challenges
+  - name: undesired-dnsbl
+    conditions:
+      - 'inDNSBL(remoteAddress)'
+    action: check
+    challenges: [js-pow-sha256]
+
+  - name: suspicious-fetchers
+    action: check
+    challenges: [js-pow-sha256]
+    conditions:
+      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
+
+  # Allow PUT/DELETE/PATCH/POST requests in general
+  - name: non-get-request
+    action: pass
+    conditions:
+      - '!(method == "HEAD" || method == "GET")'
+
+
+  - name: standard-tools
+    action: challenge
+    challenges: [self-cookie]
+    conditions:
+      - '($is-generic-robot-ua)'
+      - '($is-tool-ua)'
+      - '!($is-generic-browser)'
+
+  - name: standard-browser
+    action: challenge
+    challenges: [self-preload-link, self-meta-refresh, self-resource-load, js-pow-sha256]
+    conditions:
+      - '($is-generic-browser)'

go.mod

@@ -1,18 +1,20 @@
 module git.gammaspectra.live/git/go-away
 
-go 1.22.0
+go 1.24.0
 
-toolchain go1.22.12
+toolchain go1.24.2
 
 require (
 	codeberg.org/meta/gzipped/v2 v2.0.0-20231111234332-aa70c3194756
 	github.com/andybalholm/brotli v1.1.1
-	github.com/go-jose/go-jose/v4 v4.0.5
+	github.com/go-jose/go-jose/v4 v4.1.0
 	github.com/google/cel-go v0.24.1
 	github.com/itchyny/gojq v0.12.17
 	github.com/klauspost/compress v1.18.0
+	github.com/pires/go-proxyproto v0.8.0
 	github.com/tetratelabs/wazero v1.9.0
 	github.com/yl2chen/cidranger v1.0.2
+	golang.org/x/crypto v0.37.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -22,22 +24,14 @@ require (
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
-	golang.org/x/crypto v0.33.0 // indirect
-	golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
+	golang.org/x/exp v0.0.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250409194420-de1ac958c67a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 )
 
 // Used by github.com/antlr4-go/antlr v4.13.0 via github.com/google/cel-go
 // Ensure we have no other exp package usages by only proxying the slices functions in that package
-// Newer versions than v0.0.0-20250210185358-939b2ce775ac are not supported by Go 1.22
 replace golang.org/x/exp v0.0.0 => ./utils/exp
-
-// Pin latest versions to support Go 1.22 to prevent a package update from changing them
-// TODO: remove this when Go 1.22+ is supported by other higher users
-replace (
-	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7
-	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7
-	golang.org/x/crypto => golang.org/x/crypto v0.33.0
-)

go.sum

@@ -9,12 +9,12 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
-github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
+github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY=
+github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw=
 github.com/google/cel-go v0.24.1 h1:jsBCtxG8mM5wiUJDSGUqU0K7Mtr3w7Eyv00rw4DiZxI=
 github.com/google/cel-go v0.24.1/go.mod h1:Hdf9TqOaTNSFQA1ybQaRqATVoK7m/zcf7IMhGXP5zI8=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/itchyny/gojq v0.12.17 h1:8av8eGduDb5+rvEdaOO+zQUjA04MS0m3Ps8HiD+fceg=
 github.com/itchyny/gojq v0.12.17/go.mod h1:WBrEMkgAfAGO1LUcGOckBl5O726KPp+OlkKug0I/FEY=
 github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
@@ -23,6 +23,8 @@ github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 h1:Pdirg1gwhEcGjM
 github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43/go.mod h1:ahLMuLCUyDdXqtqGyuwGev7/PGtO7r7ocvdwDuEN/3E=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/pires/go-proxyproto v0.8.0 h1:5unRmEAPbHXHuLjDg01CxJWf91cw3lKHc/0xzKpXEe0=
+github.com/pires/go-proxyproto v0.8.0/go.mod h1:iknsfgnH8EkjrMeMyvfKByp9TiBZCKZM0jx2xmKqnVY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
@@ -42,16 +44,16 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
 github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs=
-golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw=
-google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+google.golang.org/genproto/googleapis/api v0.0.0-20250409194420-de1ac958c67a h1:OQ7sHVzkx6L57dQpzUS4ckfWJ51KDH74XHTDe23xWAs=
+google.golang.org/genproto/googleapis/api v0.0.0-20250409194420-de1ac958c67a/go.mod h1:2R6XrVC8Oc08GlNh8ujEpc7HkLiEZ16QeY7FxIs20ac=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a h1:GIqLhp/cYUkuGuiT+vJk8vhOP86L4+SP5j8yXgeVpvI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
 google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

go.work

@@ -1,6 +1,8 @@
-go 1.22.0
+go 1.24.0
+
+toolchain go1.24.2
 
 use (
 	.
 	utils/exp
-)
+)

go.work.sum

@@ -1,5 +1,6 @@
 github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
@@ -7,20 +8,30 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
 golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
 golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
 golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
 google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
+google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg=
+google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

lib/challenge.go

@@ -3,6 +3,8 @@ package lib
 import (
 	"crypto/sha256"
 	"encoding/binary"
+	"encoding/hex"
+	"errors"
 	"github.com/go-jose/go-jose/v4/jwt"
 	"net"
 	"net/http"
@@ -20,6 +22,18 @@ type ChallengeInformation struct {
 	IssuedAt  *jwt.NumericDate `json:"iat,omitempty"`
 }
 
+func getRequestScheme(r *http.Request) string {
+	if proto := r.Header.Get("X-Forwarded-Proto"); proto == "http" || proto == "https" {
+		return proto
+	}
+
+	if r.TLS != nil {
+		return "https"
+	}
+
+	return "http"
+}
+
 func getRequestAddress(r *http.Request, clientHeader string) net.IP {
 	var ipStr string
 	if clientHeader != "" {
@@ -36,15 +50,49 @@ func getRequestAddress(r *http.Request, clientHeader string) net.IP {
 		// drop port
 		ipStr = strings.Join(parts[:len(parts)-1], ":")
 	}
+	ipStr = strings.Trim(ipStr, "[]")
 	return net.ParseIP(ipStr)
 }
 
-func (state *State) GetChallengeKeyForRequest(challengeName string, until time.Time, r *http.Request) []byte {
+type ChallengeKey []byte
+
+const ChallengeKeySize = sha256.Size
+
+func (k *ChallengeKey) Set(flags ChallengeKeyFlags) {
+	(*k)[0] |= uint8(flags)
+}
+func (k *ChallengeKey) Get(flags ChallengeKeyFlags) ChallengeKeyFlags {
+	return ChallengeKeyFlags((*k)[0] & uint8(flags))
+}
+func (k *ChallengeKey) Unset(flags ChallengeKeyFlags) {
+	(*k)[0] = (*k)[0] & ^(uint8(flags))
+}
+
+type ChallengeKeyFlags uint8
+
+const (
+	ChallengeKeyFlagIsIPv4 = ChallengeKeyFlags(1 << iota)
+)
+
+func ChallengeKeyFromString(s string) (ChallengeKey, error) {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		return nil, err
+	}
+	if len(b) != ChallengeKeySize {
+		return nil, errors.New("invalid challenge key")
+	}
+	return ChallengeKey(b), nil
+}
+
+func (state *State) GetChallengeKeyForRequest(challengeName string, until time.Time, r *http.Request) ChallengeKey {
+	data := RequestDataFromContext(r.Context())
+	address := data.RemoteAddress
 	hasher := sha256.New()
 	hasher.Write([]byte("challenge\x00"))
 	hasher.Write([]byte(challengeName))
 	hasher.Write([]byte{0})
-	hasher.Write(getRequestAddress(r, state.Settings.ClientIpHeader).To16())
+	hasher.Write(address.To16())
 	hasher.Write([]byte{0})
 
 	// specific headers
@@ -52,8 +100,9 @@ func (state *State) GetChallengeKeyForRequest(challengeName string, until time.T
 		"Accept-Language",
 		// General browser information
 		"User-Agent",
-		"Sec-Ch-Ua",
-		"Sec-Ch-Ua-Platform",
+		// TODO: not sent in preload
+		//"Sec-Ch-Ua",
+		//"Sec-Ch-Ua-Platform",
 	} {
 		hasher.Write([]byte(r.Header.Get(k)))
 		hasher.Write([]byte{0})
@@ -64,5 +113,13 @@ func (state *State) GetChallengeKeyForRequest(challengeName string, until time.T
 	hasher.Write(state.publicKey)
 	hasher.Write([]byte{0})
 
-	return hasher.Sum(nil)
+	sum := ChallengeKey(hasher.Sum(nil))
+
+	sum[0] = 0
+
+	if address.To4() != nil {
+		// Is IPv4, mark
+		sum.Set(ChallengeKeyFlagIsIPv4)
+	}
+	return ChallengeKey(sum)
 }

lib/challenge/state.go

@@ -7,6 +7,7 @@ import (
 	"git.gammaspectra.live/git/go-away/utils"
 	"github.com/go-jose/go-jose/v4"
 	"github.com/go-jose/go-jose/v4/jwt"
+	"github.com/google/cel-go/cel"
 	"math/rand/v2"
 	"net/http"
 	"time"
@@ -26,9 +27,10 @@ const (
 type Id int
 
 type Challenge struct {
-	Id   Id
-	Name string
-	Path string
+	Id      Id
+	Program cel.Program
+	Name    string
+	Path    string
 
 	Verify            func(key []byte, result string, r *http.Request) (bool, error)
 	VerifyProbability float64
@@ -86,6 +88,7 @@ type VerifyResult int
 const (
 	VerifyResultNONE = VerifyResult(iota)
 	VerifyResultFAIL
+	VerifyResultSKIP
 
 	// VerifyResultPASS Client just passed this challenge
 	VerifyResultPASS
@@ -95,7 +98,7 @@ const (
 )
 
 func (r VerifyResult) Ok() bool {
-	return r > VerifyResultFAIL
+	return r >= VerifyResultPASS
 }
 
 func (r VerifyResult) String() string {
@@ -104,6 +107,8 @@ func (r VerifyResult) String() string {
 		return "NONE"
 	case VerifyResultFAIL:
 		return "FAIL"
+	case VerifyResultSKIP:
+		return "SKIP"
 	case VerifyResultPASS:
 		return "PASS"
 	case VerifyResultOK:

lib/conditions.go

@@ -0,0 +1,120 @@
+package lib
+
+import (
+	"context"
+	"fmt"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"log/slog"
+	"net"
+	"time"
+)
+
+func (state *State) initConditions() (err error) {
+	state.RulesEnv, err = cel.NewEnv(
+		cel.DefaultUTCTimeZone(true),
+		cel.Variable("remoteAddress", cel.BytesType),
+		cel.Variable("host", cel.StringType),
+		cel.Variable("method", cel.StringType),
+		cel.Variable("userAgent", cel.StringType),
+		cel.Variable("path", cel.StringType),
+		cel.Variable("query", cel.MapType(cel.StringType, cel.StringType)),
+		cel.Variable("fpJA3N", cel.StringType),
+		cel.Variable("fpJA4", cel.StringType),
+		// http.Header
+		cel.Variable("headers", cel.MapType(cel.StringType, cel.StringType)),
+		//TODO: dynamic type?
+		cel.Function("inDNSBL",
+			cel.Overload("inDNSBL_ip",
+				[]*cel.Type{cel.AnyType},
+				cel.BoolType,
+				cel.UnaryBinding(func(val ref.Val) ref.Val {
+					if state.Settings.DNSBL == nil {
+						return types.Bool(false)
+					}
+
+					var ip net.IP
+					switch v := val.Value().(type) {
+					case []byte:
+						ip = v
+					case net.IP:
+						ip = v
+					case string:
+						ip = net.ParseIP(v)
+					}
+
+					if ip == nil {
+						panic(fmt.Errorf("invalid ip %v", val.Value()))
+					}
+
+					var key [net.IPv6len]byte
+					copy(key[:], ip.To16())
+
+					result, ok := state.DecayMap.Get(key)
+					if ok {
+						return types.Bool(result.Bad())
+					}
+
+					ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+					defer cancel()
+					result, err := state.Settings.DNSBL.Lookup(ctx, ip)
+					if err != nil {
+						slog.Debug("dnsbl lookup failed", "address", ip.String(), "result", result, "err", err)
+					} else {
+						slog.Debug("dnsbl lookup", "address", ip.String(), "result", result)
+					}
+					//TODO: configure decay
+					state.DecayMap.Set(key, result, time.Hour)
+
+					return types.Bool(result.Bad())
+				}),
+			),
+		),
+		cel.Function("inNetwork",
+			cel.Overload("inNetwork_string_ip",
+				[]*cel.Type{cel.StringType, cel.AnyType},
+				cel.BoolType,
+				cel.BinaryBinding(func(lhs ref.Val, rhs ref.Val) ref.Val {
+					var ip net.IP
+					switch v := rhs.Value().(type) {
+					case []byte:
+						ip = v
+					case net.IP:
+						ip = v
+					case string:
+						ip = net.ParseIP(v)
+					}
+
+					if ip == nil {
+						panic(fmt.Errorf("invalid ip %v", rhs.Value()))
+					}
+
+					val, ok := lhs.Value().(string)
+					if !ok {
+						panic(fmt.Errorf("invalid value %v", lhs.Value()))
+					}
+
+					network, ok := state.Networks[val]
+					if !ok {
+						_, ipNet, err := net.ParseCIDR(val)
+						if err != nil {
+							panic("network not found")
+						}
+						return types.Bool(ipNet.Contains(ip))
+					} else {
+						ok, err := network.Contains(ip)
+						if err != nil {
+							panic(err)
+						}
+						return types.Bool(ok)
+					}
+				}),
+			),
+		),
+	)
+	if err != nil {
+		return err
+	}
+	return nil
+}

lib/http.go

@@ -18,7 +18,9 @@ import (
 	"io"
 	"log/slog"
 	"maps"
+	"net"
 	"net/http"
+	"net/http/pprof"
 	"path"
 	"path/filepath"
 	"strconv"
@@ -129,19 +131,30 @@ func (state *State) addTiming(w http.ResponseWriter, name, desc string, duration
 	}
 }
 
-func GetLoggerForRequest(r *http.Request, clientHeader string) *slog.Logger {
-	return slog.With(
-		"request_id", r.Header.Get("X-Away-Id"),
-		"remote_address", getRequestAddress(r, clientHeader),
+func GetLoggerForRequest(r *http.Request) *slog.Logger {
+	data := RequestDataFromContext(r.Context())
+	args := []any{
+		"request_id", hex.EncodeToString(data.Id[:]),
+		"remote_address", data.RemoteAddress.String(),
 		"user_agent", r.UserAgent(),
 		"host", r.Host,
 		"path", r.URL.Path,
 		"query", r.URL.RawQuery,
-	)
+	}
+
+	if fp := utils.GetTLSFingerprint(r); fp != nil {
+		if ja3n := fp.JA3N(); ja3n != nil {
+			args = append(args, "ja3n", ja3n.String())
+		}
+		if ja4 := fp.JA4(); ja4 != nil {
+			args = append(args, "ja4", ja4.String())
+		}
+	}
+	return slog.With(args...)
 }
 
 func (state *State) logger(r *http.Request) *slog.Logger {
-	return GetLoggerForRequest(r, state.Settings.ClientIpHeader)
+	return GetLoggerForRequest(r)
 }
 
 func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
@@ -159,29 +172,6 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 
 	start := time.Now()
 
-	//TODO better matcher! combo ast?
-	env := map[string]any{
-		"host":          host,
-		"method":        r.Method,
-		"remoteAddress": getRequestAddress(r, state.Settings.ClientIpHeader),
-		"userAgent":     r.UserAgent(),
-		"path":          r.URL.Path,
-		"query": func() map[string]string {
-			result := make(map[string]string)
-			for k, v := range r.URL.Query() {
-				result[k] = strings.Join(v, ",")
-			}
-			return result
-		}(),
-		"headers": func() map[string]string {
-			result := make(map[string]string)
-			for k, v := range r.Header {
-				result[k] = strings.Join(v, ",")
-			}
-			return result
-		}(),
-	}
-
 	state.addTiming(w, "rule-env", "Setup the rule environment", time.Since(start))
 
 	var (
@@ -211,7 +201,7 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 		start = time.Now()
-		out, _, err := rule.Program.Eval(env)
+		out, _, err := rule.Program.Eval(data.ProgramEnv)
 		ruleEvalDuration += time.Since(start)
 
 		if err != nil {
@@ -230,7 +220,6 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 					serve()
 					return
 				case policy.RuleActionCHALLENGE, policy.RuleActionCHECK:
-
 					for _, challengeId := range rule.Challenges {
 						if result := data.Challenges[challengeId]; !result.Ok() {
 							continue
@@ -249,6 +238,11 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 
 					// none matched, issue first challenge in priority
 					for _, challengeId := range rule.Challenges {
+						result := data.Challenges[challengeId]
+						if result.Ok() || result == challenge.VerifyResultSKIP {
+							// skip already ok'd challenges for some reason, and also skip skipped challenges
+							continue
+						}
 						c := state.Challenges[challengeId]
 						if c.ServeChallenge != nil {
 							result := c.ServeChallenge(w, r, state.GetChallengeKeyForRequest(c.Name, data.Expires, r), data.Expires)
@@ -264,7 +258,10 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 								}
 								state.logger(r).Warn("challenge passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
 
-								data.Challenges[c.Id] = challenge.VerifyResultOK
+								// set pass if caller didn't set one
+								if !data.Challenges[c.Id].Ok() {
+									data.Challenges[c.Id] = challenge.VerifyResultPASS
+								}
 
 								// we pass the challenge early!
 								lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
@@ -310,24 +307,26 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 
 					if reader != nil {
 						defer reader.Close()
-					}
 
-					w.Header().Set("Cache-Control", "max-age=0, private, must-revalidate, no-transform")
-					w.Header().Set("Vary", "Accept-Encoding")
-					w.Header().Set("Content-Type", mime)
-					w.Header().Set("X-Content-Type-Options", "nosniff")
-					if encoding != "" {
-						w.Header().Set("Content-Encoding", encoding)
+						w.Header().Set("Cache-Control", "max-age=0, private, must-revalidate, no-transform")
+						w.Header().Set("Vary", "Accept-Encoding")
+						w.Header().Set("Content-Type", mime)
+						w.Header().Set("X-Content-Type-Options", "nosniff")
+						if encoding != "" {
+							w.Header().Set("Content-Encoding", encoding)
+						}
+						w.WriteHeader(http.StatusOK)
+						if flusher, ok := w.(http.Flusher); ok {
+							// trigger chunked encoding
+							flusher.Flush()
+						}
+						if r != nil {
+							_, _ = io.Copy(w, reader)
+						}
+						return
+					} else {
+						http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
 					}
-					w.WriteHeader(http.StatusOK)
-					if flusher, ok := w.(http.Flusher); ok {
-						// trigger chunked encoding
-						flusher.Flush()
-					}
-					if r != nil {
-						_, _ = io.Copy(w, reader)
-					}
-					return
 				}
 			}
 		}
@@ -343,6 +342,13 @@ func (state *State) setupRoutes() error {
 
 	state.Mux.HandleFunc("/", state.handleRequest)
 
+	if state.Settings.Debug {
+		http.HandleFunc(state.UrlPath+"/debug/pprof/", pprof.Index)
+		http.HandleFunc(state.UrlPath+"/debug/pprof/profile", pprof.Profile)
+		http.HandleFunc(state.UrlPath+"/debug/pprof/symbol", pprof.Symbol)
+		http.HandleFunc(state.UrlPath+"/debug/pprof/trace", pprof.Trace)
+	}
+
 	state.Mux.Handle("GET "+state.UrlPath+"/assets/", http.StripPrefix(state.UrlPath, gzipped.FileServer(gzipped.FS(embed.AssetsFs))))
 
 	for _, c := range state.Challenges {
@@ -420,12 +426,52 @@ func (state *State) setupRoutes() error {
 }
 
 func (state *State) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+
 	var data RequestData
 	// generate random id, todo: is this fast?
 	_, _ = rand.Read(data.Id[:])
+	data.RemoteAddress = getRequestAddress(r, state.Settings.ClientIpHeader)
 	data.Challenges = make(map[challenge.Id]challenge.VerifyResult, len(state.Challenges))
 	data.Expires = time.Now().UTC().Add(DefaultValidity).Round(DefaultValidity)
 
+	var ja3n, ja4 string
+	if fp := utils.GetTLSFingerprint(r); fp != nil {
+		if ja3nPtr := fp.JA3N(); ja3nPtr != nil {
+			ja3n = ja3nPtr.String()
+			r.Header.Set("X-TLS-Fingerprint-JA3N", ja3n)
+		}
+		if ja4Ptr := fp.JA4(); ja4Ptr != nil {
+			ja4 = ja4Ptr.String()
+			r.Header.Set("X-TLS-Fingerprint-JA4", ja4)
+		}
+	}
+
+	data.ProgramEnv = map[string]any{
+		"host":          r.Host,
+		"method":        r.Method,
+		"remoteAddress": data.RemoteAddress,
+		"userAgent":     r.UserAgent(),
+		"path":          r.URL.Path,
+		"fpJA3N":        ja3n,
+		"fpJA4":         ja4,
+		"query": func() map[string]string {
+			result := make(map[string]string)
+			for k, v := range r.URL.Query() {
+				result[k] = strings.Join(v, ",")
+			}
+			return result
+		}(),
+		"headers": func() map[string]string {
+			result := make(map[string]string)
+			for k, v := range r.Header {
+				result[k] = strings.Join(v, ",")
+			}
+			return result
+		}(),
+	}
+
+	r = r.WithContext(context.WithValue(r.Context(), "_goaway_data", &data))
+
 	for _, c := range state.Challenges {
 		key := state.GetChallengeKeyForRequest(c.Name, data.Expires, r)
 		result, err := c.VerifyChallengeToken(state.publicKey, key, r)
@@ -433,12 +479,34 @@ func (state *State) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// clear invalid cookie
 			utils.ClearCookie(utils.CookiePrefix+c.Name, w)
 		}
+
+		// prevent the challenge if not solved
+		if !result.Ok() && c.Program != nil {
+			out, _, err := c.Program.Eval(data.ProgramEnv)
+			// verify eligibility
+			if err != nil {
+				state.logger(r).Error(err.Error(), "challenge", c.Name)
+			} else if out != nil && out.Type() == types.BoolType {
+				if out.Equal(types.True) != types.True {
+					// skip challenge match!
+					result = challenge.VerifyResultSKIP
+					continue
+				}
+			}
+		}
 		data.Challenges[c.Id] = result
 	}
 
 	r.Header.Set("X-Away-Id", hex.EncodeToString(data.Id[:]))
+	if state.Settings.BackendIpHeader != "" {
+		r.Header.Del(state.Settings.ClientIpHeader)
+		r.Header.Set(state.Settings.BackendIpHeader, data.RemoteAddress.String())
+	}
+	w.Header().Add("Via", fmt.Sprintf("%s %s", r.Proto, "go-away"))
 
-	r = r.WithContext(context.WithValue(r.Context(), "_goaway_data", &data))
+	// send these to client so we consistently get the headers
+	//w.Header().Set("Accept-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
+	//w.Header().Set("Critical-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
 
 	state.Mux.ServeHTTP(w, r)
 }
@@ -448,9 +516,11 @@ func RequestDataFromContext(ctx context.Context) *RequestData {
 }
 
 type RequestData struct {
-	Id         [16]byte
-	Expires    time.Time
-	Challenges map[challenge.Id]challenge.VerifyResult
+	Id            [16]byte
+	ProgramEnv    map[string]any
+	Expires       time.Time
+	Challenges    map[challenge.Id]challenge.VerifyResult
+	RemoteAddress net.IP
 }
 
 func (d *RequestData) HasValidChallenge(id challenge.Id) bool {

lib/policy/challenge.go

@@ -1,9 +1,10 @@
 package policy
 
 type Challenge struct {
-	Mode  string  `yaml:"mode"`
-	Asset *string `yaml:"asset,omitempty"`
-	Url   *string `yaml:"url,omitempty"`
+	Conditions []string `yaml:"conditions"`
+	Mode       string   `yaml:"mode"`
+	Asset      *string  `yaml:"asset,omitempty"`
+	Url        *string  `yaml:"url,omitempty"`
 
 	Parameters map[string]string `json:"parameters,omitempty"`
 	Runtime    struct {

lib/state.go

@@ -20,8 +20,6 @@ import (
 	"git.gammaspectra.live/git/go-away/utils"
 	"git.gammaspectra.live/git/go-away/utils/inline"
 	"github.com/google/cel-go/cel"
-	"github.com/google/cel-go/common/types"
-	"github.com/google/cel-go/common/types/ref"
 	"github.com/tetratelabs/wazero/api"
 	"github.com/yl2chen/cidranger"
 	"html/template"
@@ -36,6 +34,8 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -59,8 +59,40 @@ type State struct {
 	privateKey ed25519.PrivateKey
 
 	Poison map[string][]byte
+
+	ChallengeSolve sync.Map
+
+	DecayMap *utils.DecayMap[[net.IPv6len]byte, utils.DNSBLResponse]
+
+	close chan struct{}
 }
 
+func (state *State) AwaitChallenge(key []byte, ctx context.Context) challenge.VerifyResult {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var result atomic.Int64
+
+	state.ChallengeSolve.Store(string(key), ChallengeCallback(func(receivedResult challenge.VerifyResult) {
+		result.Store(int64(receivedResult))
+		cancel()
+	}))
+
+	<-ctx.Done()
+
+	return challenge.VerifyResult(result.Load())
+}
+
+func (state *State) SolveChallenge(key []byte, result challenge.VerifyResult) {
+	if f, ok := state.ChallengeSolve.LoadAndDelete(string(key)); ok && f != nil {
+		if cb, ok := f.(ChallengeCallback); ok {
+			cb(result)
+		}
+	}
+}
+
+type ChallengeCallback func(result challenge.VerifyResult)
+
 type RuleState struct {
 	Name string
 	Hash string
@@ -80,10 +112,13 @@ type StateSettings struct {
 	ChallengeTemplate      string
 	ChallengeTemplateTheme string
 	ClientIpHeader         string
+	BackendIpHeader        string
+	DNSBL                  *utils.DNSBL
 }
 
-func NewState(p policy.Policy, settings StateSettings) (state *State, err error) {
-	state = new(State)
+func NewState(p policy.Policy, settings StateSettings) (handler http.Handler, err error) {
+	state := new(State)
+	state.close = make(chan struct{})
 	state.Settings = settings
 	state.Client = &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
@@ -92,6 +127,10 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 	}
 	state.UrlPath = "/.well-known/." + state.Settings.PackageName
 
+	if state.Settings.DNSBL != nil {
+		state.DecayMap = utils.NewDecayMap[[net.IPv6len]byte, utils.DNSBLResponse]()
+	}
+
 	// set a reasonable configuration for default http proxy if there is none
 	for _, backend := range state.Settings.Backends {
 		if proxy, ok := backend.(*httputil.ReverseProxy); ok {
@@ -167,13 +206,57 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 
 	state.Wasm = wasm.NewRunner(true)
 
+	err = state.initConditions()
+	if err != nil {
+		return nil, err
+	}
+
+	var replacements []string
+	for k, entries := range p.Conditions {
+		ast, err := condition.FromStrings(state.RulesEnv, condition.OperatorOr, entries...)
+		if err != nil {
+			return nil, fmt.Errorf("conditions %s: error compiling conditions: %v", k, err)
+		}
+
+		cond, err := cel.AstToString(ast)
+		if err != nil {
+			return nil, fmt.Errorf("conditions %s: error printing condition: %v", k, err)
+		}
+
+		replacements = append(replacements, fmt.Sprintf("($%s)", k))
+		replacements = append(replacements, "("+cond+")")
+	}
+	conditionReplacer := strings.NewReplacer(replacements...)
+
 	state.Challenges = make(map[challenge.Id]challenge.Challenge)
 
 	idCounter := challenge.Id(1)
 
+	//TODO: move this to self-contained challenge files
 	for challengeName, p := range p.Challenges {
+
+		// allow nesting
+		var conditions []string
+		for _, cond := range p.Conditions {
+			cond = conditionReplacer.Replace(cond)
+			conditions = append(conditions, cond)
+		}
+
+		var program cel.Program
+		if len(conditions) > 0 {
+			ast, err := condition.FromStrings(state.RulesEnv, condition.OperatorOr, conditions...)
+			if err != nil {
+				return nil, fmt.Errorf("challenge %s: error compiling conditions: %v", challengeName, err)
+			}
+			program, err = state.RulesEnv.Program(ast)
+			if err != nil {
+				return nil, fmt.Errorf("challenge %s: error compiling program: %v", challengeName, err)
+			}
+		}
+
 		c := challenge.Challenge{
 			Id:                idCounter,
+			Program:           program,
 			Name:              challengeName,
 			Path:              fmt.Sprintf("%s/challenge/%s", state.UrlPath, challengeName),
 			VerifyProbability: p.Runtime.Probability,
@@ -240,6 +323,13 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 			}
 
 			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
+
+				data := RequestDataFromContext(r.Context())
+
+				if result := data.Challenges[c.Id]; result.Ok() {
+					return challenge.ResultPass
+				}
+
 				var cookieValue string
 				if expectedCookie != "" {
 					if cookie, err := r.Cookie(expectedCookie); err != nil || cookie == nil {
@@ -266,6 +356,9 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 				if response.StatusCode != httpCode {
 					utils.ClearCookie(utils.CookiePrefix+c.Name, w)
 					// continue other challenges!
+
+					//TODO: negatively cache failure
+
 					return challenge.ResultContinue
 				} else {
 					// bind hash of cookie contents
@@ -275,7 +368,6 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 					sum.Write(key)
 					sum.Write([]byte{0})
 					sum.Write(state.publicKey)
-
 					token, err := c.IssueChallengeToken(state.privateKey, key, sum.Sum(nil), expiry)
 					if err != nil {
 						utils.ClearCookie(utils.CookiePrefix+c.Name, w)
@@ -283,6 +375,8 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 						utils.SetCookie(utils.CookiePrefix+challengeName, token, expiry, w)
 					}
 
+					data.Challenges[c.Id] = challenge.VerifyResultPASS
+
 					// we passed it!
 					return challenge.ResultPass
 				}
@@ -290,6 +384,12 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 
 		case "cookie":
 			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
+				if chall := r.URL.Query().Get("__goaway_challenge"); chall == challengeName {
+					state.logger(r).Warn("challenge failed", "challenge", c.Name)
+					utils.ClearCookie(utils.CookiePrefix+c.Name, w)
+					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", c.Name), "")
+					return challenge.ResultStop
+				}
 
 				token, err := c.IssueChallengeToken(state.privateKey, key, nil, expiry)
 				if err != nil {
@@ -297,9 +397,14 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 				} else {
 					utils.SetCookie(utils.CookiePrefix+challengeName, token, expiry, w)
 				}
+
 				// self redirect!
-				//TODO: add redirect loop detect parameter
-				http.Redirect(w, r, r.URL.String(), http.StatusTemporaryRedirect)
+				uri, err := url.ParseRequestURI(r.URL.String())
+				values := uri.Query()
+				values.Set("__goaway_challenge", challengeName)
+				uri.RawQuery = values.Encode()
+
+				http.Redirect(w, r, uri.String(), http.StatusTemporaryRedirect)
 				return challenge.ResultStop
 			}
 		case "meta-refresh":
@@ -341,6 +446,54 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 
 				return challenge.ResultStop
 			}
+		case "preload-link":
+			deadline, _ := time.ParseDuration(p.Parameters["preload-early-hint-deadline"])
+			if deadline == 0 {
+				deadline = time.Second * 3
+			}
+
+			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
+				// this only works on HTTP/2 and HTTP/3
+
+				if r.ProtoMajor < 2 {
+					// this can happen if we are an upgraded request from HTTP/1.1 to HTTP/2 in H2C
+					if _, ok := w.(http.Pusher); !ok {
+						return challenge.ResultContinue
+					}
+				}
+
+				data := RequestDataFromContext(r.Context())
+				redirectUri := new(url.URL)
+				redirectUri.Scheme = getRequestScheme(r)
+				redirectUri.Host = r.Host
+				redirectUri.Path = c.Path + "/verify-challenge"
+
+				values := make(url.Values)
+				values.Set("result", hex.EncodeToString(key))
+				values.Set("requestId", r.Header.Get("X-Away-Id"))
+
+				redirectUri.RawQuery = values.Encode()
+
+				w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"preload\"; as=\"style\"; fetchpriority=high", redirectUri.String()))
+				defer func() {
+					// remove old header so it won't show on response!
+					w.Header().Del("Link")
+				}()
+				w.WriteHeader(http.StatusEarlyHints)
+
+				ctx, cancel := context.WithTimeout(r.Context(), deadline)
+				defer cancel()
+				if result := state.AwaitChallenge(key, ctx); result.Ok() {
+					data.Challenges[c.Id] = challenge.VerifyResultPASS
+
+					// this should serve!
+					return challenge.ResultPass
+				}
+
+				data.Challenges[c.Id] = challenge.VerifyResultFAIL
+				// we failed, continue
+				return challenge.ResultContinue
+			}
 		case "resource-load":
 			c.ServeChallenge = func(w http.ResponseWriter, r *http.Request, key []byte, expiry time.Time) challenge.Result {
 				redirectUri := new(url.URL)
@@ -437,7 +590,7 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 
 				redirect, err := utils.EnsureNoOpenRedirect(r.FormValue("redirect"))
 				if err != nil {
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, "")
+					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusBadRequest, err, "")
 					return
 				}
 
@@ -456,24 +609,37 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 					if ok, err := c.Verify(key, result, r); err != nil {
 						return err
 					} else if !ok {
+						utils.ClearCookie(utils.CookiePrefix+challengeName, w)
+						data.Challenges[c.Id] = challenge.VerifyResultFAIL
+						state.SolveChallenge(key, challenge.VerifyResultFAIL)
 						state.logger(r).Warn("challenge failed", "challenge", challengeName, "redirect", redirect)
-						utils.ClearCookie(utils.CookiePrefix+challengeName, w)
-						_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", challengeName), redirect)
-						return nil
-					}
 
-					state.logger(r).Warn("challenge passed", "challenge", challengeName, "redirect", redirect)
+						// catch happy eyeballs IPv4 -> IPv6 migration, re-direct to try again
+						if resultKey, err := ChallengeKeyFromString(result); err == nil && resultKey.Get(ChallengeKeyFlagIsIPv4) > 0 && key.Get(ChallengeKeyFlagIsIPv4) == 0 {
 
-					token, err := c.IssueChallengeToken(state.privateKey, key, []byte(result), data.Expires)
-					if err != nil {
-						utils.ClearCookie(utils.CookiePrefix+challengeName, w)
+						} else {
+							_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", challengeName), redirect)
+							return nil
+						}
 					} else {
-						utils.SetCookie(utils.CookiePrefix+challengeName, token, data.Expires, w)
+						state.logger(r).Warn("challenge passed", "challenge", challengeName, "redirect", redirect)
+
+						token, err := c.IssueChallengeToken(state.privateKey, key, []byte(result), data.Expires)
+						if err != nil {
+							utils.ClearCookie(utils.CookiePrefix+challengeName, w)
+						} else {
+							utils.SetCookie(utils.CookiePrefix+challengeName, token, data.Expires, w)
+						}
+						data.Challenges[c.Id] = challenge.VerifyResultPASS
+						state.SolveChallenge(key, challenge.VerifyResultPASS)
 					}
-					data.Challenges[c.Id] = challenge.VerifyResultPASS
 
 					switch httpCode {
 					case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+						if redirect == "" {
+							_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusBadRequest, errors.New("no redirect found"), "")
+							return nil
+						}
 						http.Redirect(w, r, redirect, httpCode)
 					default:
 						w.Header().Set("Content-Type", mimeType)
@@ -566,80 +732,6 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 		state.Challenges[c.Id] = c
 	}
 
-	state.RulesEnv, err = cel.NewEnv(
-		cel.DefaultUTCTimeZone(true),
-		cel.Variable("remoteAddress", cel.BytesType),
-		cel.Variable("host", cel.StringType),
-		cel.Variable("method", cel.StringType),
-		cel.Variable("userAgent", cel.StringType),
-		cel.Variable("path", cel.StringType),
-		cel.Variable("query", cel.MapType(cel.StringType, cel.StringType)),
-		// http.Header
-		cel.Variable("headers", cel.MapType(cel.StringType, cel.StringType)),
-		//TODO: dynamic type?
-		cel.Function("inNetwork",
-			cel.Overload("inNetwork_string_ip",
-				[]*cel.Type{cel.StringType, cel.AnyType},
-				cel.BoolType,
-				cel.BinaryBinding(func(lhs ref.Val, rhs ref.Val) ref.Val {
-					var ip net.IP
-					switch v := rhs.Value().(type) {
-					case []byte:
-						ip = v
-					case net.IP:
-						ip = v
-					case string:
-						ip = net.ParseIP(v)
-					}
-
-					if ip == nil {
-						panic(fmt.Errorf("invalid ip %v", rhs.Value()))
-					}
-
-					val, ok := lhs.Value().(string)
-					if !ok {
-						panic(fmt.Errorf("invalid value %v", lhs.Value()))
-					}
-
-					network, ok := state.Networks[val]
-					if !ok {
-						_, ipNet, err := net.ParseCIDR(val)
-						if err != nil {
-							panic("network not found")
-						}
-						return types.Bool(ipNet.Contains(ip))
-					} else {
-						ok, err := network.Contains(ip)
-						if err != nil {
-							panic(err)
-						}
-						return types.Bool(ok)
-					}
-				}),
-			),
-		),
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	var replacements []string
-	for k, entries := range p.Conditions {
-		ast, err := condition.FromStrings(state.RulesEnv, condition.OperatorOr, entries...)
-		if err != nil {
-			return nil, fmt.Errorf("conditions %s: error compiling conditions: %v", k, err)
-		}
-
-		cond, err := cel.AstToString(ast)
-		if err != nil {
-			return nil, fmt.Errorf("conditions %s: error printing condition: %v", k, err)
-		}
-
-		replacements = append(replacements, fmt.Sprintf("($%s)", k))
-		replacements = append(replacements, "("+cond+")")
-	}
-	conditionReplacer := strings.NewReplacer(replacements...)
-
 	for _, rule := range p.Rules {
 		hasher := sha256.New()
 		hasher.Write([]byte(rule.Name))
@@ -701,6 +793,20 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error)
 		return nil, err
 	}
 
+	if state.DecayMap != nil {
+		go func() {
+			ticker := time.NewTicker(17 * time.Minute)
+			for {
+				select {
+				case <-ticker.C:
+					state.DecayMap.Decay()
+				case <-state.close:
+					return
+				}
+			}
+		}()
+	}
+
 	return state, nil
 }
 

utils/decaymap.go

@@ -0,0 +1,73 @@
+package utils
+
+import (
+	"sync"
+	"time"
+)
+
+func zilch[T any]() T {
+	var zero T
+	return zero
+}
+
+type DecayMap[K, V comparable] struct {
+	data map[K]DecayMapEntry[V]
+	lock sync.RWMutex
+}
+
+type DecayMapEntry[V comparable] struct {
+	Value  V
+	expiry time.Time
+}
+
+func NewDecayMap[K, V comparable]() *DecayMap[K, V] {
+	return &DecayMap[K, V]{
+		data: make(map[K]DecayMapEntry[V]),
+	}
+}
+
+func (m *DecayMap[K, V]) Get(key K) (V, bool) {
+	m.lock.RLock()
+	value, ok := m.data[key]
+	m.lock.RUnlock()
+
+	if !ok {
+		return zilch[V](), false
+	}
+
+	if time.Now().After(value.expiry) {
+		m.lock.Lock()
+		// Since previously reading m.data[key], the value may have been updated.
+		// Delete the entry only if the expiry time is still the same.
+		if m.data[key].expiry == value.expiry {
+			delete(m.data, key)
+		}
+		m.lock.Unlock()
+
+		return zilch[V](), false
+	}
+
+	return value.Value, true
+}
+
+func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	m.data[key] = DecayMapEntry[V]{
+		Value:  value,
+		expiry: time.Now().Add(ttl),
+	}
+}
+
+func (m *DecayMap[K, V]) Decay() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	now := time.Now()
+	for key, entry := range m.data {
+		if now.After(entry.expiry) {
+			delete(m.data, key)
+		}
+	}
+}

utils/dnsbl.go

@@ -0,0 +1,75 @@
+package utils
+
+import (
+	"context"
+	"net"
+	"strconv"
+)
+
+type DNSBL struct {
+	target   string
+	resolver *net.Resolver
+}
+
+func NewDNSBL(target string, resolver *net.Resolver) *DNSBL {
+	if resolver == nil {
+		resolver = net.DefaultResolver
+	}
+	return &DNSBL{
+		target:   target,
+		resolver: resolver,
+	}
+}
+
+var nibbleTable = [16]byte{
+	'0', '1', '2', '3',
+	'4', '5', '6', '7',
+	'8', '9', 'a', 'b',
+	'c', 'd', 'e', 'f',
+}
+
+type DNSBLResponse uint8
+
+func (r DNSBLResponse) Bad() bool {
+	return r != ResponseGood && r != ResponseUnknown
+}
+
+const (
+	ResponseGood    = DNSBLResponse(0)
+	ResponseUnknown = DNSBLResponse(255)
+)
+
+func (bl DNSBL) Lookup(ctx context.Context, ip net.IP) (DNSBLResponse, error) {
+	var target []byte
+	if ip4 := ip.To4(); ip4 != nil {
+		// max length preallocate
+		target = make([]byte, 0, len(bl.target)+1+len(ip4)*4)
+
+		for i := len(ip4) - 1; i >= 0; i-- {
+			target = strconv.AppendUint(target, uint64(ip4[i]), 10)
+			target = append(target, '.')
+		}
+	} else {
+		// IPv6
+		// max length preallocate
+		target = make([]byte, 0, len(bl.target)+1+len(ip)*4)
+
+		for i := len(ip) - 1; i >= 0; i-- {
+			target = append(target, nibbleTable[ip[i]&0xf], '.', ip[i]>>4, '.')
+		}
+	}
+
+	target = append(target, bl.target...)
+
+	ips, err := bl.resolver.LookupIP(ctx, "ip4", string(target))
+	if err != nil {
+		return ResponseUnknown, err
+	}
+
+	for _, ip := range ips {
+		ip4 := ip.To4()
+		return DNSBLResponse(ip4[len(ip4)-1]), nil
+	}
+
+	return ResponseUnknown, nil
+}

utils/exp/go.mod

@@ -1,5 +1,5 @@
 module git.gammaspectra.live/git/go-away/utils/exp
 
-go 1.22.0
+go 1.24.0
 
-toolchain go1.22.12
+toolchain go1.24.2

utils/fingerprint.go

@@ -0,0 +1,339 @@
+package utils
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/http"
+	"slices"
+	"strconv"
+	"strings"
+	"sync/atomic"
+)
+
+func applyTLSFingerprinter(server *http.Server) {
+	server.TLSConfig = server.TLSConfig.Clone()
+
+	getCertificate := server.TLSConfig.GetCertificate
+	if getCertificate == nil {
+		server.TLSConfig.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			ja3n, ja4 := buildTLSFingerprint(clientHello)
+			ptr := clientHello.Context().Value(tlsFingerprintKey{})
+			if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
+				fpPtr.ja3n.Store(&ja3n)
+				fpPtr.ja4.Store(&ja4)
+			}
+
+			return nil, nil
+		}
+	} else {
+		server.TLSConfig.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			ja3n, ja4 := buildTLSFingerprint(clientHello)
+			ptr := clientHello.Context().Value(tlsFingerprintKey{})
+			if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
+				fpPtr.ja3n.Store(&ja3n)
+				fpPtr.ja4.Store(&ja4)
+			}
+
+			return getCertificate(clientHello)
+		}
+	}
+	server.ConnContext = func(ctx context.Context, c net.Conn) context.Context {
+		return context.WithValue(ctx, tlsFingerprintKey{}, &TLSFingerprint{})
+	}
+}
+
+type tlsFingerprintKey struct{}
+type TLSFingerprint struct {
+	ja3n atomic.Pointer[TLSFingerprintJA3N]
+	ja4  atomic.Pointer[TLSFingerprintJA4]
+}
+
+type TLSFingerprintJA3N [md5.Size]byte
+
+func (f TLSFingerprintJA3N) String() string {
+	return hex.EncodeToString(f[:])
+}
+
+type TLSFingerprintJA4 struct {
+	A [10]byte
+	B [6]byte
+	C [6]byte
+}
+
+func (f TLSFingerprintJA4) String() string {
+	return strings.Join([]string{
+		string(f.A[:]),
+		hex.EncodeToString(f.B[:]),
+		hex.EncodeToString(f.C[:]),
+	}, "_")
+}
+
+func (f *TLSFingerprint) JA3N() *TLSFingerprintJA3N {
+	return f.ja3n.Load()
+}
+
+func (f *TLSFingerprint) JA4() *TLSFingerprintJA4 {
+	return f.ja4.Load()
+}
+
+const greaseMask = 0x0F0F
+const greaseValue = 0x0a0a
+
+// TLS extension numbers
+const (
+	extensionServerName              uint16 = 0
+	extensionStatusRequest           uint16 = 5
+	extensionSupportedCurves         uint16 = 10 // supported_groups in TLS 1.3, see RFC 8446, Section 4.2.7
+	extensionSupportedPoints         uint16 = 11
+	extensionSignatureAlgorithms     uint16 = 13
+	extensionALPN                    uint16 = 16
+	extensionSCT                     uint16 = 18
+	extensionExtendedMasterSecret    uint16 = 23
+	extensionSessionTicket           uint16 = 35
+	extensionPreSharedKey            uint16 = 41
+	extensionEarlyData               uint16 = 42
+	extensionSupportedVersions       uint16 = 43
+	extensionCookie                  uint16 = 44
+	extensionPSKModes                uint16 = 45
+	extensionCertificateAuthorities  uint16 = 47
+	extensionSignatureAlgorithmsCert uint16 = 50
+	extensionKeyShare                uint16 = 51
+	extensionQUICTransportParameters uint16 = 57
+	extensionRenegotiationInfo       uint16 = 0xff01
+	extensionECHOuterExtensions      uint16 = 0xfd00
+	extensionEncryptedClientHello    uint16 = 0xfe0d
+)
+
+func tlsFingerprintJA3(hello *tls.ClientHelloInfo, sortExtensions bool) []byte {
+	buf := make([]byte, 0, 256)
+
+	{
+		var sslVersion uint16
+		var hasGrease bool
+		for _, v := range hello.SupportedVersions {
+			if v&greaseMask != greaseValue {
+				if v > sslVersion {
+					sslVersion = v
+				}
+			} else {
+				hasGrease = true
+			}
+		}
+
+		// maximum TLS 1.2 as specified on JA3, as TLS 1.3 is put in SupportedVersions
+		if slices.Contains(hello.Extensions, extensionSupportedVersions) && hasGrease && sslVersion > tls.VersionTLS12 {
+			sslVersion = tls.VersionTLS12
+		}
+
+		buf = strconv.AppendUint(buf, uint64(sslVersion), 10)
+		buf = append(buf, ',')
+	}
+
+	n := 0
+	for _, cipher := range hello.CipherSuites {
+		//if !slices.Contains(greaseValues[:], cipher) {
+		if cipher&greaseMask != greaseValue {
+			buf = strconv.AppendUint(buf, uint64(cipher), 10)
+			buf = append(buf, '-')
+			n = 1
+		}
+	}
+
+	buf = buf[:len(buf)-n]
+	buf = append(buf, ',')
+	n = 0
+
+	extensions := hello.Extensions
+	if sortExtensions {
+		extensions = slices.Clone(extensions)
+		slices.Sort(extensions)
+	}
+
+	for _, extension := range extensions {
+		if extension&greaseMask != greaseValue {
+			buf = strconv.AppendUint(buf, uint64(extension), 10)
+			buf = append(buf, '-')
+			n = 1
+		}
+	}
+
+	buf = buf[:len(buf)-n]
+	buf = append(buf, ',')
+	n = 0
+
+	for _, curve := range hello.SupportedCurves {
+		if curve&greaseMask != greaseValue {
+			buf = strconv.AppendUint(buf, uint64(curve), 10)
+			buf = append(buf, '-')
+			n = 1
+		}
+	}
+
+	buf = buf[:len(buf)-n]
+	buf = append(buf, ',')
+	n = 0
+
+	for _, point := range hello.SupportedPoints {
+		buf = strconv.AppendUint(buf, uint64(point), 10)
+		buf = append(buf, '-')
+		n = 1
+	}
+
+	buf = buf[:len(buf)-n]
+
+	sum := md5.Sum(buf)
+	return sum[:]
+}
+
+func tlsFingerprintJA4(hello *tls.ClientHelloInfo) (ja4 TLSFingerprintJA4) {
+	buf := make([]byte, 0, 10)
+
+	// TODO: t = TLS, q = QUIC
+	buf = append(buf, 't')
+
+	{
+		var sslVersion uint16
+		for _, v := range hello.SupportedVersions {
+			if v&greaseMask != greaseValue {
+				if v > sslVersion {
+					sslVersion = v
+				}
+			}
+		}
+
+		switch sslVersion {
+		case tls.VersionSSL30:
+			buf = append(buf, 's', '3')
+		case tls.VersionTLS10:
+			buf = append(buf, '1', '0')
+		case tls.VersionTLS11:
+			buf = append(buf, '1', '1')
+		case tls.VersionTLS12:
+			buf = append(buf, '1', '2')
+		case tls.VersionTLS13:
+			buf = append(buf, '1', '3')
+		default:
+			sslVersion -= 0x0201
+			buf = strconv.AppendUint(buf, uint64(sslVersion>>8), 10)
+			buf = strconv.AppendUint(buf, uint64(sslVersion&0xff), 10)
+		}
+
+	}
+
+	if slices.Contains(hello.Extensions, extensionServerName) && hello.ServerName != "" {
+		buf = append(buf, 'd')
+	} else {
+		buf = append(buf, 'i')
+	}
+
+	ciphers := make([]uint16, 0, len(hello.CipherSuites))
+	for _, cipher := range hello.CipherSuites {
+		if cipher&greaseMask != greaseValue {
+			ciphers = append(ciphers, cipher)
+		}
+	}
+
+	extensionCount := 0
+	extensions := make([]uint16, 0, len(hello.Extensions))
+	for _, extension := range hello.Extensions {
+		if extension&greaseMask != greaseValue {
+			extensionCount++
+			if extension != extensionALPN && extension != extensionServerName {
+				extensions = append(extensions, extension)
+			}
+		}
+	}
+
+	schemes := make([]tls.SignatureScheme, 0, len(hello.SignatureSchemes))
+
+	for _, scheme := range hello.SignatureSchemes {
+		if scheme&greaseMask != greaseValue {
+			schemes = append(schemes, scheme)
+		}
+	}
+
+	//TODO: maybe little endian
+	slices.Sort(ciphers)
+	slices.Sort(extensions)
+	//slices.Sort(schemes)
+
+	if len(ciphers) < 10 {
+		buf = append(buf, '0')
+		buf = strconv.AppendUint(buf, uint64(len(ciphers)), 10)
+	} else if len(ciphers) > 99 {
+		buf = append(buf, '9', '9')
+	} else {
+		buf = strconv.AppendUint(buf, uint64(len(ciphers)), 10)
+	}
+
+	if extensionCount < 10 {
+		buf = append(buf, '0')
+		buf = strconv.AppendUint(buf, uint64(extensionCount), 10)
+	} else if extensionCount > 99 {
+		buf = append(buf, '9', '9')
+	} else {
+		buf = strconv.AppendUint(buf, uint64(extensionCount), 10)
+	}
+
+	if len(hello.SupportedProtos) > 0 && len(hello.SupportedProtos[0]) > 1 {
+		buf = append(buf, hello.SupportedProtos[0][0], hello.SupportedProtos[0][len(hello.SupportedProtos[0])-1])
+	} else {
+		buf = append(buf, '0', '0')
+	}
+
+	copy(ja4.A[:], buf)
+
+	ja4.B = ja4SHA256(uint16SliceToHex(ciphers))
+
+	extBuf := uint16SliceToHex(extensions)
+
+	if len(schemes) > 0 {
+		extBuf = append(extBuf, '_')
+		extBuf = append(extBuf, uint16SliceToHex(schemes)...)
+	}
+
+	ja4.C = ja4SHA256(extBuf)
+
+	return ja4
+}
+
+func uint16SliceToHex[T ~uint16](in []T) (out []byte) {
+	if len(in) == 0 {
+		return out
+	}
+	out = slices.Grow(out, hex.EncodedLen(len(in)*2)+len(in))
+
+	for _, n := range in {
+		out = append(out, fmt.Sprintf("%04x", uint16(n))...)
+		out = append(out, ',')
+	}
+	out = out[:len(out)-1]
+
+	return out
+}
+
+func ja4SHA256(buf []byte) [6]byte {
+	if len(buf) == 0 {
+		return [6]byte{0, 0, 0, 0, 0, 0}
+	}
+	sum := sha256.Sum256(buf)
+
+	return [6]byte(sum[:6])
+}
+
+func buildTLSFingerprint(hello *tls.ClientHelloInfo) (ja3n TLSFingerprintJA3N, ja4 TLSFingerprintJA4) {
+	return TLSFingerprintJA3N(tlsFingerprintJA3(hello, true)), tlsFingerprintJA4(hello)
+}
+
+func GetTLSFingerprint(r *http.Request) *TLSFingerprint {
+	ptr := r.Context().Value(tlsFingerprintKey{})
+	if fpPtr, ok := ptr.(*TLSFingerprint); ok && ptr != nil && fpPtr != nil {
+		return fpPtr
+	}
+	return nil
+}

utils/http.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
@@ -11,6 +12,28 @@ import (
 	"strings"
 )
 
+func NewServer(handler http.Handler, tlsConfig *tls.Config) *http.Server {
+
+	if tlsConfig == nil {
+		proto := new(http.Protocols)
+		proto.SetHTTP1(true)
+		proto.SetUnencryptedHTTP2(true)
+		h1s := &http.Server{
+			Handler:   handler,
+			Protocols: proto,
+		}
+
+		return h1s
+	} else {
+		server := &http.Server{
+			TLSConfig: tlsConfig,
+			Handler:   handler,
+		}
+		applyTLSFingerprinter(server)
+		return server
+	}
+}
+
 func EnsureNoOpenRedirect(redirect string) (string, error) {
 	uri, err := url.Parse(redirect)
 	if err != nil {