docker: publish container images on github

Each IP entry in the table is wrapped in a div.

.drone.jsonnet

@@ -8,6 +8,7 @@ local Build(go, alpine, os, arch) = {
         arch: arch
     },
     environment: {
+        GOTOOLCHAIN: "local",
         CGO_ENABLED: "0",
         GOOS: os,
         GOARCH: arch,
@@ -51,10 +52,10 @@ local Build(go, alpine, os, arch) = {
     ]
 };
 
-local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
+local Publish(registry, repo, secret, go, alpine, os, arch, trigger, platforms, extra) = {
     kind: "pipeline",
     type: "docker",
-    name: "publish-" + go + "-alpine" + alpine,
+    name: "publish-" + go + "-alpine" + alpine + "-" + secret,
     platform: {
         os: os,
         arch: arch,
@@ -66,11 +67,14 @@ local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
             image: "plugins/buildx",
             privileged: true,
             environment: {
-                DOCKER_BUILDKIT: "1"
+                DOCKER_BUILDKIT: "1",
+                SOURCE_DATE_EPOCH: 0,
+                TZ: "UTC",
+                LC_ALL: "C",
             },
             settings: {
-                  registry: "git.gammaspectra.live",
-                  repo: "git.gammaspectra.live/git/go-away",
+                  registry: registry,
+                  repo: repo,
                   compress: true,
                   platform: platforms,
                   builder_driver: "docker-container",
@@ -80,10 +84,10 @@ local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
                   },
                   auto_tag_suffix: "alpine" + alpine,
                   username: {
-                    from_secret: "git_username",
+                    from_secret: secret + "_username",
                   },
                   password: {
-                    from_secret: "git_password",
+                    from_secret: secret + "_password",
                   },
             } + extra,
         },
@@ -91,18 +95,22 @@ local Publish(go, alpine, os, arch, trigger, platforms, extra) = {
 };
 
 #
+local containerArchitectures = ["linux/amd64", "linux/arm64", "linux/riscv64"];
+
+local alpineVersion = "3.21";
+local goVersion = "1.24";
 
 [
-    Build("1.24", "3.20", "linux", "amd64"),
-    Build("1.24", "3.20", "linux", "arm64"),
-    Build("1.24", "3.21", "linux", "amd64"),
-    Build("1.24", "3.21", "linux", "arm64"),
+    Build(goVersion, alpineVersion, "linux", "amd64"),
+    Build(goVersion, alpineVersion, "linux", "arm64"),
 
     # latest
-    Publish("1.24", "3.21", "linux", "amd64", {event: ["push"], branch: ["master"], }, ["linux/amd64", "linux/arm64"], {tags: ["latest"],}) + {name: "publish-latest"},
+    Publish("git.gammaspectra.live", "git.gammaspectra.live/git/go-away", "git", goVersion, alpineVersion, "linux", "amd64", {event: ["push"], branch: ["master"], }, containerArchitectures, {tags: ["latest"],}) + {name: "publish-latest-git"},
+    Publish("codeberg.org", "codeberg.org/weebdatahoarder/go-away", "codeberg", goVersion, alpineVersion, "linux", "amd64", {event: ["push"], branch: ["master"], }, containerArchitectures, {tags: ["latest"],}) + {name: "publish-latest-codeberg"},
+    Publish("ghcr.io", "ghcr.io/weebdatahoarder/go-away", "github", goVersion, alpineVersion, "linux", "amd64", {event: ["push"], branch: ["master"], }, containerArchitectures, {tags: ["latest"],}) + {name: "publish-latest-github"},
 
     # modern
-    Publish("1.24", "3.21", "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, ["linux/amd64", "linux/arm64"], {auto_tag: true,}),
-    # legacy
-    Publish("1.24", "3.20", "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, ["linux/amd64", "linux/arm64"], {auto_tag: true,}),
+    Publish("git.gammaspectra.live", "git.gammaspectra.live/git/go-away", "git", goVersion, alpineVersion, "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, containerArchitectures, {auto_tag: true,}),
+    Publish("codeberg.org", "codeberg.org/weebdatahoarder/go-away", "codeberg", goVersion, alpineVersion, "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, containerArchitectures, {auto_tag: true,}),
+    Publish("ghcr.io", "ghcr.io/weebdatahoarder/go-away", "github", goVersion, alpineVersion, "linux", "amd64", {event: ["promote", "tag"], target: ["production"], }, containerArchitectures, {auto_tag: true,}),
 ]

.drone.yml

@@ -3,86 +3,7 @@ environment:
   CGO_ENABLED: "0"
   GOARCH: amd64
   GOOS: linux
-kind: pipeline
-name: build-1.24-alpine3.20-amd64
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - apk update
-  - apk add --no-cache git
-  - mkdir .bin
-  - go build -v -o ./.bin/go-away ./cmd/go-away
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
-  image: golang:1.24-alpine3.20
-  name: build
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge.json -verify-challenge-out
-    0
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-success
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge-fail.json -verify-challenge-out
-    1
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-fail
-type: docker
----
-environment:
-  CGO_ENABLED: "0"
-  GOARCH: arm64
-  GOOS: linux
-kind: pipeline
-name: build-1.24-alpine3.20-arm64
-platform:
-  arch: arm64
-  os: linux
-steps:
-- commands:
-  - apk update
-  - apk add --no-cache git
-  - mkdir .bin
-  - go build -v -o ./.bin/go-away ./cmd/go-away
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
-  image: golang:1.24-alpine3.20
-  name: build
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge.json -verify-challenge-out
-    0
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-success
-- commands:
-  - ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm
-    -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out
-    ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge
-    ./embed/challenge/js-pow-sha256/test/verify-challenge-fail.json -verify-challenge-out
-    1
-  depends_on:
-  - build
-  image: alpine:3.20
-  name: test-wasm-fail
-type: docker
----
-environment:
-  CGO_ENABLED: "0"
-  GOARCH: amd64
-  GOOS: linux
+  GOTOOLCHAIN: local
 kind: pipeline
 name: build-1.24-alpine3.21-amd64
 platform:
@@ -123,6 +44,7 @@ environment:
   CGO_ENABLED: "0"
   GOARCH: arm64
   GOOS: linux
+  GOTOOLCHAIN: local
 kind: pipeline
 name: build-1.24-alpine3.21-arm64
 platform:
@@ -160,13 +82,16 @@ steps:
 type: docker
 ---
 kind: pipeline
-name: publish-latest
+name: publish-latest-git
 platform:
   arch: amd64
   os: linux
 steps:
 - environment:
     DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
   image: plugins/buildx
   name: docker
   privileged: true
@@ -182,6 +107,7 @@ steps:
     platform:
     - linux/amd64
     - linux/arm64
+    - linux/riscv64
     registry: git.gammaspectra.live
     repo: git.gammaspectra.live/git/go-away
     tags:
@@ -196,13 +122,96 @@ trigger:
 type: docker
 ---
 kind: pipeline
-name: publish-1.24-alpine3.21
+name: publish-latest-codeberg
 platform:
   arch: amd64
   os: linux
 steps:
 - environment:
     DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    builder_driver: docker-container
+    compress: true
+    password:
+      from_secret: codeberg_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    - linux/riscv64
+    registry: codeberg.org
+    repo: codeberg.org/weebdatahoarder/go-away
+    tags:
+    - latest
+    username:
+      from_secret: codeberg_username
+trigger:
+  branch:
+  - master
+  event:
+  - push
+type: docker
+---
+kind: pipeline
+name: publish-latest-github
+platform:
+  arch: amd64
+  os: linux
+steps:
+- environment:
+    DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    builder_driver: docker-container
+    compress: true
+    password:
+      from_secret: github_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    - linux/riscv64
+    registry: ghcr.io
+    repo: ghcr.io/weebdatahoarder/go-away
+    tags:
+    - latest
+    username:
+      from_secret: github_username
+trigger:
+  branch:
+  - master
+  event:
+  - push
+type: docker
+---
+kind: pipeline
+name: publish-1.24-alpine3.21-git
+platform:
+  arch: amd64
+  os: linux
+steps:
+- environment:
+    DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
   image: plugins/buildx
   name: docker
   privileged: true
@@ -219,6 +228,7 @@ steps:
     platform:
     - linux/amd64
     - linux/arm64
+    - linux/riscv64
     registry: git.gammaspectra.live
     repo: git.gammaspectra.live/git/go-away
     username:
@@ -232,33 +242,77 @@ trigger:
 type: docker
 ---
 kind: pipeline
-name: publish-1.24-alpine3.20
+name: publish-1.24-alpine3.21-codeberg
 platform:
   arch: amd64
   os: linux
 steps:
 - environment:
     DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
   image: plugins/buildx
   name: docker
   privileged: true
   settings:
     auto_tag: true
-    auto_tag_suffix: alpine3.20
+    auto_tag_suffix: alpine3.21
     build_args:
-      from: alpine:3.20
-      from_builder: golang:1.24-alpine3.20
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
     builder_driver: docker-container
     compress: true
     password:
-      from_secret: git_password
+      from_secret: codeberg_password
     platform:
     - linux/amd64
     - linux/arm64
-    registry: git.gammaspectra.live
-    repo: git.gammaspectra.live/git/go-away
+    - linux/riscv64
+    registry: codeberg.org
+    repo: codeberg.org/weebdatahoarder/go-away
     username:
-      from_secret: git_username
+      from_secret: codeberg_username
+trigger:
+  event:
+  - promote
+  - tag
+  target:
+  - production
+type: docker
+---
+kind: pipeline
+name: publish-1.24-alpine3.21-github
+platform:
+  arch: amd64
+  os: linux
+steps:
+- environment:
+    DOCKER_BUILDKIT: "1"
+    LC_ALL: C
+    SOURCE_DATE_EPOCH: 0
+    TZ: UTC
+  image: plugins/buildx
+  name: docker
+  privileged: true
+  settings:
+    auto_tag: true
+    auto_tag_suffix: alpine3.21
+    build_args:
+      from: alpine:3.21
+      from_builder: golang:1.24-alpine3.21
+    builder_driver: docker-container
+    compress: true
+    password:
+      from_secret: github_password
+    platform:
+    - linux/amd64
+    - linux/arm64
+    - linux/riscv64
+    registry: ghcr.io
+    repo: ghcr.io/weebdatahoarder/go-away
+    username:
+      from_secret: github_username
 trigger:
   event:
   - promote
@@ -268,6 +322,6 @@ trigger:
 type: docker
 ---
 kind: signature
-hmac: 8583621811fa483a6594352a8f9eeca7d66f6509f8e36bd2299b9e0723ed1451
+hmac: f27dd6fbc73d3dd6e26739576a02b6bf0f9d1c43ee9d6d1439afacdf4e4dbf96
 
 ...

CHALLENGES.md

@@ -0,0 +1,118 @@
+# Challenges
+
+Challenges can be [transparent](#transparent) (not shown to user, depends on backend or other logic), [non-JavaScript](#non-javascript) (challenges common browser properties), or [custom JavaScript](README.md#custom-javascript) (from Proof of Work to fingerprinting or Captcha is supported)
+
+## Transparent
+
+### http
+
+Verify incoming requests against a specified backend to allow the user through. Cookies and some other headers are passed.
+
+For example, this allows verifying the user cookies against the backend to have the user skip all other challenges.
+
+Example on Forgejo, checks that current user is authenticated:
+```yaml
+  http-cookie-check:
+    mode: http
+    url: http://forgejo:3000/user/stopwatches
+    # url: http://forgejo:3000/repo/search
+    # url: http://forgejo:3000/notifications/new
+    parameters:
+      http-method: GET
+      http-cookie: i_like_gitea
+      http-code: 200
+```
+
+### preload-link
+
+Requires HTTP/2+ response parsing and logic, silent challenge (does not display a challenge page).
+
+Browsers that support [103 Early Hints](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/103) are indicated to fetch a CSS resource via [Link](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link) preload that solves the challenge.
+
+The server waits until solved or defined timeout, then continues on other challenges if failed.
+
+Example:
+```yaml
+  self-preload-link:
+    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
+    mode: "preload-link"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      preload-early-hint-deadline: 3s
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+```
+
+## Non-JavaScript
+
+### cookie
+
+Requires HTTP parsing and a Cookie Jar, silent challenge (does not display a challenge page unless failed).
+
+Serves the client with a Set-Cookie that solves the challenge, and redirects it back to the same page. Browser must present the cookie to load.
+
+Several tools implement this, but usually not mass scrapers.
+
+### header-refresh
+
+Requires HTTP response parsing and logic, displays challenge site instantly.
+
+Have the browser solve the challenge by following the URL listed on HTTP [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh) instantly.
+
+
+### meta-refresh
+
+Requires HTTP and HTML response parsing and logic, displays challenge site instantly.
+
+Have the browser solve the challenge by following the URL listed on HTML `<meta http-equiv=refresh>` tag instantly. Equivalent to above.
+
+### resource-load
+
+Requires HTTP and HTML response parsing and logic, displays challenge site.
+
+Servers a challenge page with a linked resource that is loaded by the browser, which solves the challenge. Page refreshes a few seconds later via [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh).
+
+Example:
+```yaml
+  self-resource-load:
+    mode: "resource-load"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+    parameters:
+      key-code: 200
+      key-mime: text/css
+      key-content: ""
+```
+
+## Custom JavaScript
+
+### js-pow-sha256
+
+Requires JavaScript and workers, displays challenge site.
+
+Has the user solve a Proof of Work using SHA256 hashes, with configurable difficulty.
+
+Example:
+```yaml
+  js-pow-sha256:
+    # Asset must be under challenges/{name}/static/{asset}
+    # Other files here will be available under that path
+    mode: js
+    asset: load.mjs
+    parameters:
+      # difficulty is number of bits that must be set to 0 from start
+      # Anubis challenge difficulty 5 becomes 5 * 8 = 20
+      difficulty: 20
+    runtime:
+      mode: wasm
+      # Verify must be under challenges/{name}/runtime/{asset}
+      asset: runtime.wasm
+      probability: 0.02
+```
+

Dockerfile

@@ -7,6 +7,7 @@ FROM --platform=$BUILDPLATFORM ${from_builder} AS build
 
 ARG TARGETARCH
 ARG TARGETOS
+ARG GOTOOLCHAIN=local
 
 RUN apk update && apk add --no-cache \
     bash \
@@ -22,8 +23,9 @@ RUN ./build-compress.sh
 ENV CGO_ENABLED=0
 ENV GOOS=${TARGETOS}
 ENV GOARCH=${TARGETARCH}
+ENV GOTOOLCHAIN=${GOTOOLCHAIN}
 
-RUN go build -pgo=auto -v -trimpath -o "${GOBIN}/go-away" ./cmd/go-away
+RUN go build -pgo=auto -v -trimpath -ldflags=-buildid= -o "${GOBIN}/go-away" ./cmd/go-away
 RUN test -e "${GOBIN}/go-away"
 
 

README.md

@@ -1,14 +1,22 @@
+### <a id=why></a>
 # go-away
 
-Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots.
+Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots. 
 
 [![Build Status](https://ci.gammaspectra.live/api/badges/git/go-away/status.svg)](https://ci.gammaspectra.live/git/go-away)
 [![Go Reference](https://pkg.go.dev/badge/git.gammaspectra.live/git/go-away.svg)](https://pkg.go.dev/git.gammaspectra.live/git/go-away)
 
-This documentation is a work in progress. For now, see policy examples under [examples/](examples/).
+go-away sits in between your site and the Internet / upstream proxy.
 
-This Go package can be used as a command on `git.gammaspectra.live/git/go-away/cmd/go-away` or a library under `git.gammaspectra.live/git/go-away/lib`
+Incoming requests can be selected by [rules](#rich-rule-matching) to be [actioned](#extended-rule-actions) or [challenged](CHALLENGES.md#challenges) to filter suspicious requests.
 
+The tool is designed highly flexible so the operator can minimize impact to legit users, while surgically targeting heavy endpoints or scrapers.
+
+[Challenges](CHALLENGES.md#challenges) can be transparent (not shown to user, depends on backend or other logic), [non-JavaScript](#non-javascript-challenges) (challenges common browser properties), or [custom JavaScript](#custom-javascript-wasm-challenges) (from Proof of Work to fingerprinting or Captcha is supported)
+
+See _[Why do this?](#why-do-this)_ section for the challenges and reasoning behind this tool. 
+
+This documentation and go-away are in active development. See [What's left?](#what-s-left) section for a breakdown.
 
 ## Support
 
@@ -16,8 +24,9 @@ If you have some suggestion or issue, feel free to open a [New Issue](https://gi
 
 [Pull Requests](https://git.gammaspectra.live/git/go-away/pulls) are encouraged and desired.
 
-For real-time chat and other support join IRC on [##go-away](ircs://irc.libera.chat/##go-away) on Libera.Chat. The channel may not be monitored at all times, feel free to ping the operators there.
+For real-time chat and other support join IRC on [#go-away](ircs://irc.libera.chat/#go-away) on Libera.Chat [[WebIRC]](https://web.libera.chat/?nick=Guest?#go-away). The channel may not be monitored at all times, feel free to ping the operators there.
 
+A source code mirror exists on [sourcehut](https://git.sr.ht/~datahoarder/go-away), [Codeberg.org](https://codeberg.org/WeebDataHoarder/go-away), and [GitHub](https://github.com/WeebDataHoarder/go-away).
 
 ## Features
 
@@ -62,7 +71,7 @@ Internal or external templates can be loaded to customize the look of the challe
 These templates are included by default:
 
 * `anubis`: An anubis-like themed challenge.
-* `forgejo`: Uses the Forgejo template and assets from your own instance. Supports specifying themes like `forgejo-light` and `forgejo-dark`.
+* `forgejo`: Uses the Forgejo template and assets from your own instance. Supports specifying themes like `forgejo-auto`, `forgejo-light` and `forgejo-dark`.
 
 External templates for your site can be loaded specifying a full path to the `.gohtml` file. See [embed/templates/](embed/templates/) for examples to follow.
 
@@ -70,9 +79,12 @@ External templates for your site can be loaded specifying a full path to the `.g
 
 In addition to the common PASS / CHALLENGE / DENY rules, we offer CHECK and POISON.
 
-CHECK allows the client to be challenged but continue matching rules after these.
+CHECK allows the client to be challenged but continue matching rules after these, for example, chaining a list of challenges that must be passed. 
+For example, you could use this to implement browser in checks without explicitly allowing all requests, and later deferring to a secondary check/challenge.
 
 POISON sends defined responses to bad clients that will annoy them.
+This must be configured by the operator, some networks have been seen to only stop when served back this output.
+Currently, an HTML payload exists that uncompressed to about one GiB of nonsense DOM. You could use this to send garbage for would-be training data.
 
 ### Multiple challenge matching
 
@@ -89,7 +101,7 @@ For example:
 
 This rule has the user be checked against a backend, then attempts pass a few browser challenges.
 
-In this case the processing would stop at `self-meta-refresh` due to the behavior of earlier challenges.
+In this case the processing would stop at `self-meta-refresh` due to the behavior of earlier challenges (cookie check and preload link allow failing / continue due to being silent, while meta-refresh requires displaying a challenge page).
 
 Any of these listed challenges being passed in the past will allow the client through, including non-offered `self-resource-load` and `js-pow-sha256`.
 
@@ -99,9 +111,9 @@ Several challenges that do not require JavaScript are offered, some targeting th
 
 These can be used for light checking of requests that eliminate most of the low effort scraping.
 
-See [Challenges](#challenges) below for a list of them.
+See [Challenges](CHALLENGES.md#challenges) for a list of them.
 
-### Custom proof-of-work JS / WASM challenges
+### Custom JavaScript / WASM challenges
 
 A WASM interface for server-side proof generation and checking is offered. We provide `js-pow-sha256` as an example of one.
 
@@ -145,7 +157,11 @@ Results will be temporarily cached
 
 By default, [DroneBL](https://dronebl.org/) is used.
 
-### Network range loading
+### Network range and automated filtering
+
+Some specific search spiders do follow _robots.txt_ and are well behaved. However, many actors can reuse user agents, so the origin network ranges must be checked.
+
+The samples provide example network range fetching and rules for Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot.
 
 Network ranges can be loaded via fetched JSON / TXT / HTML pages, or via lists. You can filter these using _jq_ or a regex.
 
@@ -186,46 +202,11 @@ You can modify the path where challenges are served and package name, if you don
 
 No source code editing or forking necessary!
 
-## Why?
-In the past few years this small git instance has been hit by waves and waves of scraping.
-This was usually fought back by random useragent blocks for bots that did not follow [robots.txt](/robots.txt), until the past half year, where low-effort mass scraping was used more prominently.
+### IPv6 Happy Eyeballs challenge retry
 
-Recently these networks go from using residential IP blocks to sending requests at several hundred rps.
+In case a client connects over IPv4 first then IPv6 due to [Fast Fallback / Happy Eyeballs](https://en.wikipedia.org/wiki/Happy_Eyeballs), the challenge will automatically be retried.
 
-If the server gets sluggish, more requests pile up. Even when denied they scrape for weeks later. Effectively spray and pray scraping, process later.
-
-At some point about 300Mbit/s of incoming requests (not including the responses) was hitting the server. And all at nonsense URLs
-
-If AI is so smart, why not just git clone the repositories?
-
-
-Xe (anubis creator) has written about similar frustrations in several blogposts:
-
-* [Amazon's AI crawler is making my git server unstable](https://xeiaso.net/notes/2025/amazon-crawler/) [01/17/2025]
-* [Anubis works](https://xeiaso.net/notes/2025/anubis-works/) [04/12/2025]
-
-Drew DeVault (sourcehut) has posted several articles regarding the same issues:
-* [Please stop externalizing your costs directly into my face](https://drewdevault.com/2025/03/17/2025-03-17-Stop-externalizing-your-costs-on-me.html) [17/03/2025]
-  * (fun tidbit: I'm the one quoted as having the feedback discussion interrupted to deal with bots!)
-
-Others were also suffering at the same time [[1]](https://donotsta.re/notice/AreSNZlRlJv73AW7tI) [[2]](https://community.ipfire.org/t/suricata-ruleset-to-prevent-ai-scraping/11974) [[3]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[4]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[5]](https://blog.nytsoi.net/2025/03/01/obliterated-by-ai).
-
----
-Initially I deployed Anubis, and yeah, it does work!
-
-This tool started as a way to replace [Anubis](https://anubis.techaro.lol/) as it was not found as featureful as desired.
-
-go-away may not be as straight to configure as Anubis but this was chosen to reduce impact on legitimate users, and offers many more options to dynamically target new waves.
-
-### Can't scrapers adapt?
-
-Yes, they can. At the moment their spray-and-pray approach is cheap for them.
-
-If they have to start adding an active browser in their scraping, that makes their collection expensive and slow. 
-
-This would more or less eliminate the high rate low effort passive scraping and replace it with an active model. 
-
-go-anubis offers a highly configurable set of challenges and rules that you can adapt to new ways.
+This is tracked by tagging challenges with a readable flag indicating the type of address.
 
 ## Example policies
 
@@ -252,17 +233,82 @@ Important notes:
 * Add or modify rules to target specific pages on your site as desired.
 * By default Googlebot / Bingbot / DuckDuckBot / Kagibot / Qwantbot / Yandexbot are allowed by useragent and network ranges.
 
+## Why do this?
+In the past few years this small git instance has been hit by waves and waves of scraping.
+This was usually fought back by random useragent blocks for bots that did not follow [robots.txt](/robots.txt), until the past half year, where low-effort mass scraping was used more prominently.
+
+Recently these networks go from using residential IP blocks to sending requests at several hundred rps.
+
+If the server gets sluggish, more requests pile up. Even when denied they scrape for weeks later. Effectively spray and pray scraping, process later.
+
+At some point about 300Mbit/s of incoming requests (not including the responses) was hitting the server. And all of them nonsense URLs, or hitting archive/bundle downloads per commit.
+
+If AI is so smart, why not just git clone the repositories?
+
+
+Xe (anubis creator) has written about similar frustrations in several blogposts:
+
+* [Amazon's AI crawler is making my git server unstable](https://xeiaso.net/notes/2025/amazon-crawler/) [01/17/2025]
+* [Anubis works](https://xeiaso.net/notes/2025/anubis-works/) [04/12/2025]
+
+Drew DeVault (sourcehut) has posted several articles regarding the same issues:
+* [Please stop externalizing your costs directly into my face](https://drewdevault.com/2025/03/17/2025-03-17-Stop-externalizing-your-costs-on-me.html) [17/03/2025]
+  * (fun tidbit: I'm the one quoted as having the feedback discussion interrupted to deal with bots!)
+* [sourcehut Blog: You cannot have our user's data](https://sourcehut.org/blog/2025-04-15-you-cannot-have-our-users-data/)
+
+Others were also suffering at the same time [[1]](https://donotsta.re/notice/AreSNZlRlJv73AW7tI) [[2]](https://community.ipfire.org/t/suricata-ruleset-to-prevent-ai-scraping/11974) [[3]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[4]](https://gabrielsimmer.com/blog/stop-scraping-git-forge) [[5]](https://blog.nytsoi.net/2025/03/01/obliterated-by-ai).
+
+---
+Initially I deployed Anubis, and yeah, it does work!
+
+This tool started as a way to replace [Anubis](https://anubis.techaro.lol/) as it was not found as featureful as desired.
+
+go-away may not be as straight to configure as Anubis but this was chosen to reduce impact on legitimate users, and offers many more options to dynamically target new waves.
+
+### Can't scrapers adapt?
+
+Yes, they can. At the moment their spray-and-pray approach is cheap for them.
+
+If they have to start adding an active browser in their scraping, that makes their collection expensive and slow.
+
+This would more or less eliminate the high rate low effort passive scraping and replace it with an active model.
+
+go-away offers a highly configurable set of challenges and rules that you can adapt to new ways.
+
+## What's left?
+
+go-away has most of the desired features from the original checklist that was made in its development. 
+However, a few points are left before go-away can be called v1.0.0:
+
+* [ ] Several parts of the code are going through a refactor, which won't impact end users or operators.
+* [ ] Documentation is lacking and a more extensive one with inline example is in the works.
+* [ ] Policy file syntax is going to stay mostly unchanged, except in the challenges definition section.
+* [ ] Allow users to pick fallback challenges if any fail, specially with custom ones.
+* [ ] Replace Anubis-like default template with own one.
+* [ ] Define strings and multi-language support for quick modification by operators without custom templates.
+* [ ] Have highly tested paths that match examples.
+* [ ] Caching of temporary fetches, for example, network ranges.
+* [ ] Allow live and dynamic policy reloading.
+* [ ] Multiple domains / subdomains -> one backend handling, CEL rules for backends
+* [ ] Merge all rules and conditions into one large AST for higher performance.
+* [ ] Explore exposing a module for direct Caddy usage.
+* [ ] More defined way of picking HTTP/HTTP(s) listeners and certificates.
+* [ ] Expose metrics for gathering common network ranges, challenge solve rates and acting on them.
 
 ## Setup
 
-It is recommended to have another reverse proxy above (for example [Caddy](https://caddyserver.com/), nginx, HAProxy) to handle HTTPs or similar.
+go-away can take plaintext HTTP/1 and _HTTP/2_ / _h2c_ connections if desired over the same port. When doing this, it is recommended to have another reverse proxy above (for example [Caddy](https://caddyserver.com/), nginx, HAProxy) to handle HTTPs or similar.
 
-go-away for now only accepts plaintext connections, although it can take _HTTP/2_ / _h2c_ connections if desired over the same port.
+We also support the `autocert` parameter to configure HTTP(s). This will also allow TLS Fingerprinting to be done on incoming clients. This doesn't require any upstream proxies, and we recommend it's exposed directly or via SNI / Layer 4 proxying.
 
 ### Binary / Go
 
 Requires Go 1.24+. Builds statically without CGo usage.
 
+We have Go 1.22+ support on the [go1.22 branch](https://git.gammaspectra.live/git/go-away/src/branch/go1.22).
+It will be regularly rebased to keep current with recent releases, at least until v1.0.0.
+Some features, such as TLS Fingerprinting, are not available on Go 1.22.
+
 ```shell
 git clone https://git.gammaspectra.live/git/go-away.git && cd go-away
 
@@ -358,121 +404,17 @@ services:
 
 ```
 
-## Challenges
-
-#### http
-
-Verify incoming requests against a specified backend to allow the user through. Cookies and some other headers are passed.
-
-For example, this allows verifying the user cookies against the backend to have the user skip all other challenges.
-
-Example on Forgejo, checks that current user is authenticated:
-```yaml
-  http-cookie-check:
-    mode: http
-    url: http://forgejo:3000/user/stopwatches
-    # url: http://forgejo:3000/repo/search
-    # url: http://forgejo:3000/notifications/new
-    parameters:
-      http-method: GET
-      http-cookie: i_like_gitea
-      http-code: 200
-```
-
-#### preload-link
-
-Requires HTTP/2+ response parsing and logic, silent challenge (does not display a challenge page).
-
-Browsers that support [103 Early Hints](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/103) are indicated to fetch a CSS resource via [Link](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link) preload that solves the challenge.
-
-The server waits until solved or defined timeout, then continues on other challenges if failed.
-
-Example:
-```yaml
-  self-preload-link:
-    condition: '"Sec-Fetch-Mode" in headers && headers["Sec-Fetch-Mode"] == "navigate"'
-    mode: "preload-link"
-    runtime:
-      # verifies that result = key
-      mode: "key"
-      probability: 0.1
-    parameters:
-      preload-early-hint-deadline: 3s
-      key-code: 200
-      key-mime: text/css
-      key-content: ""
-```
-
-#### header-refresh
-
-Requires HTTP response parsing and logic, displays challenge site instantly.
-
-Have the browser solve the challenge by following the URL listed on HTTP [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh) instantly.
-
-
-#### meta-refresh
-
-Requires HTTP and HTML response parsing and logic, displays challenge site instantly.
-
-Have the browser solve the challenge by following the URL listed on HTML `<meta http-equiv=refresh>` tag instantly. Equivalent to above.
-
-#### resource-load
-
-Requires HTTP and HTML response parsing and logic, displays challenge site.
-
-Servers a challenge page with a linked resource that is loaded by the browser, which solves the challenge. Page refreshes a few seconds later via [Refresh](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Refresh).
-
-Example:
-```yaml
-  self-resource-load:
-    mode: "resource-load"
-    runtime:
-      # verifies that result = key
-      mode: "key"
-      probability: 0.1
-    parameters:
-      key-code: 200
-      key-mime: text/css
-      key-content: ""
-```
-
-#### cookie
-
-Requires HTTP parsing and a Cookie Jar, silent challenge (does not display a challenge page unless failed).
-
-Serves the client with a Set-Cookie that solves the challenge, and redirects it back to the same page. Browser must present the cookie to load.
-
-Several tools implement this, but usually not mass scrapers.
-
-#### js-pow-sha256
-
-Requires JavaScript and workers, displays challenge site.
-
-Has the user solve a Proof of Work using SHA256 hashes, with configurable difficulty.
-
-Example:
-```yaml
-  js-pow-sha256:
-    # Asset must be under challenges/{name}/static/{asset}
-    # Other files here will be available under that path
-    mode: js
-    asset: load.mjs
-    parameters:
-      # difficulty is number of bits that must be set to 0 from start
-      # Anubis challenge difficulty 5 becomes 5 * 8 = 20
-      difficulty: 20
-    runtime:
-      mode: wasm
-      # Verify must be under challenges/{name}/runtime/{asset}
-      asset: runtime.wasm
-      probability: 0.02
-```
 
 
+## Other Similar Projects
+* [Anubis](https://anubis.techaro.lol/): Proxy that uses JavaScript proof of work to weight request based on rules [[source]](https://github.com/TecharoHQ/anubis)
+* [anticrawl](https://flak.tedunangst.com/post/anticrawl): Go http handler / proxy for regex based rules [[source]](https://humungus.tedunangst.com/r/anticrawl)
 
 
 ## Development
 
+This Go package can be used as a command on `git.gammaspectra.live/git/go-away/cmd/go-away` or a library under `git.gammaspectra.live/git/go-away/lib`
+
 ### Compiling WASM runtime challenge modules
 
 Custom WASM runtime modules follow the WASI `wasip1` preview syscall API.
@@ -494,11 +436,11 @@ func (p Allocation) Size() uint32 {
 
 
 // MakeChallenge MakeChallengeInput / MakeChallengeOutput are valid JSON.
-// See lib/challenge/interface.go for a definition
+// See lib/challenge/wasm/interface/interface.go for a definition
 func MakeChallenge(in Allocation[MakeChallengeInput]) Allocation[MakeChallengeOutput]
 
 // VerifyChallenge VerifyChallengeInput is valid JSON.
-// See lib/challenge/interface.go for a definition
+// See lib/challenge/wasm/interface/interface.go for a definition
 func VerifyChallenge(in Allocation[VerifyChallengeInput]) VerifyChallengeOutput
 
 func malloc(size uint32) uintptr
@@ -506,4 +448,8 @@ func free(size uintptr)
 
 ```
 
-Modules will be recreated for each call, so there is no state leftover
+Modules will be recreated for each call, so there is no state leftover.
+
+
+
+

cmd/go-away/main.go

@@ -140,7 +140,7 @@ func main() {
 
 	policyFile := flag.String("policy", "", "path to policy YAML file")
 	challengeTemplate := flag.String("challenge-template", "anubis", "name or path of the challenge template to use (anubis, forgejo)")
-	challengeTemplateTheme := flag.String("challenge-template-theme", "", "name of the challenge template theme to use (forgejo => [forgejo-dark, forgejo-light, gitea...])")
+	challengeTemplateTheme := flag.String("challenge-template-theme", "", "name of the challenge template theme to use (forgejo => [forgejo-auto, forgejo-dark, forgejo-light, gitea...])")
 
 	packageName := flag.String("package-path", internalPackageName, "package name to expose in .well-known url path")
 
@@ -219,7 +219,7 @@ func main() {
 	for _, backend := range backends {
 		parts := strings.Split(backend, "=")
 		if len(parts) != 2 {
-			log.Fatal(fmt.Errorf("invalid backend definition: %s", backend))
+			log.Fatal(fmt.Errorf("invalid backend definition: %s, expected 2 parts, got %v", backend, parts))
 		}
 		parsedBackends[parts[0]] = parts[1]
 	}

embed/templates/challenge-forgejo.gohtml

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-{{$theme := "forgejo-dark"}}
+{{$theme := "forgejo-auto"}}
 {{ if .Theme }}
     {{$theme = .Theme}}
 {{ end }}

examples/forgejo.yml

@@ -1,5 +1,5 @@
 # Example cmdline (forward requests from upstream to port :8080)
-# $ go-away --bind :8080 --backend git.example.com=http://forgejo:3000 --policy examples/forgejo.yml --challenge-template forgejo --challenge-template-theme forgejo-dark
+# $ go-away --bind :8080 --backend git.example.com=http://forgejo:3000 --policy examples/forgejo.yml --challenge-template forgejo --challenge-template-theme forgejo-auto
 
 
 
@@ -67,8 +67,8 @@ networks:
     - url: https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
       jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
   duckduckbot:
-    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
-      regex: "<li>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</li>"
+    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
+      regex: "<li><div>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</div></li>"
   yandexbot:
     # todo: detected as bot
     # - url: https://yandex.com/ips
@@ -227,11 +227,14 @@ conditions:
     - 'userAgent.startsWith("reqwest/")'
 
   is-suspicious-crawler:
+    # TLS Fingerprint for specific agent without ALPN
+    - '(userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")) && fpJA4.matches("^t[0-9a-z]+00_")'
+    # Old engines
     - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
     # Old IE browsers
     - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
     # Old Linux browsers
-    - 'userAgent.contains("Linux i[63]86") || userAgent.contains("FreeBSD i[63]86")'
+    - 'userAgent.matches("Linux i[63]86") || userAgent.matches("FreeBSD i[63]86")'
     # Old Windows browsers
     - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
     # Old mobile browsers
@@ -373,31 +376,36 @@ rules:
 
   - name: preview-fetchers
     conditions:
+      # These summary cards are included in most previews at the end of the url
       - 'path.endsWith("/-/summary-card")'
       #- 'userAgent.contains("facebookexternalhit/")'
-      - 'userAgent.contains("Twitterbot/")'
-      - '"X-Purpose" in headers && headers["X-Purpose"] == "preview"'
+      #- 'userAgent.contains("Twitterbot/")'
+    action: pass
+
+  # Allow loading and embedding of core pages without challenges
+  # Extended pages like linking to files or tabs are not covered here, but might be included in other challenges
+  - name: homesite
+    conditions:
+      # Match root of site
+      - 'path == "/"'
+
+      # Match root of any repository or user, or issue or pr
+      # generic /*/*/ match gave too many options for scrapers to trigger random endpoints
+      # this is a negative match of endpoints that Forgejo holds as reserved as users or orgs
+      # see https://codeberg.org/forgejo/forgejo/src/branch/forgejo/models/user/user.go#L582
+      - '(path.matches("^/[^/]+/[^/]+/?$") || path.matches("^/[^/]+/[^/]+/(issues|pulls)/[0-9]+$") || (path.matches("^/[^/]+/?$") && size(query) == 0)) && !path.matches("(?i)^/(api|metrics|v2|assets|attachments|avatar|avatars|repo-avatars|captcha|login|org|repo|user|admin|devtest|explore|issues|pulls|milestones|notifications|ghost)(/|$)")'
     action: pass
 
   - name: desired-crawlers
     conditions:
       - 'userAgent.contains("+https://kagi.com/bot") && inNetwork("kagibot", remoteAddress)'
-      - '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-InspectionTool") || userAgent.contains("Googlebot")) && inNetwork("googlebot", remoteAddress)'
+      - '(userAgent.contains("+http://www.google.com/bot.html") || userAgent.contains("Google-PageRenderer") || userAgent.contains("Google-InspectionTool") || userAgent.contains("Googlebot")) && inNetwork("googlebot", remoteAddress)'
       - 'userAgent.contains("+http://www.bing.com/bingbot.htm") && inNetwork("bingbot", remoteAddress)'
       - 'userAgent.contains("+http://duckduckgo.com/duckduckbot.html") && inNetwork("duckduckbot", remoteAddress)'
       - 'userAgent.contains("+https://help.qwant.com/bot/") && inNetwork("qwantbot", remoteAddress)'
       - 'userAgent.contains("+http://yandex.com/bots") && inNetwork("yandexbot", remoteAddress)'
     action: pass
 
-  - name: homesite
-    conditions:
-      - 'path == "/"'
-      # generic /*/*/ match gave too many options for scrapers to trigger random endpoints
-      # edit this with preferential users/orgs for now
-      # todo: create negative match?
-      - 'path.matches("(?i)^/(WeebDataHoarder|P2Pool|mirror|git|S\\.O\\.N\\.G|FM10K|Sillycom|pwgen2155|kaitou|metonym)/[^/]+$")'
-    action: pass
-
   # check a sequence of challenges
   - name: heavy-operations/0
     action: check
@@ -408,6 +416,12 @@ rules:
     challenges: [self-resource-load, js-pow-sha256, http-cookie-check]
     conditions: ['($is-heavy-resource)']
 
+  - name: standard-bots
+    action: check
+    challenges: [self-meta-refresh, self-resource-load]
+    conditions:
+      - '($is-generic-robot-ua)'
+
   # Allow all source downloads not caught in browser above
   # todo: limit this as needed?
   - name: source-download
@@ -437,12 +451,16 @@ rules:
     conditions:
       - '!(method == "HEAD" || method == "GET")'
 
+  - name: plaintext-browser
+    action: challenge
+    challenges: [http-cookie-check, self-meta-refresh, self-cookie]
+    conditions:
+      - 'userAgent.startsWith("Lynx/")'
 
   - name: standard-tools
     action: challenge
     challenges: [self-cookie]
     conditions:
-      - '($is-generic-robot-ua)'
       - '($is-tool-ua)'
       - '!($is-generic-browser)'
 

examples/generic.yml

@@ -16,8 +16,8 @@ networks:
     - url: https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
       jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
   duckduckbot:
-    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
-      regex: "<li>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</li>"
+    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
+      regex: "<li><div>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</div></li>"
   yandexbot:
     # todo: detected as bot
     # - url: https://yandex.com/ips
@@ -258,6 +258,11 @@ rules:
     conditions:
       - '!(method == "HEAD" || method == "GET")'
 
+  - name: plaintext-browser
+    action: challenge
+    challenges: [self-meta-refresh, self-cookie]
+    conditions:
+      - 'userAgent.startsWith("Lynx/")'
 
   - name: standard-tools
     action: challenge

go.mod

@@ -24,14 +24,10 @@ require (
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
-	golang.org/x/exp v0.0.0 // indirect
+	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
 	golang.org/x/net v0.39.0 // indirect
 	golang.org/x/text v0.24.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250409194420-de1ac958c67a // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 )
-
-// Used by github.com/antlr4-go/antlr v4.13.0 via github.com/google/cel-go
-// Ensure we have no other exp package usages by only proxying the slices functions in that package
-replace golang.org/x/exp v0.0.0 => ./utils/exp

go.sum

@@ -46,14 +46,16 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva
 github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
 golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
 golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
+golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
 golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
 golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
-google.golang.org/genproto/googleapis/api v0.0.0-20250409194420-de1ac958c67a h1:OQ7sHVzkx6L57dQpzUS4ckfWJ51KDH74XHTDe23xWAs=
-google.golang.org/genproto/googleapis/api v0.0.0-20250409194420-de1ac958c67a/go.mod h1:2R6XrVC8Oc08GlNh8ujEpc7HkLiEZ16QeY7FxIs20ac=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a h1:GIqLhp/cYUkuGuiT+vJk8vhOP86L4+SP5j8yXgeVpvI=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e h1:UdXH7Kzbj+Vzastr5nVfccbmFsmYNygVLSPk1pEfDoY=
+google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e/go.mod h1:085qFyf2+XaZlRdCgKNCIZ3afY2p4HHZdoIRpId8F4A=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e h1:ztQaXfzEXTmCBvbtWYRhJxW+0iJcz2qXfd38/e9l7bA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
 google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

go.work

@@ -1,8 +0,0 @@
-go 1.24.0
-
-toolchain go1.24.2
-
-use (
-	.
-	utils/exp
-)

go.work.sum

@@ -1,37 +0,0 @@
-github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
-golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
-golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
-golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
-golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
-google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
-google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg=
-google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

lib/http.go

@@ -323,10 +323,10 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 						if r != nil {
 							_, _ = io.Copy(w, reader)
 						}
-						return
 					} else {
 						http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
 					}
+					return
 				}
 			}
 		}

lib/policy/network.go

@@ -40,7 +40,7 @@ func (n Network) FetchPrefixes(c *http.Client) (output []net.IPNet, err error) {
 			return nil, err
 		}
 		defer response.Body.Close()
-		if response.StatusCode != 200 {
+		if response.StatusCode != http.StatusOK {
 			return nil, fmt.Errorf("unexpected status code: %d", response.StatusCode)
 		}
 		reader = response.Body

lib/state.go

@@ -189,7 +189,8 @@ func NewState(p policy.Policy, settings StateSettings) (handler http.Handler, er
 			}
 			prefixes, err := e.FetchPrefixes(state.Client)
 			if err != nil {
-				return nil, fmt.Errorf("networks %s: error fetching prefixes: %v", k, err)
+				slog.Error("error fetching network url list", "network", k, "url", *e.Url)
+				continue
 			}
 			for _, prefix := range prefixes {
 				err = ranger.Insert(cidranger.NewBasicRangerEntry(prefix))

utils/exp/go.mod

@@ -1,5 +0,0 @@
-module git.gammaspectra.live/git/go-away/utils/exp
-
-go 1.24.0
-
-toolchain go1.24.2

utils/exp/slices/func.go

@@ -1,7 +0,0 @@
-package slices
-
-import "slices"
-
-func EqualFunc[S1 ~[]E1, S2 ~[]E2, E1, E2 any](s1 S1, s2 S2, eq func(E1, E2) bool) bool {
-	return slices.EqualFunc(s1, s2, eq)
-}