CI attempt Uno

Defining full path don't work and cause a panic, I'll have to patch that
up.

.drone.jsonnet

@@ -22,8 +22,8 @@ local Build(mirror, go, alpine, os, arch) = {
                 "apk update",
                 "apk add --no-cache git",
                 "mkdir .bin",
-                "go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away",
+                "go build -v -pgo=auto -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/go-away ./cmd/go-away",
-                "go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime",
+                "go build -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime",
             ],
         },
         {
@@ -144,8 +144,8 @@ local goVersion = "1.24";
 local mirror = "https://mirror.gcr.io";
 
 [
-    Build(mirror, goVersion, alpineVersion, "linux", "amd64") + {"trigger": {event: ["push"], branch: ["*"], }},
+    Build(mirror, goVersion, alpineVersion, "linux", "amd64") + {"trigger": {event: ["push", "tag"], }},
-    Build(mirror, goVersion, alpineVersion, "linux", "arm64") + {"trigger": {event: ["push"], branch: ["*"], }},
+    Build(mirror, goVersion, alpineVersion, "linux", "arm64") + {"trigger": {event: ["push", "tag"], }},
 
     # Test PRs
     Build(mirror, goVersion, alpineVersion, "linux", "amd64") + {"name": "test-pr", "trigger": {event: ["pull_request"], }},

.drone.yml

@@ -14,8 +14,10 @@ steps:
   - apk update
   - apk add --no-cache git
   - mkdir .bin
-  - go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away
+  - go build -v -pgo=auto -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
+    -o ./.bin/go-away ./cmd/go-away
+  - go build -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/test-wasm-runtime
+    ./cmd/test-wasm-runtime
   image: golang:1.24-alpine3.21
   mirror: https://mirror.gcr.io
   name: build
@@ -66,10 +68,9 @@ steps:
   mirror: https://mirror.gcr.io
   name: test-wasm-fail
 trigger:
-  branch:
-  - '*'
   event:
   - push
+  - tag
 type: docker
 ---
 environment:
@@ -87,8 +88,10 @@ steps:
   - apk update
   - apk add --no-cache git
   - mkdir .bin
-  - go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away
+  - go build -v -pgo=auto -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
+    -o ./.bin/go-away ./cmd/go-away
+  - go build -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/test-wasm-runtime
+    ./cmd/test-wasm-runtime
   image: golang:1.24-alpine3.21
   mirror: https://mirror.gcr.io
   name: build
@@ -139,10 +142,9 @@ steps:
   mirror: https://mirror.gcr.io
   name: test-wasm-fail
 trigger:
-  branch:
-  - '*'
   event:
   - push
+  - tag
 type: docker
 ---
 environment:
@@ -160,8 +162,10 @@ steps:
   - apk update
   - apk add --no-cache git
   - mkdir .bin
-  - go build -v -pgo=auto -v -trimpath -ldflags=-buildid= -o ./.bin/go-away ./cmd/go-away
+  - go build -v -pgo=auto -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie
-  - go build -v -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
+    -o ./.bin/go-away ./cmd/go-away
+  - go build -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/test-wasm-runtime
+    ./cmd/test-wasm-runtime
   image: golang:1.24-alpine3.21
   mirror: https://mirror.gcr.io
   name: build
@@ -505,6 +509,6 @@ trigger:
 type: docker
 ---
 kind: signature
-hmac: 5200d5eb519acb0f74a7b62b103399da23d6e994d63c20052b41b10a4654b37a
+hmac: 9a3872c0b58810924c4342c9dbd338e16da20631c9a0848e3abd2bf6773f9ba6
 
 ...

.gitea/workflows/build.yaml

@@ -0,0 +1,71 @@
+---
+name: CI/CD Pipeline
+on:
+  push:
+    branches: [master, build-test]
+  pull_request:
+  release:
+    types: [published]
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        architecture: [amd64, arm64]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.24'
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git
+      - name: Build go-away
+        run: |
+          mkdir .bin
+          go build -v -pgo=auto -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/go-away ./cmd/go-away
+          go build -v -trimpath -ldflags='-buildid= -bindnow' -buildmode pie -o ./.bin/test-wasm-runtime ./cmd/test-wasm-runtime
+      - name: Check policy for Forgejo
+        run: |
+          ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/forgejo.yml --policy-snippets examples/snippets/
+      - name: Check policy for Generic
+        run: |
+          ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/generic.yml --policy-snippets examples/snippets/
+      - name: Check policy for SPA
+        run: |
+          ./.bin/go-away --check --slog-level DEBUG --backend example.com=http://127.0.0.1:80 --policy examples/spa.yml --policy-snippets examples/snippets/
+      - name: Test WASM Runtime Success
+        run: |
+          ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge ./embed/challenge/js-pow-sha256/test/verify-challenge.json -verify-challenge-out 0
+      - name: Test WASM Runtime Fail
+        run: |
+          ./.bin/test-wasm-runtime -wasm ./embed/challenge/js-pow-sha256/runtime/runtime.wasm -make-challenge ./embed/challenge/js-pow-sha256/test/make-challenge.json -make-challenge-out ./embed/challenge/js-pow-sha256/test/make-challenge-out.json -verify-challenge ./embed/challenge/js-pow-sha256/test/verify-challenge-fail.json -verify-challenge-out 1
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log in to Git Forge registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.projectsegfau.lt
+          username: ${{ secrets.GIT_USERNAME }}
+          password: ${{ secrets.GIT_TOKEN }}
+      - name: Build and push Docker images
+        env:
+          SOURCE_DATE_EPOCH: 0
+          TZ: UTC
+        run: |-
+          docker buildx build \
+            --platform linux/amd64,linux/arm64,linux/riscv64 \
+            --tag git.projectsegfau.lt/${{ secrets.GIT_USERNAME }}/go-away:latest \
+            --push \
+            .

Dockerfile

@@ -24,18 +24,26 @@ ENV CGO_ENABLED=0
 ENV GOOS=${TARGETOS}
 ENV GOARCH=${TARGETARCH}
 ENV GOTOOLCHAIN=${GOTOOLCHAIN}
+ENV BUILDMODE=pie
+
+# riscv64 requires GCC for pie buildmode
+# see https://github.com/golang/go/issues/64875
+RUN if [[ "$GOARCH" == "riscv64" ]]; then export BUILDMODE=exe; fi && \
+    go build -v \
+    -pgo=auto \
+    -trimpath -ldflags='-buildid= -bindnow' -buildmode $BUILDMODE \
+    -o "${GOBIN}/go-away" ./cmd/go-away
 
-RUN go build -pgo=auto -v -trimpath -ldflags=-buildid= -o "${GOBIN}/go-away" ./cmd/go-away
 RUN test -e "${GOBIN}/go-away"
 
 
-FROM --platform=$TARGETPLATFORM ${from}
+FROM ${from}
 
 COPY --from=build /go/bin/go-away /bin/go-away
 COPY examples/snippets/ /snippets/
 COPY docker-entrypoint.sh /
 
-ENV TZ UTC
+ENV TZ=UTC
 
 ENV GOAWAY_METRICS_BIND=""
 ENV GOAWAY_DEBUG_BIND=""
@@ -52,7 +60,6 @@ ENV GOAWAY_CHALLENGE_TEMPLATE_LOGO=""
 ENV GOAWAY_SLOG_LEVEL="WARN"
 ENV GOAWAY_CLIENT_IP_HEADER=""
 ENV GOAWAY_BACKEND_IP_HEADER=""
-ENV GOAWAY_JWT_PRIVATE_KEY_SEED=""
 ENV GOAWAY_BACKEND=""
 ENV GOAWAY_ACME_AUTOCERT=""
 ENV GOAWAY_CACHE="/cache"
@@ -63,6 +70,6 @@ EXPOSE 8080/udp
 EXPOSE 9090/tcp
 EXPOSE 6060/tcp
 
-ENV JWT_PRIVATE_KEY_SEED="${GOAWAY_JWT_PRIVATE_KEY_SEED}"
+# Use GOAWAY_JWT_PRIVATE_KEY_SEED or JWT_PRIVATE_KEY_SEED secret mount to expose this value to docker
 
 ENTRYPOINT ["/docker-entrypoint.sh"]

README.md

@@ -3,6 +3,7 @@
 
 Self-hosted abuse detection and rule enforcement against low-effort mass AI scraping and bots. Uses conventional non-nuclear options.
 
+[![Latest Release](https://img.shields.io/gitea/v/release/git/go-away?gitea_url=https%3A%2F%2Fgit.gammaspectra.live)](https://git.gammaspectra.live/git/go-away/releases)
 [![Build Status](https://ci.gammaspectra.live/api/badges/git/go-away/status.svg)](https://ci.gammaspectra.live/git/go-away)
 [![Go Reference](https://pkg.go.dev/badge/git.gammaspectra.live/git/go-away.svg)](https://pkg.go.dev/git.gammaspectra.live/git/go-away)
 

build-wasm.sh

@@ -9,18 +9,17 @@ mkdir -p .bin/ 2>/dev/null
 
 # Setup tinygo first
 if [[ ! -d .bin/tinygo ]]; then
-  git clone --depth=1 --branch v0.37.0 https://github.com/tinygo-org/tinygo.git .bin/tinygo
+  git clone --depth=1 --branch v0.38.0 https://github.com/tinygo-org/tinygo.git .bin/tinygo
   pushd .bin/tinygo
   git submodule update --init --recursive
 
   go mod download -x && go mod verify
 
-  make binaryen STATIC=1
-  make wasi-libc
-
   make llvm-source
   make llvm-build
 
+  make binaryen STATIC=1
+
   make build/release
 else
   pushd .bin/tinygo

cmd/go-away/main.go

@@ -154,7 +154,9 @@ func main() {
 	var seed []byte
 
 	var kValue string
-	if kValue = os.Getenv("JWT_PRIVATE_KEY_SEED"); kValue != "" {
+	if kValue = os.Getenv("GOAWAY_JWT_PRIVATE_KEY_SEED"); kValue != "" {
+		// prefer first
+	} else if kValue = os.Getenv("JWT_PRIVATE_KEY_SEED"); kValue != "" {
 
 	} else if *jwtPrivateKeySeed != "" {
 		kValue = *jwtPrivateKeySeed
@@ -210,7 +212,7 @@ func main() {
 			fatal(fmt.Errorf("backend %s: failed to make reverse proxy: %w", k, err))
 		}
 
-		backend.ErrorLog = slog.NewLogLogger(slog.With("backend", k).Handler(), slog.LevelError)
+		backend.ErrorLog = slog.NewLogLogger(slog.With("backend", k).Handler(), slog.LevelDebug)
 		createdBackends[k] = backend
 	}
 
@@ -291,7 +293,7 @@ func main() {
 		fatal(fmt.Errorf("failed to create server: %w", err))
 	}
 
-	server.ErrorLog = slog.NewLogLogger(slog.With("server", "http").Handler(), slog.LevelError)
+	server.ErrorLog = slog.NewLogLogger(slog.With("server", "http").Handler(), slog.LevelDebug)
 
 	go func() {
 		handler, err := loadPolicyState()
@@ -302,6 +304,7 @@ func main() {
 		swap(handler)
 		slog.Warn(
 			"handler configuration loaded",
+			"key_fingerprint", hex.EncodeToString(handler.PrivateKeyFingerprint()),
 		)
 
 		// allow reloading from now on
@@ -336,7 +339,7 @@ func main() {
 			debugServer := http.Server{
 				Addr:     opt.BindDebug,
 				Handler:  mux,
-				ErrorLog: slog.NewLogLogger(slog.With("server", "debug").Handler(), slog.LevelError),
+				ErrorLog: slog.NewLogLogger(slog.With("server", "debug").Handler(), slog.LevelDebug),
 			}
 
 			slog.Warn(
@@ -356,7 +359,7 @@ func main() {
 			metricsServer := http.Server{
 				Addr:     opt.BindMetrics,
 				Handler:  mux,
-				ErrorLog: slog.NewLogLogger(slog.With("server", "metrics").Handler(), slog.LevelError),
+				ErrorLog: slog.NewLogLogger(slog.With("server", "metrics").Handler(), slog.LevelDebug),
 			}
 
 			slog.Warn(

embed/templates/challenge-redlib.gohtml

@@ -0,0 +1,62 @@
+<!DOCTYPE html>
+<html lang="en" class="fixed_navbar">
+<head>
+    <title>{{ .Title }}</title>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {{ range .MetaTags }}
+    <meta {{ range $key, $value := . }}{{ $key | attr }}="{{ $value }}" {{end}}/>
+    {{ end }}
+    {{ range .LinkTags }}
+    <link {{ range $key, $value := . }}{{ $key | attr }}="{{ $value }}" {{end}}/>
+    {{ end }}
+    {{ range .HeaderTags }}
+    {{ . }}
+    {{ end }}
+    <link rel="stylesheet" type="text/css" href="/style.css?v=0.36.0">
+</head>
+
+<body class="fixed_navbar">
+    <!-- NAVIGATION BAR -->
+    <nav class="fixed_navbar">
+            <div id="logo">
+            <a id="redlib" href="/"><span id="red">red</span><span id="lib">lib.</span></a>
+        </div>
+    </nav>
+
+    <!-- MAIN CONTENT -->
+    <main>
+        <div id="error">
+        <h1 id="status">Please wait while we verify you aren't a robot!</h1>
+
+            {{ if .Challenge }}
+            <h3 id="status">{{ .Strings.Get "status_loading_challenge" }} : {{.Challenge }}...</h3>
+            {{ else if .Error }}
+                <h3 id="status">{{ .Strings.Get "status_error" }} {{ .Error }}</h3>
+            {{ else }}
+                <h3 id="status">{{ .Strings.Get "status_loading" }}</h3>
+            {{ end }}
+        <details style="padding-top: 5px;">
+            <summary>{{ .Strings.Get "details_title" }}</summary>
+
+            {{.Strings.Get "details_text"}}
+        </details>
+            {{ if .Redirect }}
+                <h3><a href="{{ .Redirect }}">{{ .Strings.Get "button_refresh_page" }}</a></h3>
+        </div>
+        {{ end }}
+                {{if .EndTags }}
+            <noscript>
+                {{ .Strings.Get "noscript_warning" }}
+            </noscript>
+        {{end}}
+    </main>
+
+    <!-- FOOTER -->
+    <footer>
+        <div class="footer-buttons">
+        <p><small>{{ .Strings.Get "details_contact_admin_with_request_id" }}: <em>{{ .Id }}</em></small></p>
+        </div>
+    </footer>
+</body>
+</html>

examples/config.yml

@@ -24,7 +24,7 @@ bind:
 #bind-debug: ":6060"
 
 # Bind the Prometheus metrics onto /metrics path on this port
-#bind-metrics ":9090"
+#bind-metrics: ":9090"
 
 # These links will be shown on the presented challenge or error pages
 links:

examples/forgejo.yml

@@ -50,7 +50,7 @@ conditions:
 
   is-suspicious-crawler:
     # TLS Fingerprint for specific agent without ALPN
-    - '(userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")) && ("ja4" in fp && fp.ja4.matches("^t[0-9a-z]+00_")) && !(userAgent.contains("facebookexternalhit/") || userAgent.contains("Twitterbot/"))'
+    - '(userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")) && ("ja4" in fp && fp.ja4.matches("^t[0-9a-z]+00_")) && !(userAgent.contains("compatible;") || userAgent.contains("+http") || userAgent.contains("facebookexternalhit/") || userAgent.contains("Twitterbot/"))'
     # Old engines
     - 'userAgent.contains("Presto/") || userAgent.contains("Trident/")'
     # Old IE browsers
@@ -104,6 +104,15 @@ rules:
       - *is-bot-yandexbot
     action: pass
 
+  # Matches private networks and localhost.
+  # Uncomment this if you want to let your own tools this way
+  #  - name: allow-private-networks
+  #    conditions:
+  #      # Allows localhost and private networks CIDR
+  #      - *is-network-localhost
+  #      - *is-network-private
+  #    action: pass
+
   - name: undesired-networks
     conditions:
       - 'remoteAddress.network("huawei-cloud") || remoteAddress.network("alibaba-cloud") || remoteAddress.network("zenlayer-inc")'
@@ -147,7 +156,7 @@ rules:
       - name: 0
         action: check
         settings:
-          challenges: [js-pow-sha256, http-cookie-check]
+          challenges: [js-refresh, http-cookie-check]
       - name: 1
         action: check
         settings:
@@ -173,7 +182,7 @@ rules:
       - 'path.matches("^/[^/]+/[^/]+/archive/.*\\.(bundle|zip|tar\\.gz)") && ($is-generic-browser)'
     action: challenge
     settings:
-      challenges: [ js-pow-sha256 ]
+      challenges: [ js-refresh ]
 
   - name: allow-git-operations
     conditions:
@@ -242,18 +251,11 @@ rules:
       - name: 0
         action: check
         settings:
-          challenges: [preload-link, header-refresh, js-pow-sha256, http-cookie-check]
+          challenges: [preload-link, header-refresh, js-refresh, http-cookie-check]
       - name: 1
         action: check
         settings:
-          challenges: [ resource-load, js-pow-sha256, http-cookie-check ]
+          challenges: [ resource-load, js-refresh, http-cookie-check ]
-
-  - name: standard-bots
-    action: check
-    settings:
-      challenges: [meta-refresh, resource-load]
-    conditions:
-      - '($is-generic-robot-ua)'
 
   # Allow all source downloads not caught in browser above
   # todo: limit this as needed?
@@ -274,7 +276,7 @@ rules:
       # if DNSBL fails, check additional challenges
       fail: check
       fail-settings:
-        challenges: [js-pow-sha256, http-cookie-check]
+        challenges: [js-refresh, http-cookie-check]
 
   # Allow PUT/DELETE/PATCH/POST requests in general
   - name: non-get-request
@@ -294,7 +296,7 @@ rules:
         # Map OpenGraph or similar <meta> tags back to the reply, even if denied/challenged
         proxy-meta-tags: "true"
         # proxy-safe-link-tags: "true"
-        
+
       # Set additional response headers
       #response-headers:
       # X-Clacks-Overhead:
@@ -321,7 +323,7 @@ rules:
   - name: standard-browser
     action: challenge
     settings:
-      challenges: [http-cookie-check, preload-link, meta-refresh, resource-load, js-pow-sha256]
+      challenges: [http-cookie-check, preload-link, meta-refresh, resource-load, js-refresh, js-pow-sha256]
     conditions:
       - '($is-generic-browser)'
 

examples/generic.yml

@@ -10,7 +10,7 @@ networks:
 
 challenges:
   # Challenges will get included from snippets
-  
+
 conditions:
   # Conditions will get replaced on rules AST when found as ($condition-name)
 
@@ -27,7 +27,7 @@ conditions:
     # Old IE browsers
     - 'userAgent.matches("MSIE ([2-9]|10|11)\\.")'
     # Old Linux browsers
-    - 'userAgent.contains("Linux i[63]86") || userAgent.contains("FreeBSD i[63]86")'
+    - 'userAgent.matches("Linux i[63]86") || userAgent.matches("FreeBSD i[63]86")'
     # Old Windows browsers
     - 'userAgent.matches("Windows (3|95|98|CE)") || userAgent.matches("Windows NT [1-5]\\.")'
     # Old mobile browsers
@@ -60,6 +60,15 @@ rules:
       - *is-bot-yandexbot
     action: pass
 
+  # Matches private networks and localhost.
+  # Uncomment this if you want to let your own tools this way
+  #  - name: allow-private-networks
+  #    conditions:
+  #      # Allows localhost and private networks CIDR
+  #      - *is-network-localhost
+  #      - *is-network-private
+  #    action: pass
+
   - name: undesired-crawlers
     conditions:
       - '($is-headless-chromium)'
@@ -98,7 +107,7 @@ rules:
       - name: 0
         action: check
         settings:
-          challenges: [js-pow-sha256]
+          challenges: [js-refresh]
       - name: 1
         action: check
         settings:
@@ -122,12 +131,12 @@ rules:
       # if DNSBL fails, check additional challenges
       fail: check
       fail-settings:
-        challenges: [js-pow-sha256]
+        challenges: [js-refresh]
 
   - name: suspicious-fetchers
     action: check
     settings:
-      challenges: [js-pow-sha256]
+      challenges: [js-refresh]
     conditions:
       - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
 
@@ -170,7 +179,7 @@ rules:
   - name: standard-browser
     action: challenge
     settings:
-      challenges: [preload-link, meta-refresh, resource-load, js-pow-sha256]
+      challenges: [preload-link, meta-refresh, resource-load, js-refresh]
     conditions:
       - '($is-generic-browser)'
 

examples/snippets/challenge-js-refresh.yml

@@ -0,0 +1,6 @@
+challenges:
+  js-refresh:
+    # Challenges with a redirect via window.location (requires HTML parsing and JavaScript logic)
+    runtime: "refresh"
+    parameters:
+      refresh-via: "javascript"

examples/snippets/networks-private.yml

@@ -0,0 +1,22 @@
+networks:
+  localhost:
+    # localhost and loopback addresses
+    - prefixes:
+      - "127.0.0.0/8"
+      - "::1/128"
+  private:
+    # Private network CIDR blocks
+    - prefixes:
+        # private networks
+        - "10.0.0.0/8"
+        - "172.16.0.0/12"
+        - "192.168.0.0/16"
+        - "fc00::/7"
+        # CGNAT
+        - "100.64.0.0/10"
+
+conditions:
+  is-network-localhost:
+    - &is-network-localhost 'remoteAddress.network("localhost")'
+  is-network-private:
+    - &is-network-private 'remoteAddress.network("private")'

lib/action/block.go

@@ -28,7 +28,9 @@ func (a Block) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Reques
 	data := challenge.RequestDataFromContext(r.Context())
 
 	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
 	w.Header().Set("Connection", "close")
+
 	data.ResponseHeaders(w)
 	w.WriteHeader(a.Code)
 	_, _ = w.Write([]byte(fmt.Errorf("access blocked: blocked by administrative rule %s/%s", data.Id.String(), a.RuleHash).Error()))

lib/action/code.go

@@ -42,7 +42,11 @@ type CodeSettings struct {
 type Code int
 
 func (a Code) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request, done func() (backend http.Handler)) (next bool, err error) {
-	challenge.RequestDataFromContext(r.Context()).ResponseHeaders(w)
+	data := challenge.RequestDataFromContext(r.Context())
+
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+
+	data.ResponseHeaders(w)
 
 	w.WriteHeader(int(a))
 	return false, nil

lib/action/drop.go

@@ -33,6 +33,8 @@ func (a Drop) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Request
 	w.Header().Set("Content-Type", "text/plain")
 	w.Header().Set("Content-Length", "0")
 	w.Header().Set("Connection", "close")
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+
 	w.WriteHeader(http.StatusForbidden)
 
 	return false, nil

lib/challenge/data.go

@@ -295,8 +295,8 @@ func (d *RequestData) EvaluateChallenges(w http.ResponseWriter, r *http.Request)
 	challengeMap, err := d.verifyChallengeState()
 	if err != nil {
 		if !errors.Is(err, http.ErrNoCookie) {
-			//clear invalid cookie and continue
+			//queue resend invalid cookie and continue
-			utils.ClearCookie(d.cookieName, w, r)
+			d.challengeMapModified = true
 		}
 		challengeMap = make(TokenChallengeMap)
 	}
@@ -329,6 +329,9 @@ func (d *RequestData) ResponseHeaders(w http.ResponseWriter) {
 	//w.Header().Set("Accept-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
 	//w.Header().Set("Critical-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
 
+	// send Vary header to mark that response may vary based on Cookie values and other client headers
+	w.Header().Set("Vary", "Cookie, Accept, Accept-Encoding, Accept-Language, User-Agent")
+
 	if d.State.Settings().MainName != "" {
 		w.Header().Add("Via", fmt.Sprintf("%s %s@%s", d.r.Proto, d.State.Settings().MainName, d.State.Settings().MainVersion))
 	}
@@ -350,7 +353,7 @@ func (d *RequestData) RequestHeaders(headers http.Header) {
 		if d.State.Settings().ClientIpHeader != "" {
 			headers.Del(d.State.Settings().ClientIpHeader)
 		}
-		headers.Set(d.State.Settings().BackendIpHeader, d.RemoteAddress.String())
+		headers.Set(d.State.Settings().BackendIpHeader, d.RemoteAddress.Addr().Unmap().String())
 	}
 
 	for id, result := range d.ChallengeVerify {
@@ -365,7 +368,7 @@ func (d *RequestData) RequestHeaders(headers http.Header) {
 		}
 	}
 
-	if ja4, ok := d.fp["fp4"]; ok {
+	if ja4, ok := d.fp["ja4"]; ok {
 		headers.Set("X-TLS-Fingerprint-JA4", ja4)
 	}
 

lib/challenge/helper.go

@@ -8,7 +8,9 @@ import (
 	"git.gammaspectra.live/git/go-away/utils"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
+	"time"
 )
 
 var ErrInvalidToken = errors.New("invalid token")
@@ -47,6 +49,7 @@ const (
 	QueryArgRequestId = QueryArgPrefix + "_id"
 	QueryArgChallenge = QueryArgPrefix + "_challenge"
 	QueryArgToken     = QueryArgPrefix + "_token"
+	QueryArgBust      = QueryArgPrefix + "_bust"
 )
 
 const MakeChallengeUrlSuffix = "/make-challenge"
@@ -91,12 +94,13 @@ func VerifyUrl(r *http.Request, reg *Registration, token string) (*url.URL, erro
 	uri.Path = reg.Path + VerifyChallengeUrlSuffix
 
 	data := RequestDataFromContext(r.Context())
-	values := uri.Query()
+	values, _ := utils.ParseRawQuery(r.URL.RawQuery)
-	values.Set(QueryArgRequestId, data.Id.String())
+	values.Set(QueryArgRequestId, url.QueryEscape(data.Id.String()))
-	values.Set(QueryArgRedirect, redirectUrl.String())
+	values.Set(QueryArgRedirect, url.QueryEscape(redirectUrl.String()))
-	values.Set(QueryArgToken, token)
+	values.Set(QueryArgToken, url.QueryEscape(token))
-	values.Set(QueryArgChallenge, reg.Name)
+	values.Set(QueryArgChallenge, url.QueryEscape(reg.Name))
-	uri.RawQuery = values.Encode()
+	values.Set(QueryArgBust, url.QueryEscape(strconv.FormatInt(time.Now().UTC().UnixMilli(), 10)))
+	uri.RawQuery = utils.EncodeRawQuery(values)
 
 	return uri, nil
 }
@@ -108,13 +112,13 @@ func RedirectUrl(r *http.Request, reg *Registration) (*url.URL, error) {
 	}
 
 	data := RequestDataFromContext(r.Context())
-	values := uri.Query()
+	values, _ := utils.ParseRawQuery(r.URL.RawQuery)
-	values.Set(QueryArgRequestId, data.Id.String())
+	values.Set(QueryArgRequestId, url.QueryEscape(data.Id.String()))
 	if ref := r.Referer(); ref != "" {
-		values.Set(QueryArgReferer, r.Referer())
+		values.Set(QueryArgReferer, url.QueryEscape(r.Referer()))
 	}
-	values.Set(QueryArgChallenge, reg.Name)
+	values.Set(QueryArgChallenge, url.QueryEscape(reg.Name))
-	uri.RawQuery = values.Encode()
+	uri.RawQuery = utils.EncodeRawQuery(values)
 
 	return uri, nil
 }

lib/challenge/http/http.go

@@ -108,12 +108,14 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 			}
 		}
 
+		data := challenge.RequestDataFromContext(r.Context())
+
 		request, err := http.NewRequest(params.HttpMethod, params.Url, nil)
 		if err != nil {
 			return challenge.VerifyResultFail
 		}
 
-		var excludeHeaders = []string{"Host", "Content-Length"}
+		var excludeHeaders = []string{"Host", "Content-Length", "Upgrade", "Accept-Encoding", "Range"}
 		for k, v := range r.Header {
 			if slices.Contains(excludeHeaders, k) {
 				// skip these parameters
@@ -121,10 +123,12 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 			}
 			request.Header[k] = v
 		}
-		// set id
+
-		request.Header.Set("X-Away-Id", challenge.RequestDataFromContext(r.Context()).Id.String())
+		// set id, ip, and other headers
+		data.RequestHeaders(request.Header)
 
 		// set request info in X headers
+		request.Header.Set("X-Away-Method", r.Method)
 		request.Header.Set("X-Away-Host", r.Host)
 		request.Header.Set("X-Away-Path", r.URL.Path)
 		request.Header.Set("X-Away-Query", r.URL.RawQuery)
@@ -136,8 +140,6 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 		defer response.Body.Close()
 		defer io.Copy(io.Discard, response.Body)
 
-		data := challenge.RequestDataFromContext(r.Context())
-
 		if response.StatusCode != params.HttpCode {
 			data.IssueChallengeToken(reg, key, sum, expiry, false)
 			return challenge.VerifyResultNotOK

lib/challenge/key.go

@@ -52,15 +52,13 @@ func GetChallengeKeyForRequest(state StateInterface, reg *Registration, until ti
 	hasher.Write([]byte{0})
 
 	// specific headers
-	for _, k := range []string{
+	for _, k := range reg.KeyHeaders {
-		"Accept-Language",
+		hasher.Write([]byte(k))
-		// General browser information
+		hasher.Write([]byte{0})
-		"User-Agent",
+		for _, v := range r.Header.Values(k) {
-		// TODO: not sent in preload
+			hasher.Write([]byte(v))
-		//"Sec-Ch-Ua",
+			hasher.Write([]byte{1})
-		//"Sec-Ch-Ua-Platform",
+		}
-	} {
-		hasher.Write([]byte(r.Header.Get(k)))
 		hasher.Write([]byte{0})
 	}
 	hasher.Write([]byte{0})

lib/challenge/preload-link/preload-link.go

@@ -44,6 +44,9 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 
 	reg.Class = challenge.ClassTransparent
 
+	// some of regular headers are not sent in default headers
+	reg.KeyHeaders = challenge.MinimalKeyHeaders
+
 	ob := challenge.NewAwaiter[string]()
 
 	reg.Object = ob
@@ -66,9 +69,9 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 		}
 
 		// remove redirect args
-		values := uri.Query()
+		values, _ := utils.ParseRawQuery(uri.RawQuery)
 		values.Del(challenge.QueryArgRedirect)
-		uri.RawQuery = values.Encode()
+		uri.RawQuery = utils.EncodeRawQuery(values)
 
 		// Redirect URI must be absolute to work
 		uri.Scheme = utils.GetRequestScheme(r)
@@ -98,6 +101,7 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 
 	mux.HandleFunc("GET "+reg.Path+challenge.VerifyChallengeUrlSuffix, func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/css; charset=utf-8")
+		w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
 		w.Header().Set("Content-Length", "0")
 
 		data := challenge.RequestDataFromContext(r.Context())

lib/challenge/refresh/refresh.go

@@ -1,9 +1,12 @@
 package refresh
 
 import (
+	"encoding/json"
+	"fmt"
 	"git.gammaspectra.live/git/go-away/lib/challenge"
 	"github.com/goccy/go-yaml"
 	"github.com/goccy/go-yaml/ast"
+	"html/template"
 	"net/http"
 	"time"
 )
@@ -45,7 +48,17 @@ func FillRegistration(state challenge.StateInterface, reg *challenge.Registratio
 			return challenge.VerifyResultFail
 		}
 
-		if params.Mode == "meta" {
+		if params.Mode == "javascript" {
+			data, err := json.Marshal(uri.String())
+			if err != nil {
+				return challenge.VerifyResultFail
+			}
+			state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
+				"EndTags": []template.HTML{
+					template.HTML(fmt.Sprintf("<script type=\"text/javascript\">window.location = %s;</script>", string(data))),
+				},
+			})
+		} else if params.Mode == "meta" {
 			state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
 				"MetaTags": []map[string]string{
 					{

lib/challenge/register.go

@@ -35,6 +35,24 @@ var idCounter Id
 // DefaultDuration TODO: adjust
 const DefaultDuration = time.Hour * 24 * 7
 
+var DefaultKeyHeaders = []string{
+	// General browser information
+	"User-Agent",
+	// Accept headers
+	"Accept-Language",
+	"Accept-Encoding",
+
+	// NOTE: not sent in preload
+	"Sec-Ch-Ua",
+	"Sec-Ch-Ua-Platform",
+}
+
+var MinimalKeyHeaders = []string{
+	"Accept-Language",
+	// General browser information
+	"User-Agent",
+}
+
 func (r Register) Create(state StateInterface, name string, pol policy.Challenge, replacer *strings.Replacer) (*Registration, Id, error) {
 	runtime, ok := Runtimes[pol.Runtime]
 	if !ok {
@@ -42,9 +60,10 @@ func (r Register) Create(state StateInterface, name string, pol policy.Challenge
 	}
 
 	reg := &Registration{
-		Name:     name,
+		Name:       name,
-		Path:     path.Join(state.UrlPath(), "challenge", name),
+		Path:       path.Join(state.UrlPath(), "challenge", name),
-		Duration: pol.Duration,
+		Duration:   pol.Duration,
+		KeyHeaders: DefaultKeyHeaders,
 	}
 
 	if reg.Duration == 0 {
@@ -126,6 +145,9 @@ type Registration struct {
 	Verify            VerifyFunc
 	VerifyProbability float64
 
+	// KeyHeaders The client headers used in key generation, in this order
+	KeyHeaders []string
+
 	// IssueChallenge Issues a challenge to a request.
 	// If Class is ClassTransparent and VerifyResult is !VerifyResult.Ok(), continue with other challenges
 	// TODO: have this return error as well

lib/challenge/resource-load/resource-load.go

@@ -23,9 +23,13 @@ func FillRegistrationHeader(state challenge.StateInterface, reg *challenge.Regis
 			return challenge.VerifyResultFail
 		}
 
+		redirectUri, err := challenge.RedirectUrl(r, reg)
+		if err != nil {
+			return challenge.VerifyResultFail
+		}
 		// self redirect!
 		//TODO: adjust deadline
-		w.Header().Set("Refresh", "2; url="+r.URL.String())
+		w.Header().Set("Refresh", "2; url="+redirectUri.String())
 
 		state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
 			"LinkTags": []map[string]string{
@@ -44,6 +48,7 @@ func FillRegistrationHeader(state challenge.StateInterface, reg *challenge.Regis
 	mux.HandleFunc("GET "+reg.Path+challenge.VerifyChallengeUrlSuffix, challenge.VerifyHandlerFunc(state, reg, nil, func(state challenge.StateInterface, data *challenge.RequestData, w http.ResponseWriter, r *http.Request, verifyResult challenge.VerifyResult, err error, redirect string) {
 		//TODO: add other types inside css that need to be loaded!
 		w.Header().Set("Content-Type", "text/css; charset=utf-8")
+		w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
 		w.Header().Set("Content-Length", "0")
 
 		data.ResponseHeaders(w)

lib/challenge/script.go

@@ -23,6 +23,7 @@ func ServeChallengeScript(w http.ResponseWriter, r *http.Request, reg *Registrat
 		//TODO: log
 		panic(err)
 	}
+
 	data.ResponseHeaders(w)
 	w.WriteHeader(http.StatusOK)
 
@@ -30,7 +31,7 @@ func ServeChallengeScript(w http.ResponseWriter, r *http.Request, reg *Registrat
 		"Id":              data.Id.String(),
 		"Path":            reg.Path,
 		"Parameters":      paramData,
-		"Random":          utils.CacheBust(),
+		"Random":          utils.StaticCacheBust(),
 		"Challenge":       reg.Name,
 		"ChallengeScript": script,
 		"Strings":         data.State.Strings(),

lib/challenge/wasm/registration.go

@@ -97,7 +97,7 @@ func FillJavaScriptRegistration(state challenge.StateInterface, reg *challenge.R
 	reg.IssueChallenge = func(w http.ResponseWriter, r *http.Request, key challenge.Key, expiry time.Time) challenge.VerifyResult {
 		state.ChallengePage(w, r, state.Settings().ChallengeResponseCode, reg, map[string]any{
 			"EndTags": []template.HTML{
-				template.HTML(fmt.Sprintf("<script async type=\"module\" src=\"%s?cacheBust=%s\"></script>", reg.Path+"/script.mjs", utils.CacheBust())),
+				template.HTML(fmt.Sprintf("<script async type=\"module\" src=\"%s?cacheBust=%s\"></script>", reg.Path+"/script.mjs", utils.StaticCacheBust())),
 			},
 		})
 		return challenge.VerifyResultNone
@@ -164,6 +164,9 @@ func FillJavaScriptRegistration(state challenge.StateInterface, reg *challenge.R
 				w.Header()[k] = v
 			}
 			w.Header().Set("Content-Length", fmt.Sprintf("%d", len(out.Data)))
+			w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+
+			data.ResponseHeaders(w)
 			w.WriteHeader(out.Code)
 			_, _ = w.Write(out.Data)
 			return nil

lib/http.go

@@ -246,19 +246,20 @@ func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 		if fromChallenge {
 			r.Header.Del("Referer")
 		}
-		q := r.URL.Query()
 
+		q := r.URL.Query()
 		if ref := q.Get(challenge.QueryArgReferer); ref != "" {
 			r.Header.Set("Referer", ref)
 		}
 
+		rawQ, _ := utils.ParseRawQuery(r.URL.RawQuery)
 		// delete query parameters that were set by go-away
-		for k := range q {
+		for k := range rawQ {
 			if strings.HasPrefix(k, challenge.QueryArgPrefix) {
-				q.Del(k)
+				rawQ.Del(k)
 			}
 		}
-		r.URL.RawQuery = q.Encode()
+		r.URL.RawQuery = utils.EncodeRawQuery(rawQ)
 
 		data.ExtraHeaders.Set("X-Away-Rule", ruleName)
 		data.ExtraHeaders.Set("X-Away-Action", string(ruleAction))

lib/settings/backend.go

@@ -106,6 +106,12 @@ func (b Backend) Create() (*httputil.ReverseProxy, error) {
 	if b.IpHeader != "" || b.Host != "" || !b.Transparent {
 		director := proxy.Director
 		proxy.Director = func(req *http.Request) {
+			if !b.Transparent {
+				if data := challenge.RequestDataFromContext(req.Context()); data != nil {
+					data.RequestHeaders(req.Header)
+				}
+			}
+
 			if b.IpHeader != "" && !b.Transparent {
 				if ip := utils.GetRemoteAddress(req.Context()); ip != nil {
 					req.Header.Set(b.IpHeader, ip.Addr().Unmap().String())
@@ -114,12 +120,6 @@ func (b Backend) Create() (*httputil.ReverseProxy, error) {
 			if b.Host != "" {
 				req.Host = b.Host
 			}
-
-			if !b.Transparent {
-				if data := challenge.RequestDataFromContext(req.Context()); data != nil {
-					data.RequestHeaders(req.Header)
-				}
-			}
 			director(req)
 		}
 	}

lib/state.go

@@ -114,9 +114,9 @@ func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSetti
 				return nil, fmt.Errorf("error loading template %s: %w", state.opt.ChallengeTemplate, err)
 			}
 			state.opt.ChallengeTemplate = name
+		} else {
+			return nil, fmt.Errorf("no template defined for %s", state.opt.ChallengeTemplate)
 		}
-
-		return nil, fmt.Errorf("no template defined for %s", state.opt.ChallengeTemplate)
 	}
 
 	state.networks = make(map[string]func() cidranger.Ranger)

lib/template.go

@@ -78,7 +78,7 @@ func (state *State) ChallengePage(w http.ResponseWriter, r *http.Request, status
 	data := challenge.RequestDataFromContext(r.Context())
 	input := make(map[string]any)
 	input["Id"] = data.Id.String()
-	input["Random"] = utils.CacheBust()
+	input["Random"] = utils.StaticCacheBust()
 
 	input["Path"] = state.UrlPath()
 	input["Links"] = state.opt.Links
@@ -100,6 +100,7 @@ func (state *State) ChallengePage(w http.ResponseWriter, r *http.Request, status
 	state.addCachedTags(data, r, input)
 
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
 
 	buf := bytes.NewBuffer(make([]byte, 0, 8192))
 
@@ -116,12 +117,13 @@ func (state *State) ChallengePage(w http.ResponseWriter, r *http.Request, status
 func (state *State) ErrorPage(w http.ResponseWriter, r *http.Request, status int, err error, redirect string) {
 	data := challenge.RequestDataFromContext(r.Context())
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
 
 	buf := bytes.NewBuffer(make([]byte, 0, 8192))
 
 	input := map[string]any{
 		"Id":        data.Id.String(),
-		"Random":    utils.CacheBust(),
+		"Random":    utils.StaticCacheBust(),
 		"Error":     err.Error(),
 		"Path":      state.UrlPath(),
 		"Theme":     "",

utils/http.go

@@ -7,11 +7,13 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"maps"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/netip"
 	"net/url"
+	"slices"
 	"strings"
 	"time"
 )
@@ -167,15 +169,51 @@ func GetRemoteAddress(ctx context.Context) *netip.AddrPort {
 	return &ip
 }
 
-func CacheBust() string {
+func RandomCacheBust(n int) string {
-	return cacheBust
+	buf := make([]byte, n)
-}
-
-var cacheBust string
-
-func init() {
-
-	buf := make([]byte, 16)
 	_, _ = rand.Read(buf)
-	cacheBust = base64.RawURLEncoding.EncodeToString(buf)
+	return base64.RawURLEncoding.EncodeToString(buf)
+}
+
+var staticCacheBust = RandomCacheBust(16)
+
+func StaticCacheBust() string {
+	return staticCacheBust
+}
+
+func ParseRawQuery(rawQuery string) (m url.Values, err error) {
+	m = make(url.Values)
+	for rawQuery != "" {
+		var key string
+		key, rawQuery, _ = strings.Cut(rawQuery, "&")
+		if strings.Contains(key, ";") {
+			err = fmt.Errorf("invalid semicolon separator in query")
+			continue
+		}
+		if key == "" {
+			continue
+		}
+		key, value, _ := strings.Cut(key, "=")
+		m[key] = append(m[key], value)
+	}
+	return m, err
+}
+
+func EncodeRawQuery(v url.Values) string {
+	if len(v) == 0 {
+		return ""
+	}
+	var buf strings.Builder
+	for _, k := range slices.Sorted(maps.Keys(v)) {
+		vs := v[k]
+		for _, v := range vs {
+			if buf.Len() > 0 {
+				buf.WriteByte('&')
+			}
+			buf.WriteString(k)
+			buf.WriteByte('=')
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
 }