Minimize wasm runtime external dependencies, do not use JSON on verify-challenge output

2025-04-01 05:29:00 +02:00
parent 8d45668d05
commit bfcb0ccada
10 changed files with 470 additions and 91 deletions
--- a/challenge/inline/hex.go
+++ b/challenge/inline/hex.go
@@ -0,0 +1,95 @@
+package inline
+
+import (
+	"errors"
+)
+
+const (
+	hextable        = "0123456789abcdef"
+	reverseHexTable = "" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" +
+		"\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+		"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+)
+
+// EncodedLen returns the length of an encoding of n source bytes.
+// Specifically, it returns n * 2.
+func EncodedLen(n int) int { return n * 2 }
+
+// Encode encodes src into [EncodedLen](len(src))
+// bytes of dst. As a convenience, it returns the number
+// of bytes written to dst, but this value is always [EncodedLen](len(src)).
+// Encode implements hexadecimal encoding.
+func Encode(dst, src []byte) int {
+	j := 0
+	for _, v := range src {
+		dst[j] = hextable[v>>4]
+		dst[j+1] = hextable[v&0x0f]
+		j += 2
+	}
+	return len(src) * 2
+}
+
+// ErrLength reports an attempt to decode an odd-length input
+// using [Decode] or [DecodeString].
+// The stream-based Decoder returns [io.ErrUnexpectedEOF] instead of ErrLength.
+var ErrLength = errors.New("encoding/hex: odd length hex string")
+
+// InvalidByteError values describe errors resulting from an invalid byte in a hex string.
+type InvalidByteError byte
+
+func (e InvalidByteError) Error() string {
+	return "encoding/hex: invalid byte"
+}
+
+// DecodedLen returns the length of a decoding of x source bytes.
+// Specifically, it returns x / 2.
+func DecodedLen(x int) int { return x / 2 }
+
+// Decode decodes src into [DecodedLen](len(src)) bytes,
+// returning the actual number of bytes written to dst.
+//
+// Decode expects that src contains only hexadecimal
+// characters and that src has even length.
+// If the input is malformed, Decode returns the number
+// of bytes decoded before the error.
+func Decode(dst, src []byte) (int, error) {
+	i, j := 0, 1
+	for ; j < len(src); j += 2 {
+		p := src[j-1]
+		q := src[j]
+
+		a := reverseHexTable[p]
+		b := reverseHexTable[q]
+		if a > 0x0f {
+			return i, InvalidByteError(p)
+		}
+		if b > 0x0f {
+			return i, InvalidByteError(q)
+		}
+		dst[i] = (a << 4) | b
+		i++
+	}
+	if len(src)%2 == 1 {
+		// Check for invalid char before reporting bad length,
+		// since the invalid char (if present) is an earlier problem.
+		if reverseHexTable[src[j-1]] > 0x0f {
+			return i, InvalidByteError(src[j-1])
+		}
+		return i, ErrLength
+	}
+	return i, nil
+}
--- a/challenge/inline/mime.go
+++ b/challenge/inline/mime.go
@@ -0,0 +1,234 @@
+package inline
+
+// from textproto
+
+// A MIMEHeader represents a MIME-style header mapping
+// keys to sets of values.
+type MIMEHeader map[string][]string
+
+// Add adds the key, value pair to the header.
+// It appends to any existing values associated with key.
+func (h MIMEHeader) Add(key, value string) {
+	key = CanonicalMIMEHeaderKey(key)
+	h[key] = append(h[key], value)
+}
+
+// Set sets the header entries associated with key to
+// the single element value. It replaces any existing
+// values associated with key.
+func (h MIMEHeader) Set(key, value string) {
+	h[CanonicalMIMEHeaderKey(key)] = []string{value}
+}
+
+// Get gets the first value associated with the given key.
+// It is case insensitive; [CanonicalMIMEHeaderKey] is used
+// to canonicalize the provided key.
+// If there are no values associated with the key, Get returns "".
+// To use non-canonical keys, access the map directly.
+func (h MIMEHeader) Get(key string) string {
+	if h == nil {
+		return ""
+	}
+	v := h[CanonicalMIMEHeaderKey(key)]
+	if len(v) == 0 {
+		return ""
+	}
+	return v[0]
+}
+
+// Values returns all values associated with the given key.
+// It is case insensitive; [CanonicalMIMEHeaderKey] is
+// used to canonicalize the provided key. To use non-canonical
+// keys, access the map directly.
+// The returned slice is not a copy.
+func (h MIMEHeader) Values(key string) []string {
+	if h == nil {
+		return nil
+	}
+	return h[CanonicalMIMEHeaderKey(key)]
+}
+
+// Del deletes the values associated with key.
+func (h MIMEHeader) Del(key string) {
+	delete(h, CanonicalMIMEHeaderKey(key))
+}
+
+// CanonicalMIMEHeaderKey returns the canonical format of the
+// MIME header key s. The canonicalization converts the first
+// letter and any letter following a hyphen to upper case;
+// the rest are converted to lowercase. For example, the
+// canonical key for "accept-encoding" is "Accept-Encoding".
+// MIME header keys are assumed to be ASCII only.
+// If s contains a space or invalid header field bytes, it is
+// returned without modifications.
+func CanonicalMIMEHeaderKey(s string) string {
+	// Quick check for canonical encoding.
+	upper := true
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if !validHeaderFieldByte(c) {
+			return s
+		}
+		if upper && 'a' <= c && c <= 'z' {
+			s, _ = canonicalMIMEHeaderKey([]byte(s))
+			return s
+		}
+		if !upper && 'A' <= c && c <= 'Z' {
+			s, _ = canonicalMIMEHeaderKey([]byte(s))
+			return s
+		}
+		upper = c == '-'
+	}
+	return s
+}
+
+const toLower = 'a' - 'A'
+
+// validHeaderFieldByte reports whether c is a valid byte in a header
+// field name. RFC 7230 says:
+//
+//	header-field   = field-name ":" OWS field-value OWS
+//	field-name     = token
+//	tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
+//	        "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
+//	token = 1*tchar
+func validHeaderFieldByte(c byte) bool {
+	// mask is a 128-bit bitmap with 1s for allowed bytes,
+	// so that the byte c can be tested with a shift and an and.
+	// If c >= 128, then 1<<c and 1<<(c-64) will both be zero,
+	// and this function will return false.
+	const mask = 0 |
+		(1<<(10)-1)<<'0' |
+		(1<<(26)-1)<<'a' |
+		(1<<(26)-1)<<'A' |
+		1<<'!' |
+		1<<'#' |
+		1<<'$' |
+		1<<'%' |
+		1<<'&' |
+		1<<'\'' |
+		1<<'*' |
+		1<<'+' |
+		1<<'-' |
+		1<<'.' |
+		1<<'^' |
+		1<<'_' |
+		1<<'`' |
+		1<<'|' |
+		1<<'~'
+	return ((uint64(1)<<c)&(mask&(1<<64-1)) |
+		(uint64(1)<<(c-64))&(mask>>64)) != 0
+}
+
+// canonicalMIMEHeaderKey is like CanonicalMIMEHeaderKey but is
+// allowed to mutate the provided byte slice before returning the
+// string.
+//
+// For invalid inputs (if a contains spaces or non-token bytes), a
+// is unchanged and a string copy is returned.
+//
+// ok is true if the header key contains only valid characters and spaces.
+// ReadMIMEHeader accepts header keys containing spaces, but does not
+// canonicalize them.
+func canonicalMIMEHeaderKey(a []byte) (_ string, ok bool) {
+	if len(a) == 0 {
+		return "", false
+	}
+
+	// See if a looks like a header key. If not, return it unchanged.
+	noCanon := false
+	for _, c := range a {
+		if validHeaderFieldByte(c) {
+			continue
+		}
+		// Don't canonicalize.
+		if c == ' ' {
+			// We accept invalid headers with a space before the
+			// colon, but must not canonicalize them.
+			// See https://go.dev/issue/34540.
+			noCanon = true
+			continue
+		}
+		return string(a), false
+	}
+	if noCanon {
+		return string(a), true
+	}
+
+	upper := true
+	for i, c := range a {
+		// Canonicalize: first letter upper case
+		// and upper case after each dash.
+		// (Host, User-Agent, If-Modified-Since).
+		// MIME headers are ASCII only, so no Unicode issues.
+		if upper && 'a' <= c && c <= 'z' {
+			c -= toLower
+		} else if !upper && 'A' <= c && c <= 'Z' {
+			c += toLower
+		}
+		a[i] = c
+		upper = c == '-' // for next time
+	}
+
+	// The compiler recognizes m[string(byteSlice)] as a special
+	// case, so a copy of a's bytes into a new string does not
+	// happen in this map lookup:
+	if v := commonHeader[string(a)]; v != "" {
+		return v, true
+	}
+	return string(a), true
+}
+
+func init() {
+	initCommonHeader()
+}
+
+// commonHeader interns common header strings.
+var commonHeader map[string]string
+
+func initCommonHeader() {
+	commonHeader = make(map[string]string)
+	for _, v := range []string{
+		"Accept",
+		"Accept-Charset",
+		"Accept-Encoding",
+		"Accept-Language",
+		"Accept-Ranges",
+		"Cache-Control",
+		"Cc",
+		"Connection",
+		"Content-Id",
+		"Content-Language",
+		"Content-Length",
+		"Content-Transfer-Encoding",
+		"Content-Type",
+		"Cookie",
+		"Date",
+		"Dkim-Signature",
+		"Etag",
+		"Expires",
+		"From",
+		"Host",
+		"If-Modified-Since",
+		"If-None-Match",
+		"In-Reply-To",
+		"Last-Modified",
+		"Location",
+		"Message-Id",
+		"Mime-Version",
+		"Pragma",
+		"Received",
+		"Return-Path",
+		"Server",
+		"Set-Cookie",
+		"Subject",
+		"To",
+		"User-Agent",
+		"Via",
+		"X-Forwarded-For",
+		"X-Imforwards",
+		"X-Powered-By",
+	} {
+		commonHeader[v] = v
+	}
+}
--- a/challenge/interface.go
+++ b/challenge/interface.go
@@ -1,14 +1,10 @@
 package challenge

 import (
-	"crypto/sha256"
 	"encoding/json"
-	"fmt"
-	"net/http"
+	"git.gammaspectra.live/git/go-away/challenge/inline"
 )

-const ChallengeKeySize = sha256.Size
-
 type MakeChallenge func(in Allocation) (out Allocation)

 type Allocation uint64
@@ -36,7 +32,7 @@ func MakeChallengeDecode(callback func(in MakeChallengeInput, out *MakeChallenge
 		outStruct.Error = err.Error()
 	} else {
 		outStruct.Code = 200
-		outStruct.Headers = make(http.Header)
+		outStruct.Headers = make(inline.MIMEHeader)

 		func() {
 			// encapsulate err
@@ -48,7 +44,7 @@ func MakeChallengeDecode(callback func(in MakeChallengeInput, out *MakeChallenge
 					if err, ok := recovered.(error); ok {
 						outStruct.Error = err.Error()
 					} else {
-						outStruct.Error = fmt.Sprintf("%v", recovered)
+						outStruct.Error = "error"
 					}
 				}
 			}()
@@ -92,26 +88,26 @@ func VerifyChallengeDecode(callback func(in VerifyChallengeInput) VerifyChalleng
 }

 type MakeChallengeInput struct {
-	Key []byte `json:"key"`
+	Key []byte

-	Parameters map[string]string `json:"parameters,omitempty"`
+	Parameters map[string]string

-	Headers http.Header `json:"headers,omitempty"`
-	Data    []byte      `json:"data,omitempty"`
+	Headers inline.MIMEHeader
+	Data    []byte
 }

 type MakeChallengeOutput struct {
-	Data    []byte      `json:"data"`
-	Code    int         `json:"code"`
-	Headers http.Header `json:"headers,omitempty"`
-	Error   string      `json:"error,omitempty"`
+	Data    []byte
+	Code    int
+	Headers inline.MIMEHeader
+	Error   string
 }

 type VerifyChallengeInput struct {
-	Key        []byte            `json:"key"`
-	Parameters map[string]string `json:"parameters,omitempty"`
+	Key        []byte
+	Parameters map[string]string

-	Result []byte `json:"result,omitempty"`
+	Result []byte
 }

 type VerifyChallengeOutput uint64
--- a/challenge/js-pow-sha256/runtime/runtime.go
+++ b/challenge/js-pow-sha256/runtime/runtime.go
@@ -4,20 +4,19 @@ import (
 	"crypto/sha256"
 	"crypto/subtle"
 	"encoding/binary"
-	"encoding/hex"
-	"encoding/json"
 	"git.gammaspectra.live/git/go-away/challenge"
+	"git.gammaspectra.live/git/go-away/challenge/inline"
+	"math/bits"
 	"strconv"
-	"strings"
 )

-//go:generate tinygo build -target wasip1 -buildmode=c-shared -scheduler=none -gc=leaking -o runtime.wasm runtime.go
+//go:generate tinygo build -target wasip1 -buildmode=c-shared -opt=2 -scheduler=none -gc=leaking -no-debug -o runtime.wasm runtime.go
 func main() {

 }

 func getChallenge(key []byte, params map[string]string) ([]byte, uint64) {
-	difficulty := uint64(5)
+	difficulty := uint64(20)
 	var err error
 	if diffStr, ok := params["difficulty"]; ok {
 		difficulty, err = strconv.ParseUint(diffStr, 10, 64)
@@ -34,22 +33,32 @@ func getChallenge(key []byte, params map[string]string) ([]byte, uint64) {
 //go:wasmexport MakeChallenge
 func MakeChallenge(in challenge.Allocation) (out challenge.Allocation) {
 	return challenge.MakeChallengeDecode(func(in challenge.MakeChallengeInput, out *challenge.MakeChallengeOutput) {
-		type Result struct {
-			Challenge  string `json:"challenge"`
-			Difficulty uint64 `json:"difficulty"`
+		c, difficulty := getChallenge(in.Key, in.Parameters)
+
+		// create target
+		target := make([]byte, len(c))
+		nBits := difficulty
+		for i := 0; i < len(target); i++ {
+			var v uint8
+			for j := 0; j < 8; j++ {
+				v <<= 1
+				if nBits == 0 {
+					v |= 1
+				} else {
+					nBits--
+				}
+			}
+			target[i] = v
 		}

-		challenge, difficulty := getChallenge(in.Key, in.Parameters)
+		dst := make([]byte, inline.EncodedLen(len(c)))
+		dst = dst[:inline.Encode(dst, c)]

-		data, err := json.Marshal(Result{
-			Challenge:  hex.EncodeToString(challenge),
-			Difficulty: difficulty,
-		})
-		if err != nil {
-			panic(err)
-		}
-		out.Data = data
-		out.Headers.Set("Content-Type", "text/javascript; charset=utf-8")
+		targetDst := make([]byte, inline.EncodedLen(len(target)))
+		targetDst = targetDst[:inline.Encode(targetDst, target)]
+
+		out.Data = []byte("{\"challenge\": \"" + string(dst) + "\", \"target\": \"" + string(targetDst) + "\", \"difficulty\": " + strconv.FormatUint(difficulty, 10) + "}")
+		out.Headers.Set("Content-Type", "application/json; charset=utf-8")
 	}, in)
 }

@@ -58,32 +67,30 @@ func VerifyChallenge(in challenge.Allocation) (out challenge.VerifyChallengeOutp
 	return challenge.VerifyChallengeDecode(func(in challenge.VerifyChallengeInput) challenge.VerifyChallengeOutput {
 		c, difficulty := getChallenge(in.Key, in.Parameters)

-		type Result struct {
-			Hash  string `json:"hash"`
-			Nonce uint64 `json:"nonce"`
-		}
-		var result Result
-		err := json.Unmarshal(in.Result, &result)
-
+		result := make([]byte, inline.DecodedLen(len(in.Result)))
+		n, err := inline.Decode(result, in.Result)
 		if err != nil {
 			panic(err)
 		}
+		result = result[:n]

-		if !strings.HasPrefix(result.Hash, strings.Repeat("0", int(difficulty))) {
+		// verify we used same challenge
+		if subtle.ConstantTimeCompare(result[:len(result)-8], c) != 1 {
 			return challenge.VerifyChallengeOutputFailed
 		}

-		resultBinary, err := hex.DecodeString(result.Hash)
-		if err != nil {
-			panic(err)
+		hash := sha256.Sum256(result)
+
+		var leadingZeroesCount int
+		for i := 0; i < len(hash); i++ {
+			leadingZeroes := bits.LeadingZeros8(hash[i])
+			leadingZeroesCount += leadingZeroes
+			if leadingZeroes < 8 {
+				break
+			}
 		}

-		buf := make([]byte, 0, len(c)+8)
-		buf = append(buf, c[:]...)
-		buf = binary.LittleEndian.AppendUint64(buf, result.Nonce)
-		calculated := sha256.Sum256(buf)
-
-		if subtle.ConstantTimeCompare(resultBinary, calculated[:]) != 1 {
+		if leadingZeroesCount < int(difficulty) {
 			return challenge.VerifyChallengeOutputFailed
 		}

--- a/challenge/js-pow-sha256/static/load.mjs
+++ b/challenge/js-pow-sha256/static/load.mjs
@@ -1,6 +1,7 @@
 let _worker;
 let _webWorkerURL;
 let _challenge;
+let _target;
 let _difficulty;

 async function setup(config) {
@@ -10,7 +11,7 @@ async function setup(config) {
    const title = document.getElementById('title');
    const spinner = document.getElementById('spinner');

-    const { challenge, difficulty } = await fetch(config.Path + "/make-challenge", { method: "POST" })
+    const { challenge, target, difficulty } = await fetch(config.Path + "/make-challenge", { method: "POST" })
        .then(r => {
            if (!r.ok) {
                throw new Error("Failed to fetch config");
@@ -22,6 +23,7 @@ async function setup(config) {
        });

    _challenge = challenge;
+    _target = target;
    _difficulty = difficulty;

    _webWorkerURL = URL.createObjectURL(new Blob([
@@ -46,6 +48,7 @@ function challenge() {

        _worker.postMessage({
            challenge: _challenge,
+            target: _target,
            difficulty: _difficulty,
        });

@@ -94,7 +97,7 @@ function processTask() {

        addEventListener('message', async (event) => {
            let data = decodeHex(event.data.challenge);
-            let target  = decodeHex("0".repeat(event.data.difficulty) + "f".repeat(64 - event.data.difficulty));
+            let target  = decodeHex(event.data.target);

            let nonce = new Uint8Array(8);
            let buf = new Uint8Array(data.length + nonce.length);
@@ -107,10 +110,7 @@ function processTask() {
                if (lessThan(result, target)){
                    const nonceNumber = Number(new BigUint64Array(nonce.buffer).at(0))
                    postMessage({
-                        result: {
-                            hash: encodeHex(result),
-                            nonce: nonceNumber,
-                        },
+                        result: encodeHex(buf),
                        info: `iterations ${nonceNumber}`,
                    });
                    return
--- a/http.go
+++ b/http.go
@@ -180,9 +180,9 @@ func (state *State) setupRoutes() error {
 				err := func() (err error) {
 					expiry := time.Now().UTC().Add(DefaultValidity).Round(DefaultValidity)
 					key := state.GetChallengeKeyForRequest(challengeName, expiry, r)
-					result := []byte(r.FormValue("result"))
+					result := r.FormValue("result")

-					if ok, err := c.Verify(key, string(result)); err != nil {
+					if ok, err := c.Verify(key, result); err != nil {
 						return err
 					} else if !ok {
 						ClearCookie(CookiePrefix+challengeName, w)
@@ -190,7 +190,7 @@ func (state *State) setupRoutes() error {
 						return nil
 					}

-					token, err := state.IssueChallengeToken(challengeName, key, result, expiry)
+					token, err := state.IssueChallengeToken(challengeName, key, []byte(result), expiry)
 					if err != nil {
 						ClearCookie(CookiePrefix+challengeName, w)
 					} else {
--- a/policy.go
+++ b/policy.go
@@ -93,7 +93,7 @@ type PolicyNetwork struct {
 	Prefixes []string `yaml:"prefixes,omitempty"`
 }

-func (n PolicyNetwork) FetchPrefixes() (output []net.IPNet, err error) {
+func (n PolicyNetwork) FetchPrefixes(c *http.Client) (output []net.IPNet, err error) {
 	if len(n.Prefixes) > 0 {
 		for _, prefix := range n.Prefixes {
 			ipNet, err := parseCIDROrIP(prefix)
@@ -106,7 +106,7 @@ func (n PolicyNetwork) FetchPrefixes() (output []net.IPNet, err error) {

 	var reader io.Reader
 	if n.Url != nil {
-		response, err := http.DefaultClient.Get(*n.Url)
+		response, err := c.Get(*n.Url)
 		if err != nil {
 			return nil, err
 		}
--- a/policy.yml
+++ b/policy.yml
@@ -1,7 +1,8 @@

 # Define networks to be used later below
 networks:
-  # todo: support ASN lookups
+  # todo: support direct ASN lookups
+  # todo: cache these values
  huawei-cloud:
    # AS136907
    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/136907/aggregated.json
@@ -10,6 +11,44 @@ networks:
    # AS45102
    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/45102/aggregated.json
      jq-path: '.subnets.ipv4[], .subnets.ipv6[]'
+  aws-cloud:
+    - url: https://ip-ranges.amazonaws.com/ip-ranges.json
+      jq-path: '(.prefixes[] | select(has("ip_prefix")) | .ip_prefix), (.prefixes[] | select(has("ipv6_prefix")) | .ipv6_prefix)'
+  google-cloud:
+    - url: https://www.gstatic.com/ipranges/cloud.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  oracle-cloud:
+    - url: https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json
+      jq-path: '.regions[] | .cidrs[] | .cidr'
+  azure-cloud:
+    # todo: https://www.microsoft.com/en-us/download/details.aspx?id=56519 does not provide direct JSON
+    - url: https://raw.githubusercontent.com/femueller/cloud-ip-ranges/refs/heads/master/microsoft-azure-ip-ranges.json
+      jq-path: '.values[] | .properties.addressPrefixes[]'
+
+  digitalocean:
+    - url: https://www.digitalocean.com/geo/google.csv
+      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
+  linode:
+    - url: https://geoip.linode.com/
+      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
+  vultr:
+    - url: "https://geofeed.constant.com/?json"
+      jq-path: '.subnets[] | .ip_prefix'
+  cloudflare:
+    - url: https://www.cloudflare.com/ips-v4
+      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
+    - url: https://www.cloudflare.com/ips-v6
+      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"
+
+  icloud-private-relay:
+    - url: https://mask-api.icloud.com/egress-ip-ranges.csv
+      regex: "(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+),"
+  tunnelbroker-relay:
+    # HE Tunnelbroker
+    - url: https://tunnelbroker.net/export/google
+      regex: "(?P<prefix>([0-9a-f:]+::)/[0-9]+),"
+
+
  googlebot:
    - url: https://developers.google.com/static/search/apis/ipranges/googlebot.json
      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
@@ -44,11 +83,6 @@ networks:
  kagibot:
    - url: https://kagi.com/bot
      regex: "\\n(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+) "
-  cloudflare:
-    - url: https://www.cloudflare.com/ips-v4
-      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
-    - url: https://www.cloudflare.com/ips-v6
-      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"


 # todo: define interface
@@ -59,7 +93,7 @@ challenges:
    mode: js
    asset: load.mjs
    parameters:
-      difficulty: 5
+      difficulty: 20
    runtime:
      mode: wasm
      # Verify must be under challenges/{name}/runtime/{asset}
@@ -88,7 +122,7 @@ challenges:

  http-cookie-check:
    mode: http
-    url: http://172.20.5.5:3002/user/stopwatches
+    url: http://gitea:3000/user/stopwatches
    # url: http://gitea:3000/repo/search
    # url: http://gitea:3000/notifications/new
    parameters:
@@ -135,7 +169,7 @@ rules:
      # Typo'd opera botnet
      - 'userAgent.matches("^Opera/[0-9.]+\\.\\(")'
      # AI bullshit stuff, they do not respect robots.txt even while they read it
-      - 'userAgent.contains("Amazonbot") || userAgent.contains("Bytespider") || userAgent.contains("ClaudeBot") || userAgent.contains("meta-externalagent/")'
+      - 'userAgent.contains("Amazonbot") || userAgent.contains("Bytespider")|| userAgent.contains("CCBot") || userAgent.contains("ClaudeBot") || userAgent.contains("meta-externalagent/")'
    action: deny

  - name: suspicious-crawlers
@@ -210,7 +244,8 @@ rules:
  - name: preview-fetchers
    conditions:
      - 'path.endsWith("/-/summary-card")'
-      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("Twitterbot/")'
+      #- 'userAgent.contains("facebookexternalhit/")'
+      - 'userAgent.contains("Twitterbot/")'
      - '"X-Purpose" in headers && headers["X-Purpose"] == "preview"'
    action: pass

@@ -230,6 +265,13 @@ rules:
      - 'path.matches("(?i)^/(WeebDataHoarder|P2Pool|mirror|git|S\\.O\\.N\\.G|FM10K|Sillycom|pwgen2155|kaitou|metonym)/[^/]+$")'
    action: pass

+  - name: suspicious-fetchers
+    action: challenge
+    challenges: [js-pow-sha256, http-cookie-check]
+    conditions:
+      - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")'
+
+
  - name: standard-browser
    action: challenge
    challenges: [http-cookie-check, self-meta-refresh, js-pow-sha256]
--- a/state.go
+++ b/state.go
@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"git.gammaspectra.live/git/go-away/challenge"
+	"git.gammaspectra.live/git/go-away/challenge/inline"
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
@@ -20,6 +21,7 @@ import (
 	"github.com/yl2chen/cidranger"
 	"io"
 	"io/fs"
+	"log/slog"
 	"net"
 	"net/http"
 	"net/url"
@@ -101,7 +103,10 @@ func NewState(policy Policy, packagePath string, backend http.Handler) (state *S
 	for k, network := range policy.Networks {
 		ranger := cidranger.NewPCTrieRanger()
 		for _, e := range network {
-			prefixes, err := e.FetchPrefixes()
+			if e.Url != nil {
+				slog.Debug("loading network url list", "network", k, "url", *e.Url)
+			}
+			prefixes, err := e.FetchPrefixes(state.Client)
 			if err != nil {
 				return nil, fmt.Errorf("networks %s: error fetching prefixes: %v", k, err)
 			}
@@ -335,7 +340,7 @@ func NewState(policy Policy, packagePath string, backend http.Handler) (state *S
 					in := challenge.MakeChallengeInput{
 						Key:        state.GetChallengeKeyForRequest(challengeName, time.Now().UTC().Add(DefaultValidity).Round(DefaultValidity), r),
 						Parameters: p.Parameters,
-						Headers:    r.Header,
+						Headers:    inline.MIMEHeader(r.Header),
 					}
 					in.Data, err = io.ReadAll(r.Body)
 					if err != nil {
--- a/templates/challenge.mjs
+++ b/templates/challenge.mjs
@@ -54,7 +54,7 @@ const u = (url = "", params = {}) => {
        setTimeout(() => {
            const redir = window.location.href;
            window.location.href = u("{{ .Path }}/verify-challenge", {
-                result: JSON.stringify(result),
+                result: result,
                redirect: redir,
                elapsedTime: t1 - t0,
            });