Condition, rules, state and action refactor / rewrite

Add nested rules
Add backend action, allow wildcard in backends
Remove poison from tree, update README with action table

Allow defining pass/fail actions on challenge,

Remove redirect/referer parameters on backend pass

Set challenge cookie tied to host

Rewrite DNSBL condition into a challenge

Allow passing an arbitrary path for assets to js challenges

Optimize programs exhaustively on compilation

Activation instead of map for CEL context, faster map access, new network override

Return valid host on cookie setting in case Host is an IP address.
bug: does not work with IPv6, see https://github.com/golang/go/issues/65521

Apply TLS fingerprinter on GetConfigForClient instead of GetCertificate

Cleanup go-away cookies before passing to backend

Code action for specifically replying with an HTTP code

lib/http.go

@@ -1,28 +1,16 @@
 package lib
 
 import (
-	"bytes"
 	"codeberg.org/meta/gzipped/v2"
-	"context"
-	"crypto/rand"
-	"encoding/base64"
-	"encoding/hex"
-	"errors"
 	"fmt"
 	"git.gammaspectra.live/git/go-away/embed"
+	"git.gammaspectra.live/git/go-away/lib/action"
 	"git.gammaspectra.live/git/go-away/lib/challenge"
-	"git.gammaspectra.live/git/go-away/lib/policy"
 	"git.gammaspectra.live/git/go-away/utils"
-	"github.com/google/cel-go/common/types"
 	"html/template"
-	"io"
 	"log/slog"
-	"maps"
-	"net"
 	"net/http"
 	"net/http/pprof"
-	"path"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -30,20 +18,11 @@ import (
 
 var templates map[string]*template.Template
 
-var cacheBust string
-
-// DefaultValidity TODO: adjust
-const DefaultValidity = time.Hour * 24 * 7
-
 func init() {
 
-	buf := make([]byte, 16)
-	_, _ = rand.Read(buf)
-	cacheBust = base64.RawURLEncoding.EncodeToString(buf)
-
 	templates = make(map[string]*template.Template)
 
-	dir, err := embed.TemplatesFs.ReadDir("templates")
+	dir, err := embed.TemplatesFs.ReadDir(".")
 	if err != nil {
 		panic(err)
 	}
@@ -51,7 +30,7 @@ func init() {
 		if e.IsDir() {
 			continue
 		}
-		data, err := embed.TemplatesFs.ReadFile(filepath.Join("templates", e.Name()))
+		data, err := embed.TemplatesFs.ReadFile(e.Name())
 		if err != nil {
 			panic(err)
 		}
@@ -72,69 +51,16 @@ func initTemplate(name, data string) error {
 	return nil
 }
 
-func (state *State) challengePage(w http.ResponseWriter, id string, status int, challenge string, params map[string]any) error {
-	input := make(map[string]any)
-	input["Id"] = id
-	input["Random"] = cacheBust
-	input["Challenge"] = challenge
-	input["Path"] = state.UrlPath
-	input["Theme"] = state.Settings.ChallengeTemplateTheme
-
-	maps.Copy(input, params)
-
-	if _, ok := input["Title"]; !ok {
-		input["Title"] = "Checking you are not a bot"
-	}
-
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-
-	buf := bytes.NewBuffer(make([]byte, 0, 8192))
-
-	err := templates["challenge-"+state.Settings.ChallengeTemplate+".gohtml"].Execute(buf, input)
-	if err != nil {
-		_ = state.errorPage(w, id, http.StatusInternalServerError, err, "")
-	} else {
-		w.WriteHeader(status)
-		_, _ = w.Write(buf.Bytes())
-	}
-	return nil
-}
-
-func (state *State) errorPage(w http.ResponseWriter, id string, status int, err error, redirect string) error {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-
-	buf := bytes.NewBuffer(make([]byte, 0, 8192))
-
-	err2 := templates["challenge-"+state.Settings.ChallengeTemplate+".gohtml"].Execute(buf, map[string]any{
-		"Id":          id,
-		"Random":      cacheBust,
-		"Error":       err.Error(),
-		"Path":        state.UrlPath,
-		"Theme":       state.Settings.ChallengeTemplateTheme,
-		"Title":       "Oh no! " + http.StatusText(status),
-		"HideSpinner": true,
-		"Challenge":   "",
-		"Redirect":    redirect,
-	})
-	if err2 != nil {
-		panic(err2)
-	} else {
-		w.WriteHeader(status)
-		_, _ = w.Write(buf.Bytes())
-	}
-	return nil
-}
-
 func (state *State) addTiming(w http.ResponseWriter, name, desc string, duration time.Duration) {
-	if state.Settings.Debug {
+	if state.Settings().Debug {
 		w.Header().Add("Server-Timing", fmt.Sprintf("%s;desc=%s;dur=%d", name, strconv.Quote(desc), duration.Milliseconds()))
 	}
 }
 
 func GetLoggerForRequest(r *http.Request) *slog.Logger {
-	data := RequestDataFromContext(r.Context())
+	data := challenge.RequestDataFromContext(r.Context())
 	args := []any{
-		"request_id", hex.EncodeToString(data.Id[:]),
+		"request_id", data.Id.String(),
 		"remote_address", data.RemoteAddress.String(),
 		"user_agent", r.UserAgent(),
 		"host", r.Host,
@@ -153,272 +79,95 @@ func GetLoggerForRequest(r *http.Request) *slog.Logger {
 	return slog.With(args...)
 }
 
-func (state *State) logger(r *http.Request) *slog.Logger {
-	return GetLoggerForRequest(r)
-}
-
 func (state *State) handleRequest(w http.ResponseWriter, r *http.Request) {
 	host := r.Host
 
-	data := RequestDataFromContext(r.Context())
+	data := challenge.RequestDataFromContext(r.Context())
 
-	backend, ok := state.Settings.Backends[host]
-	if !ok {
+	backend := state.GetBackend(host)
+	if backend == nil {
 		http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
 		return
 	}
 
-	lg := state.logger(r)
+	lg := state.Logger(r)
 
-	start := time.Now()
-
-	state.addTiming(w, "rule-env", "Setup the rule environment", time.Since(start))
-
-	var (
-		ruleEvalDuration time.Duration
-	)
-
-	serve := func() {
-		state.addTiming(w, "rule-eval", "Evaluate access rules", ruleEvalDuration)
-		backend.ServeHTTP(w, r)
-	}
-
-	fail := func(code int, err error) {
-		state.addTiming(w, "rule-eval", "Evaluate access rules", ruleEvalDuration)
-		_ = state.errorPage(w, r.Header.Get("X-Away-Id"), code, err, "")
-	}
-
-	setAwayState := func(rule RuleState) {
-		r.Header.Set("X-Away-Rule", rule.Name)
-		r.Header.Set("X-Away-Hash", rule.Hash)
-		r.Header.Set("X-Away-Action", string(rule.Action))
-		data.Headers(state, r.Header)
-	}
-
-	for _, rule := range state.Rules {
-		// skip rules that have host match
-		if rule.Host != nil && *rule.Host != host {
-			continue
+	cleanupRequest := func(r *http.Request, fromChallenge bool) {
+		if fromChallenge {
+			r.Header.Del("Referer")
+		}
+		if ref := r.FormValue(challenge.QueryArgReferer); ref != "" {
+			r.Header.Set("Referer", ref)
 		}
-		start = time.Now()
-		out, _, err := rule.Program.Eval(data.ProgramEnv)
-		ruleEvalDuration += time.Since(start)
 
-		if err != nil {
-			fail(http.StatusInternalServerError, err)
-			lg.Error(err.Error(), "rule", rule.Name, "rule_hash", rule.Hash)
-			panic(err)
-			return
-		} else if out != nil && out.Type() == types.BoolType {
-			if out.Equal(types.True) == types.True {
-				switch rule.Action {
-				default:
-					panic(fmt.Errorf("unknown action %s", rule.Action))
-				case policy.RuleActionPASS:
-					lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash)
-					setAwayState(rule)
-					serve()
-					return
-				case policy.RuleActionCHALLENGE, policy.RuleActionCHECK:
-					for _, challengeId := range rule.Challenges {
-						if result := data.Challenges[challengeId]; !result.Ok() {
-							continue
-						} else {
-							if rule.Action == policy.RuleActionCHECK {
-								goto nextRule
-							}
-
-							// we passed the challenge!
-							lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", state.Challenges[challengeId].Name)
-							setAwayState(rule)
-							serve()
-							return
-						}
-					}
-
-					// none matched, issue first challenge in priority
-					for _, challengeId := range rule.Challenges {
-						result := data.Challenges[challengeId]
-						if result.Ok() || result == challenge.VerifyResultSKIP {
-							// skip already ok'd challenges for some reason, and also skip skipped challenges
-							continue
-						}
-						c := state.Challenges[challengeId]
-						if c.ServeChallenge != nil {
-							result := c.ServeChallenge(w, r, state.GetChallengeKeyForRequest(c.Name, data.Expires, r), data.Expires)
-							switch result {
-							case challenge.ResultStop:
-								lg.Info("request challenged", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
-								return
-							case challenge.ResultContinue:
-								continue
-							case challenge.ResultPass:
-								if rule.Action == policy.RuleActionCHECK {
-									goto nextRule
-								}
-								state.logger(r).Warn("challenge passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
-
-								// set pass if caller didn't set one
-								if !data.Challenges[c.Id].Ok() {
-									data.Challenges[c.Id] = challenge.VerifyResultPASS
-								}
-
-								// we pass the challenge early!
-								lg.Debug("request passed", "rule", rule.Name, "rule_hash", rule.Hash, "challenge", c.Name)
-								setAwayState(rule)
-								serve()
-								return
-							}
-						} else {
-							panic("challenge not found")
-						}
-					}
-				case policy.RuleActionDENY:
-					lg.Info("request denied", "rule", rule.Name, "rule_hash", rule.Hash)
-					//TODO: config error code
-					fail(http.StatusForbidden, fmt.Errorf("access denied: denied by administrative rule %s/%s", r.Header.Get("X-Away-Id"), rule.Hash))
-					return
-				case policy.RuleActionBLOCK:
-					lg.Info("request blocked", "rule", rule.Name, "rule_hash", rule.Hash)
-					//TODO: config error code
-					//TODO: configure block
-					fail(http.StatusForbidden, fmt.Errorf("access denied: blocked by administrative rule %s/%s", r.Header.Get("X-Away-Id"), rule.Hash))
-					return
-				case policy.RuleActionPOISON:
-					lg.Info("request poisoned", "rule", rule.Name, "rule_hash", rule.Hash)
-
-					mime := "text/html"
-					switch path.Ext(r.URL.Path) {
-					case ".css":
-					case ".json", ".js", ".mjs":
-
-					}
-
-					encodings := strings.Split(r.Header.Get("Accept-Encoding"), ",")
-					for i, encoding := range encodings {
-						encodings[i] = strings.TrimSpace(strings.ToLower(encoding))
-					}
-
-					reader, encoding := state.getPoison(mime, encodings)
-					if reader == nil {
-						mime = "application/octet-stream"
-						reader, encoding = state.getPoison(mime, encodings)
-					}
-
-					if reader != nil {
-						defer reader.Close()
-
-						w.Header().Set("Cache-Control", "max-age=0, private, must-revalidate, no-transform")
-						w.Header().Set("Vary", "Accept-Encoding")
-						w.Header().Set("Content-Type", mime)
-						w.Header().Set("X-Content-Type-Options", "nosniff")
-						if encoding != "" {
-							w.Header().Set("Content-Encoding", encoding)
-						}
-						w.WriteHeader(http.StatusOK)
-						if flusher, ok := w.(http.Flusher); ok {
-							// trigger chunked encoding
-							flusher.Flush()
-						}
-						if r != nil {
-							_, _ = io.Copy(w, reader)
-						}
-					} else {
-						http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
-					}
-					return
-				}
+		q := r.URL.Query()
+		// delete query parameters that were set by go-away
+		for k := range q {
+			if strings.HasPrefix(k, challenge.QueryArgPrefix) {
+				q.Del(k)
 			}
 		}
+		r.URL.RawQuery = q.Encode()
 
-	nextRule:
+		data.Headers(r.Header)
+
+		// delete cookies set by go-away to prevent user tracking that way
+		cookies := r.Cookies()
+		r.Header.Del("Cookie")
+		for _, c := range cookies {
+			if !strings.HasPrefix(c.Name, utils.CookiePrefix) {
+				r.AddCookie(c)
+			}
+		}
 	}
 
-	serve()
-	return
+	for _, rule := range state.rules {
+		next, err := rule.Evaluate(lg, w, r, func() http.Handler {
+			cleanupRequest(r, true)
+			return backend
+		})
+		if err != nil {
+			state.ErrorPage(w, r, http.StatusInternalServerError, err, "")
+			panic(err)
+			return
+		}
+
+		if !next {
+			return
+		}
+	}
+
+	// default pass
+	_, _ = action.Pass{}.Handle(lg, w, r, func() http.Handler {
+		r.Header.Set("X-Away-Rule", "DEFAULT")
+		r.Header.Set("X-Away-Action", "PASS")
+
+		cleanupRequest(r, false)
+		return backend
+	})
 }
 
 func (state *State) setupRoutes() error {
 
 	state.Mux.HandleFunc("/", state.handleRequest)
 
-	if state.Settings.Debug {
-		http.HandleFunc(state.UrlPath+"/debug/pprof/", pprof.Index)
-		http.HandleFunc(state.UrlPath+"/debug/pprof/profile", pprof.Profile)
-		http.HandleFunc(state.UrlPath+"/debug/pprof/symbol", pprof.Symbol)
-		http.HandleFunc(state.UrlPath+"/debug/pprof/trace", pprof.Trace)
+	if state.Settings().Debug {
+		//TODO: split this to a different listener, metrics listener
+		http.HandleFunc(state.urlPath+"/debug/pprof/", pprof.Index)
+		http.HandleFunc(state.urlPath+"/debug/pprof/profile", pprof.Profile)
+		http.HandleFunc(state.urlPath+"/debug/pprof/symbol", pprof.Symbol)
+		http.HandleFunc(state.urlPath+"/debug/pprof/trace", pprof.Trace)
 	}
 
-	state.Mux.Handle("GET "+state.UrlPath+"/assets/", http.StripPrefix(state.UrlPath, gzipped.FileServer(gzipped.FS(embed.AssetsFs))))
+	state.Mux.Handle("GET "+state.urlPath+"/assets/", http.StripPrefix(state.UrlPath()+"/assets/", gzipped.FileServer(gzipped.FS(embed.AssetsFs))))
 
-	for _, c := range state.Challenges {
-		if c.ServeStatic != nil {
-			state.Mux.Handle("GET "+c.Path+"/static/", c.ServeStatic)
-		}
+	for _, reg := range state.challenges {
 
-		if c.ServeScript != nil {
-			state.Mux.Handle("GET "+c.ServeScriptPath, c.ServeScript)
-		}
-
-		if c.ServeMakeChallenge != nil {
-			state.Mux.Handle(fmt.Sprintf("POST %s/make-challenge", c.Path), c.ServeMakeChallenge)
-		}
-
-		if c.ServeVerifyChallenge != nil {
-			state.Mux.Handle(fmt.Sprintf("GET %s/verify-challenge", c.Path), c.ServeVerifyChallenge)
-		} else if c.Verify != nil {
-			state.Mux.HandleFunc(fmt.Sprintf("GET %s/verify-challenge", c.Path), func(w http.ResponseWriter, r *http.Request) {
-				redirect, err := utils.EnsureNoOpenRedirect(r.FormValue("redirect"))
-				if redirect == "" {
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, "")
-					return
-				}
-
-				err = func() (err error) {
-
-					data := RequestDataFromContext(r.Context())
-
-					key := state.GetChallengeKeyForRequest(c.Name, data.Expires, r)
-					result := r.FormValue("result")
-
-					requestId, err := hex.DecodeString(r.FormValue("requestId"))
-					if err == nil {
-						// override
-						r.Header.Set("X-Away-Id", hex.EncodeToString(requestId))
-					}
-
-					start := time.Now()
-					ok, err := c.Verify(key, result, r)
-					state.addTiming(w, "challenge-verify", "Verify client challenge", time.Since(start))
-
-					if err != nil {
-						state.logger(r).Error(fmt.Errorf("challenge error: %w", err).Error(), "challenge", c.Name, "redirect", redirect)
-						return err
-					} else if !ok {
-						state.logger(r).Warn("challenge failed", "challenge", c.Name, "redirect", redirect)
-						utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-						_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusForbidden, fmt.Errorf("access denied: failed challenge %s", c.Name), redirect)
-						return nil
-					}
-					state.logger(r).Info("challenge passed", "challenge", c.Name, "redirect", redirect)
-
-					token, err := c.IssueChallengeToken(state.privateKey, key, []byte(result), data.Expires)
-					if err != nil {
-						utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-					} else {
-						utils.SetCookie(utils.CookiePrefix+c.Name, token, data.Expires, w)
-					}
-					data.Challenges[c.Id] = challenge.VerifyResultPASS
-
-					http.Redirect(w, r, redirect, http.StatusTemporaryRedirect)
-					return nil
-				}()
-				if err != nil {
-					utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-					_ = state.errorPage(w, r.Header.Get("X-Away-Id"), http.StatusInternalServerError, err, redirect)
-					return
-				}
-			})
+		if reg.Handler != nil {
+			state.Mux.Handle(reg.Path+"/", reg.Handler)
+		} else if reg.Verify != nil {
+			// default verify
+			state.Mux.HandleFunc(reg.Path+challenge.VerifyChallengeUrlSuffix, challenge.VerifyHandlerFunc(state, reg, nil, nil))
 		}
 	}
 
@@ -426,116 +175,12 @@ func (state *State) setupRoutes() error {
 }
 
 func (state *State) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	r, data := challenge.CreateRequestData(r, state)
 
-	var data RequestData
-	// generate random id, todo: is this fast?
-	_, _ = rand.Read(data.Id[:])
-	data.RemoteAddress = getRequestAddress(r, state.Settings.ClientIpHeader)
-	data.Challenges = make(map[challenge.Id]challenge.VerifyResult, len(state.Challenges))
-	data.Expires = time.Now().UTC().Add(DefaultValidity).Round(DefaultValidity)
+	data.EvaluateChallenges(w, r)
 
-	var ja3n, ja4 string
-	if fp := utils.GetTLSFingerprint(r); fp != nil {
-		if ja3nPtr := fp.JA3N(); ja3nPtr != nil {
-			ja3n = ja3nPtr.String()
-			r.Header.Set("X-TLS-Fingerprint-JA3N", ja3n)
-		}
-		if ja4Ptr := fp.JA4(); ja4Ptr != nil {
-			ja4 = ja4Ptr.String()
-			r.Header.Set("X-TLS-Fingerprint-JA4", ja4)
-		}
-	}
-
-	data.ProgramEnv = map[string]any{
-		"host":          r.Host,
-		"method":        r.Method,
-		"remoteAddress": data.RemoteAddress,
-		"userAgent":     r.UserAgent(),
-		"path":          r.URL.Path,
-		"fpJA3N":        ja3n,
-		"fpJA4":         ja4,
-		"query": func() map[string]string {
-			result := make(map[string]string)
-			for k, v := range r.URL.Query() {
-				result[k] = strings.Join(v, ",")
-			}
-			return result
-		}(),
-		"headers": func() map[string]string {
-			result := make(map[string]string)
-			for k, v := range r.Header {
-				result[k] = strings.Join(v, ",")
-			}
-			return result
-		}(),
-	}
-
-	r = r.WithContext(context.WithValue(r.Context(), "_goaway_data", &data))
-
-	for _, c := range state.Challenges {
-		key := state.GetChallengeKeyForRequest(c.Name, data.Expires, r)
-		result, err := c.VerifyChallengeToken(state.publicKey, key, r)
-		if err != nil && !errors.Is(err, http.ErrNoCookie) {
-			// clear invalid cookie
-			utils.ClearCookie(utils.CookiePrefix+c.Name, w)
-		}
-
-		// prevent the challenge if not solved
-		if !result.Ok() && c.Program != nil {
-			out, _, err := c.Program.Eval(data.ProgramEnv)
-			// verify eligibility
-			if err != nil {
-				state.logger(r).Error(err.Error(), "challenge", c.Name)
-			} else if out != nil && out.Type() == types.BoolType {
-				if out.Equal(types.True) != types.True {
-					// skip challenge match!
-					result = challenge.VerifyResultSKIP
-					continue
-				}
-			}
-		}
-		data.Challenges[c.Id] = result
-	}
-
-	r.Header.Set("X-Away-Id", hex.EncodeToString(data.Id[:]))
-	if state.Settings.BackendIpHeader != "" {
-		r.Header.Del(state.Settings.ClientIpHeader)
-		r.Header.Set(state.Settings.BackendIpHeader, data.RemoteAddress.String())
-	}
+	// TODO: make this configurable!
 	w.Header().Add("Via", fmt.Sprintf("%s %s", r.Proto, "go-away"))
 
-	// send these to client so we consistently get the headers
-	//w.Header().Set("Accept-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
-	//w.Header().Set("Critical-CH", "Sec-CH-UA, Sec-CH-UA-Platform")
-
 	state.Mux.ServeHTTP(w, r)
 }
-
-func RequestDataFromContext(ctx context.Context) *RequestData {
-	return ctx.Value("_goaway_data").(*RequestData)
-}
-
-type RequestData struct {
-	Id            [16]byte
-	ProgramEnv    map[string]any
-	Expires       time.Time
-	Challenges    map[challenge.Id]challenge.VerifyResult
-	RemoteAddress net.IP
-}
-
-func (d *RequestData) HasValidChallenge(id challenge.Id) bool {
-	return d.Challenges[id].Ok()
-}
-
-func (d *RequestData) Headers(state *State, headers http.Header) {
-	for id, result := range d.Challenges {
-		if result.Ok() {
-			c, ok := state.Challenges[id]
-			if !ok {
-				panic("challenge not found")
-			}
-
-			headers.Set(fmt.Sprintf("X-Away-Challenge-%s-Result", c.Name), result.String())
-		}
-	}
-}