Initial commit

2025-03-31 16:24:08 +02:00
commit 06bc5107d6
27 changed files with 2500 additions and 0 deletions
--- a/policy.yml
+++ b/policy.yml
@@ -0,0 +1,150 @@
+# Define groups of useragents to use later below for matching
+user-agents:
+  default-browser:
+    - "^Mozilla/"
+    - "^Opera/"
+  bad-crawlers:
+    - "Amazonbot"
+  headless-browser:
+    - "HeadlessChrome"
+    - "HeadlessChromium"
+    - "^Lightpanda/"
+    - "^$"
+  rss:
+    - "FeedFetcher-Google"
+  git:
+    - "^git/"
+    - "^go-git/"
+    - "^JGit[/-]"
+    - "^GoModuleMirror/"
+
+# Define networks to be used later below
+networks:
+  # todo: support ASN lookups
+  huawei-cloud:
+    # AS136907
+    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/136907/aggregated.json
+      jq-path: '.subnets.ipv4[], .subnets.ipv6[]'
+  alibaba-cloud:
+    # AS45102
+    - url: https://raw.githubusercontent.com/ipverse/asn-ip/refs/heads/master/as/45102/aggregated.json
+      jq-path: '.subnets.ipv4[], .subnets.ipv6[]'
+  googlebot:
+    - url: https://developers.google.com/static/search/apis/ipranges/googlebot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  bingbot:
+    - url: https://www.bing.com/toolbox/bingbot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  qwantbot:
+    - url: https://help.qwant.com/wp-content/uploads/sites/2/2025/01/qwantbot.json
+      jq-path: '(.prefixes[] | select(has("ipv4Prefix")) | .ipv4Prefix), (.prefixes[] | select(has("ipv6Prefix")) | .ipv6Prefix)'
+  duckduckbot:
+    - url: https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
+      regex: "<li>(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)</li>"
+  yandexbot:
+    # todo: detected as bot
+    # - url: https://yandex.com/ips
+    #  regex: "<span>(?P<prefix>(([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)|([0-9a-f:]+::))/[0-9]+)[ \\\\t]*</span><br/>"
+    - prefixes:
+      - "5.45.192.0/18"
+      - "5.255.192.0/18"
+      - "37.9.64.0/18"
+      - "37.140.128.0/18"
+      - "77.88.0.0/18"
+      - "84.252.160.0/19"
+      - "87.250.224.0/19"
+      - "90.156.176.0/22"
+      - "93.158.128.0/18"
+      - "95.108.128.0/17"
+      - "141.8.128.0/18"
+      - "178.154.128.0/18"
+      - "185.32.187.0/24"
+      - "2a02:6b8::/29"
+  kagibot:
+    - url: https://kagi.com/bot
+      regex: "\\n(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+) "
+  cloudflare:
+    - url: https://www.cloudflare.com/ips-v4
+      regex: "(?P<prefix>[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/[0-9]+)"
+    - url: https://www.cloudflare.com/ips-v6
+      regex: "(?P<prefix>[0-9a-f:]+::/[0-9]+)"
+
+
+conditions:
+  # Checks to detect a headless chromium via headers only
+  is-headless-chromium:
+    - 'userAgent.contains("HeadlessChrome") || userAgent.contains("HeadlessChromium")'
+    - 'headers["Sec-Ch-Ua"].contains("HeadlessChrome") || headers["Sec-Ch-Ua"].contains("HeadlessChromium")'
+    - '(userAgent.contains("Chrome/") || userAgent.contains("Chromium/")) && (headers["Accept-Language"] == "" || headers["Accept-Encoding"] == "")'
+  is-static-asset:
+    - 'path == "/robots.txt"'
+    - 'path == "/favicon.ico"'
+    - 'path == "/apple-touch-icon.png"'
+    - 'path == "/apple-touch-icon-precomposed.png"'
+    - 'path.startsWith("/assets/")'
+    - 'path.startsWith("/repo-avatars/")'
+    - 'path.startsWith("/avatars/")'
+    - 'path.startsWith("/avatar/")'
+
+
+# todo: define interface
+challenges:
+  js-pow-sha256:
+    # Asset must be under challenges/{name}/static/{asset}
+    # Other files here will be available under that path
+    mode: js
+    asset: load.mjs
+    parameters:
+      difficulty: 4
+    runtime:
+      mode: wasm
+      # Verify must be under challenges/{name}/runtime/{asset}
+      asset: runtime.wasm
+      probability: 0.02
+
+  # Challenges with a cookie, self redirect
+  self-cookie:
+    mode: "cookie"
+
+  # Challenges with a redirect via header
+  self-header-refresh:
+    mode: "header-refresh"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+
+  # Challenges with a redirect via meta
+  self-meta-refresh:
+    mode: "meta-refresh"
+    runtime:
+      # verifies that result = key
+      mode: "key"
+      probability: 0.1
+
+  http-cookie-check:
+    mode: http
+    url: http://172.20.5.5:3002/user/stopwatches
+    # url: http://gitea:3000/repo/search
+    # url: http://gitea:3000/notifications/new
+    parameters:
+      http-method: GET
+      http-code: 200
+
+rules:
+  - name: blocked-networks
+    conditions:
+      - 'inNetwork("huawei-cloud", remoteAddress) || inNetwork("alibaba-cloud", remoteAddress)'
+    action: deny
+
+  - name: golang-proxy
+    conditions:
+      - 'userAgent.startsWith("GoModuleMirror/") || (userAgent.startsWith("Go-http-client/") && query["go-get"] == "1")'
+    action: pass
+
+  - name: standard-browser
+    action: challenge
+    challenges: [http-cookie-check, self-meta-refresh, js-pow-sha256]
+    conditions:
+      - 'userAgent.startsWith("Mozilla/") || userAgent.startsWith("Opera/")'
+