From bdaf15d8358dd900121d7a673ed0a5fdaea6bebe Mon Sep 17 00:00:00 2001 From: poli0981 <127664709+poli0981@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:28:19 +0700 Subject: [PATCH] feat(worker): rate-limit every client before the gate Two Workers Rate Limiting bindings in wrangler.jsonc, checked first in the router so a flood costs no siteverify call, HMAC check or asset fetch: - RATE_LIMIT: 120 requests a minute for everything the Worker handles (gate page, app shell, legal pages, /__gate/*, any method). - VERIFY_RATE_LIMIT: 10 a minute for POST /__gate/verify on top. Clients are keyed by IPv4 address or IPv6 /64. Over the limit: 429 with Retry-After: 60 (plain text, or JSON for the verify call). Like the gate, it fails open when a binding is missing or limit() throws, and logs that once per isolate. A test keeps wrangler.jsonc's bindings and period in step with the code. HOSTING documents both layers and the zone's WAF rate limiting rule (Free plan: one rule, per IP, 10 s) as the first one. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- docs/ARCHITECTURE.md | 2 +- docs/HOSTING.md | 61 +++++++++++-- docs/TECH-SPEC.md | 10 ++- tests/worker/rate-limit.test.ts | 154 ++++++++++++++++++++++++++++++++ tests/worker/router.test.ts | 110 +++++++++++++++++++++++ worker/env.ts | 14 +++ worker/rate-limit.ts | 97 ++++++++++++++++++++ worker/router.ts | 10 ++- wrangler.jsonc | 21 +++++ 10 files changed, 469 insertions(+), 12 deletions(-) create mode 100644 tests/worker/rate-limit.test.ts create mode 100644 worker/rate-limit.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1593029..89e7293 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,7 +61,7 @@ src/ file-ops.ts # save/open text files on every platform validation.ts, url-extractors.ts, log-storage.ts, … hooks/, components/, pages/, i18n/, styles/ -worker/ # Cloudflare Worker: gate, cookies, headers, routing +worker/ # Cloudflare Worker: gate, cookies, headers, rate limits, routing build-plugins/ # legal pages at build time; web-only extras src-tauri/src/ # lib.rs (plugins, tray), storage.rs (name-only file # commands), file_dialog.rs (native Save/Open) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 246a7ae..a0b1a21 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -388,7 +388,7 @@ Export: `ExportBackupDialog` → `collectBackup(chosen sections)` → `saveTextF ## 6. Web Delivery -The web build is served at by a Cloudflare Worker (`worker/`, configured by `wrangler.jsonc`) that Cloudflare Workers Builds deploys from `main`. The Worker puts a Turnstile check in front of the app (verified server-side; a signed `HttpOnly` `__Host-ytg_gate` cookie remembers the pass for 24 hours), serves the prerendered legal pages at `/legal/*` to everyone, and sets the security headers (a CSP per page type). Agreeing to the terms on the gate page sets the `ytg_terms` cookie, which the app reads so a web visitor meets one blocking screen, not two. Build-time extras for the web only — static legal pages, `/.well-known/security.txt`, the cookieless Web Analytics beacon — come from `build-plugins/`. The desktop and Android builds use none of this and contain no analytics. Request flow, cookies, headers and operations: [HOSTING.md](HOSTING.md). +The web build is served at by a Cloudflare Worker (`worker/`, configured by `wrangler.jsonc`) that Cloudflare Workers Builds deploys from `main`. The Worker puts a Turnstile check in front of the app (verified server-side; a signed `HttpOnly` `__Host-ytg_gate` cookie remembers the pass for 24 hours), serves the prerendered legal pages at `/legal/*` to everyone, sets the security headers (a CSP per page type), and answers `429` to a client that sends too many requests (Workers Rate Limiting bindings, behind a WAF rate limiting rule at the zone). Agreeing to the terms on the gate page sets the `ytg_terms` cookie, which the app reads so a web visitor meets one blocking screen, not two. Build-time extras for the web only — static legal pages, `/.well-known/security.txt`, the cookieless Web Analytics beacon — come from `build-plugins/`. The desktop and Android builds use none of this and contain no analytics. Request flow, cookies, headers and operations: [HOSTING.md](HOSTING.md). ## 7. Desktop Packaging (Tauri) diff --git a/docs/HOSTING.md b/docs/HOSTING.md index 30eff07..397c944 100644 --- a/docs/HOSTING.md +++ b/docs/HOSTING.md @@ -49,13 +49,52 @@ If `TURNSTILE_SECRET` is missing the gate **fails open** (the site stays up and - **`/.well-known/security.txt`** (RFC 9116) is regenerated on every build, so `Expires` is always about six months ahead. - **Legal pages** are rendered from the repository's Markdown at build time, to static `/legal/*.html` and to a lazy chunk for the in-app Legal Center. +## Rate limiting + +Two layers, cheapest first. Both count per client IP — before the gate there is no other identity — so the limits are generous enough for several people behind one shared address (mobile carrier NAT, an office, a school). + +| Layer | Configured in | Counts | Limit | Over the limit | +| ----------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------ | +| 1. WAF rate limiting rule | Cloudflare dashboard, [by hand](#waf-rate-limiting-rule-one-time) | Every request at the edge except `/assets/*`, per IP; verified bots (search engines) excluded | 60 / 10 s | Cloudflare's own `429` page (error 1015) for 10 s. The Worker never runs, so a blocked flood costs nothing | +| 2. Worker `RATE_LIMIT` | `wrangler.jsonc` › `ratelimits`, checked first in [`worker/rate-limit.ts`](../worker/rate-limit.ts) | Every request the Worker handles — the gate page, the app shell, legal pages, `/__gate/*`, any method — per IPv4 address or IPv6 /64 | 120 / 60 s | `429` plain text, `Retry-After: 60`, `Cache-Control: no-store`, the baseline headers; nothing else runs | +| 2. Worker `VERIFY_RATE_LIMIT` | `wrangler.jsonc` › `ratelimits` | `POST /__gate/verify`, on top of `RATE_LIMIT` | 10 / 60 s | `429` JSON `{"ok":false,"error":"rate-limited"}` before any siteverify call; the gate page shows its "try again" message | + +- **Approximate, per location.** The Workers Rate Limiting API keeps its counters per Cloudflare location and updates them asynchronously, so a burst can pass the limit by a little. It turns a flood from one client into cheap `429`s; it is not a quota. Many addresses at once (a botnet) are the job of the WAF, Bot Fight Mode and the Turnstile gate itself. +- **IPv6 by /64.** One connection usually holds a whole /64, so counting single addresses would let a client rotate past the limit. +- **Fails open**, like the gate: a missing binding or a failing `limit()` call lets the request through and logs once per isolate. +- **Not counted by the Worker**: `/assets/*` and the public files that skip the Worker (see [Request flow](#request-flow)). Static-asset requests are free and are not Worker invocations; only the WAF rule sees them. +- **Changing a limit**: edit `limit` in `wrangler.jsonc` and merge. `period` may only be 10 or 60 and must equal `RATE_LIMIT_PERIOD_SECONDS` in [`worker/env.ts`](../worker/env.ts) (it is the `Retry-After`); `namespace_id` must be unique in the Cloudflare account, and changing it starts the counters from zero. `tests/worker/rate-limit.test.ts` checks the bindings against the code. +- **Seeing it work**: the bindings don't appear in the dashboard. The Worker's `429`s are in its logs (Observability); the WAF's blocks are under Security › Events. + +### WAF rate limiting rule (one-time) + +The zone is on the Free plan, which allows **one** rate limiting rule: counted by IP, over 10 seconds, blocking for 10 seconds, with Cloudflare's default response (a custom response needs Pro). It runs at the edge before the Worker, so it is the first layer. + +1. Cloudflare dashboard › `ytgenerator.stream` › **Security** › **Security rules** › **Create rule** › **Rate limiting rules**. +2. **Rule name**: `Flood guard`. +3. **If incoming requests match** › **Edit expression**: + + ```txt + (not http.request.uri.path contains "/assets/" and not cf.bot_management.verified_bot) + ``` + + In the builder that is _URI Path_ · _does not contain_ · `/assets/` **And** _Verified Bot_ · _equals_ · Off. Content-hashed build output is left out because a first visit fetches dozens of those files at once and a returning browser caches them for a year; verified bots so that search engines are never blocked. + +4. **With the same characteristics**: IP (the only choice on Free). +5. **When rate exceeds**: **Requests** `60`, **Period** 10 seconds. A page load is about five counted requests (the page, `theme-init.js`, favicon, manifest, an icon), so 60 is a dozen reloads in ten seconds. +6. **Then take action**: **Block**; **Duration** 10 seconds. +7. **Deploy**. + +If the dashboard refuses the expression, match all incoming requests instead and raise **Requests** to `100` — a first visit with an empty cache is about 40 requests once the build output counts (measured on the v1.3.2 build: the page, `theme-init.js` and 33 files under `/assets/`, plus favicon and manifest). + ## Configuration and secrets -| Name | Kind | Where | -| -------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------- | -| `TURNSTILE_SITE_KEY` | public var | `wrangler.jsonc` | -| `GATE_TTL_SECONDS` | public var | `wrangler.jsonc` | -| `TURNSTILE_SECRET` | **Secret** | Cloudflare dashboard (Worker › Settings › Variables and Secrets) or `npx wrangler secret put TURNSTILE_SECRET`. Never committed | +| Name | Kind | Where | +| --------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `TURNSTILE_SITE_KEY` | public var | `wrangler.jsonc` | +| `GATE_TTL_SECONDS` | public var | `wrangler.jsonc` | +| `TURNSTILE_SECRET` | **Secret** | Cloudflare dashboard (Worker › Settings › Variables and Secrets) or `npx wrangler secret put TURNSTILE_SECRET`. Never committed | +| `RATE_LIMIT`, `VERIFY_RATE_LIMIT` | rate limit bindings | `wrangler.jsonc` › `ratelimits` (see [Rate limiting](#rate-limiting)) | It must be a **Secret**, not a plain-text variable: `wrangler deploy` removes plain-text variables that `wrangler.jsonc` doesn't declare, and keeps secrets. @@ -70,6 +109,7 @@ These live in Cloudflare, not in the repository: 3. **Web Analytics** — the manual beacon is used; turn off _automatic setup_ for the hostname so visits aren't counted twice. 4. **Redirect rule** — `www.ytgenerator.stream/*` → `concat("https://ytgenerator.stream", http.request.uri.path)`, 301, preserving the query string. 5. **HSTS** — enabled at the zone with `includeSubDomains` and `preload`. +6. **WAF rate limiting rule** — `Flood guard`, 60 requests per 10 s per IP outside `/assets/`; see [WAF rate limiting rule](#waf-rate-limiting-rule-one-time). ## Local development @@ -98,3 +138,14 @@ curl -s https://ytgenerator.stream/.well-known/security.txt curl -sI https://ytgenerator.stream/legal/privacy # public, script-free CSP curl -sI https://www.ytgenerator.stream/x?y=1 # 301 to https://ytgenerator.stream/x?y=1 ``` + +Rate limits: a burst from one address should turn into `429`s. This locks that address out of the site for up to a minute. + +```bash +# Worker layer: about 120 × 200, then 429 with Retry-After: 60 +for i in $(seq 1 130); do curl -s -o /dev/null -w "%{http_code}\n" https://ytgenerator.stream/; done | sort | uniq -c +# verify: 403 (foreign Origin) ten times, then 429 — no siteverify call is made +for i in $(seq 1 12); do curl -s -o /dev/null -w "%{http_code}\n" -X POST -H "Origin: https://example.com" -H "Content-Type: application/json" -d "{}" https://ytgenerator.stream/__gate/verify; done | sort | uniq -c +``` + +With the WAF rule in place, a fast enough burst (more than 60 in 10 s) is stopped at the edge first: Cloudflare's `429` (error 1015) instead of the Worker's. diff --git a/docs/TECH-SPEC.md b/docs/TECH-SPEC.md index 189bd51..0c5db05 100644 --- a/docs/TECH-SPEC.md +++ b/docs/TECH-SPEC.md @@ -307,6 +307,10 @@ The only Cloudflare file in the repository; Workers Builds runs `npx wrangler de "!/404.html" ] }, + "ratelimits": [ + { "name": "RATE_LIMIT", "namespace_id": "1001", "simple": { "limit": 120, "period": 60 } }, + { "name": "VERIFY_RATE_LIMIT", "namespace_id": "1002", "simple": { "limit": 10, "period": 60 } } + ], "vars": { "TURNSTILE_SITE_KEY": "0x4AAAAAAFKFAMUx9rQ5H6bQ", "GATE_TTL_SECONDS": "86400" @@ -315,9 +319,9 @@ The only Cloudflare file in the repository; Workers Builds runs `npx wrangler de } ``` -Every request goes through the Worker first except content-hashed build output and the public files browsers and crawlers fetch without cookies. `not_found_handling` is `404-page`, not the SPA fallback, so no path can reach `index.html` without passing the gate. `TURNSTILE_SECRET` is a dashboard Secret, never committed. Request flow, cookies, headers and operations: [HOSTING.md](HOSTING.md). +Every request goes through the Worker first except content-hashed build output and the public files browsers and crawlers fetch without cookies. `not_found_handling` is `404-page`, not the SPA fallback, so no path can reach `index.html` without passing the gate. `TURNSTILE_SECRET` is a dashboard Secret, never committed. The two `ratelimits` bindings (Workers Rate Limiting API, v1.3.2) cap each client — an IPv4 address or an IPv6 /64 — at 120 Worker requests and 10 verify calls a minute, behind the zone's WAF rate limiting rule. Request flow, cookies, headers, rate limits and operations: [HOSTING.md](HOSTING.md). -The Worker (`worker/`): `index.ts` (entry; injects the clock, `fetch` and random ids), `router.ts` (routing), `env.ts` (bindings, constants, the gate lifetime clamped to 5 min – 30 days), `turnstile.ts` (siteverify call and verdict: success, hostname, action `enter`, token age ≤ 300 s), `cookie.ts` (the HMAC-signed `__Host-ytg_gate` cookie, key derived from the Turnstile secret, bound to a User-Agent hash), `headers.ts` (CSP per page type and the baseline security headers), `gate-page.ts` + `gate-i18n.ts` (the gate page in the app's eight languages). It uses only standard Web APIs, so it type-checks against the DOM lib and runs under Vitest in Node. +The Worker (`worker/`): `index.ts` (entry; injects the clock, `fetch` and random ids), `router.ts` (routing), `rate-limit.ts` (the per-client limit checked before anything else: client key, fail-open check, the `429`), `env.ts` (bindings, constants, the gate lifetime clamped to 5 min – 30 days), `turnstile.ts` (siteverify call and verdict: success, hostname, action `enter`, token age ≤ 300 s), `cookie.ts` (the HMAC-signed `__Host-ytg_gate` cookie, key derived from the Turnstile secret, bound to a User-Agent hash), `headers.ts` (CSP per page type and the baseline security headers), `gate-page.ts` + `gate-i18n.ts` (the gate page in the app's eight languages). It uses only standard Web APIs, so it type-checks against the DOM lib and runs under Vitest in Node. ## 2. Core Engine Interfaces @@ -641,7 +645,7 @@ Then `sonarcloud` (after `check` succeeds): ## 6. Testing Strategy -Vitest runs `tests/**/*.test.{ts,tsx}` and `src/**/*.test.{ts,tsx}` in the **node** environment, with `globals: true`: 984 tests in 51 files at v1.3.1. Directories: `tests/engine`, `store`, `utils` (backup detect / sanitize / restore, quick entry, validation, the migration page), `worker` (router, cookie, Turnstile verdicts, headers, gate language), `build` (legal pages, web-only plugins), `config`, `hooks`, `i18n`, `scripts`. No test renders components: React Testing Library is installed but jsdom is not, so a rendering test would need both jsdom and a `// @vitest-environment jsdom` docblock. +Vitest runs `tests/**/*.test.{ts,tsx}` and `src/**/*.test.{ts,tsx}` in the **node** environment, with `globals: true`: 984 tests in 51 files at v1.3.1. Directories: `tests/engine`, `store`, `utils` (backup detect / sanitize / restore, quick entry, validation, the migration page), `worker` (router, rate limits, cookie, Turnstile verdicts, headers, gate language), `build` (legal pages, web-only plugins), `config`, `hooks`, `i18n`, `scripts`. No test renders components: React Testing Library is installed but jsdom is not, so a rendering test would need both jsdom and a `// @vitest-environment jsdom` docblock. Engine tests build a full `GeneratorInput` and pass a mock translator (`tests/helpers/mock-t.ts`) that reads the real `templates.json` files (en, vi, ja): diff --git a/tests/worker/rate-limit.test.ts b/tests/worker/rate-limit.test.ts new file mode 100644 index 0000000..6d2cd4e --- /dev/null +++ b/tests/worker/rate-limit.test.ts @@ -0,0 +1,154 @@ +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { describe, it, expect, vi } from "vitest"; +import { RATE_LIMIT_PERIOD_SECONDS, type Env } from "../../worker/env"; +import { isRateLimited, rateLimitKey, tooManyRequests } from "../../worker/rate-limit"; + +interface RateLimitConfig { + name: string; + namespace_id: string; + simple: { limit: number; period: number }; +} + +/** wrangler.jsonc without its `//` comments (it has no trailing commas). */ +function readWranglerConfig(): { ratelimits?: RateLimitConfig[] } { + const text = readFileSync(resolve(import.meta.dirname, "../../wrangler.jsonc"), "utf8"); + let json = ""; + let inString = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]!; + if (inString) { + json += ch; + if (ch === "\\") json += text[++i] ?? ""; + else if (ch === '"') inString = false; + } else if (ch === '"') { + inString = true; + json += ch; + } else if (ch === "/" && text[i + 1] === "/") { + while (i < text.length && text[i] !== "\n") i++; + json += "\n"; + } else { + json += ch; + } + } + return JSON.parse(json) as { ratelimits?: RateLimitConfig[] }; +} + +function envWith(overrides: Partial): Env { + return { + ASSETS: { fetch: async () => new Response("") }, + TURNSTILE_SITE_KEY: "site-key", + TURNSTILE_SECRET: "secret", + ...overrides, + }; +} + +const request = (ip?: string) => + new Request("https://ytgenerator.stream/", { + headers: ip === undefined ? {} : { "CF-Connecting-IP": ip }, + }); + +describe("rateLimitKey", () => { + it.each([ + [null, "unknown"], + ["", "unknown"], + ["203.0.113.9", "203.0.113.9"], + [" 203.0.113.9 ", "203.0.113.9"], + ["2001:DB8:0001:0002:0003:4:5:6", "2001:db8:1:2::/64"], + ["2001:db8:1:2::1", "2001:db8:1:2::/64"], + ["2001:db8::1", "2001:db8:0:0::/64"], + ["::1", "0:0:0:0::/64"], + ["fe80::", "fe80:0:0:0::/64"], + ["::", "0:0:0:0::/64"], + ["::ffff:198.51.100.4", "198.51.100.4"], + ["64:ff9b::192.0.2.1", "64:ff9b:0:0::/64"], + ["1:2:3:4:5:6:192.0.2.1", "1:2:3:4::/64"], + ])("%s → %s", (ip, key) => { + expect(rateLimitKey(ip)).toBe(key); + }); + + it.each(["1::2::3", "1:2:3", "1:2:3:4:5:6:7:8:9", "1:2:3:4:5:6:7::8", "zz::1", "12345::1"])( + "keeps the unparseable %s as it is", + (ip) => { + expect(rateLimitKey(ip)).toBe(ip); + }, + ); +}); + +describe("isRateLimited", () => { + it("lets traffic through without a binding, and says so once", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const env = envWith({}); + expect(await isRateLimited(request("203.0.113.9"), env, true)).toBe(false); + expect(await isRateLimited(request("203.0.113.9"), env, false)).toBe(false); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + }); + + it("lets traffic through when the limiter fails, and says so once", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const broken = { + limit: vi.fn(async (): Promise<{ success: boolean }> => { + throw new Error("unavailable"); + }), + }; + const env = envWith({ RATE_LIMIT: broken, VERIFY_RATE_LIMIT: broken }); + expect(await isRateLimited(request("203.0.113.9"), env, true)).toBe(false); + expect(await isRateLimited(request("203.0.113.9"), env, true)).toBe(false); + expect(broken.limit).toHaveBeenCalledTimes(4); + expect(error).toHaveBeenCalledTimes(1); + error.mockRestore(); + }); + + it("asks each limiter with the client's key", async () => { + const general = { limit: vi.fn(async () => ({ success: true })) }; + const verify = { limit: vi.fn(async () => ({ success: false })) }; + const env = envWith({ RATE_LIMIT: general, VERIFY_RATE_LIMIT: verify }); + expect(await isRateLimited(request("2001:db8::7"), env, false)).toBe(false); + expect(await isRateLimited(request("2001:db8::7"), env, true)).toBe(true); + expect(general.limit).toHaveBeenCalledWith({ key: "2001:db8:0:0::/64" }); + expect(verify.limit).toHaveBeenCalledTimes(1); + expect(await isRateLimited(request(), env, false)).toBe(false); + expect(general.limit).toHaveBeenLastCalledWith({ key: "unknown" }); + }); +}); + +describe("tooManyRequests", () => { + it("answers the verify call in JSON and everything else in plain text", async () => { + const json = tooManyRequests(true); + const text = tooManyRequests(false); + for (const res of [json, text]) { + expect(res.status).toBe(429); + expect(res.headers.get("Retry-After")).toBe(String(RATE_LIMIT_PERIOD_SECONDS)); + expect(res.headers.get("Cache-Control")).toBe("no-store"); + } + expect(await json.json()).toEqual({ ok: false, error: "rate-limited" }); + expect(text.headers.get("Content-Type")).toBe("text/plain; charset=utf-8"); + }); +}); + +describe("wrangler.jsonc ratelimits", () => { + const limits = readWranglerConfig().ratelimits ?? []; + + it("binds the two limiters the Worker reads", () => { + expect(limits.map((entry) => entry.name)).toEqual(["RATE_LIMIT", "VERIFY_RATE_LIMIT"]); + }); + + it("uses one period, the one Retry-After announces", () => { + for (const entry of limits) expect(entry.simple.period).toBe(RATE_LIMIT_PERIOD_SECONDS); + }); + + it("gives each limiter its own namespace and a positive limit", () => { + const ids = limits.map((entry) => entry.namespace_id); + expect(new Set(ids).size).toBe(ids.length); + for (const entry of limits) { + expect(entry.namespace_id).toMatch(/^[1-9]\d*$/); + expect(entry.simple.limit).toBeGreaterThan(0); + } + }); + + it("keeps verify stricter than the general limit", () => { + const [general, verify] = limits; + expect(verify!.simple.limit).toBeLessThan(general!.simple.limit); + }); +}); diff --git a/tests/worker/router.test.ts b/tests/worker/router.test.ts index 4f2f3e2..bdfdd6a 100644 --- a/tests/worker/router.test.ts +++ b/tests/worker/router.test.ts @@ -28,10 +28,27 @@ function fakeAssets() { return binding; } +/** A rate limiter that lets `allowed` requests per key through. */ +function fakeLimiter(allowed = Infinity) { + const keys: string[] = []; + const counts = new Map(); + return { + keys, + async limit({ key }: { key: string }) { + keys.push(key); + const count = (counts.get(key) ?? 0) + 1; + counts.set(key, count); + return { success: count <= allowed }; + }, + }; +} + function makeEnv(overrides: Partial = {}) { const assets = fakeAssets(); const env: Env = { ASSETS: assets, + RATE_LIMIT: fakeLimiter(), + VERIFY_RATE_LIMIT: fakeLimiter(), TURNSTILE_SITE_KEY: "site-key", TURNSTILE_SECRET: SECRET, GATE_TTL_SECONDS: "86400", @@ -314,3 +331,96 @@ describe("worker routing — missing secret", () => { errors.mockRestore(); }); }); + +describe("worker routing — rate limits", () => { + const fromIp = ( + ip: string, + path = "/", + init: { method?: string; body?: string; headers?: Record } = {}, + ) => req(path, { ...init, headers: { "CF-Connecting-IP": ip, ...init.headers } }); + + function verifyFrom(ip: string) { + return fromIp(ip, "/__gate/verify", { + method: "POST", + headers: { Origin: ORIGIN, "Content-Type": "application/json" }, + body: JSON.stringify({ token: "tok", consent: true }), + }); + } + + it("answers 429 with Retry-After once a client is over the limit, before any asset fetch", async () => { + const { env, assets } = makeEnv({ RATE_LIMIT: fakeLimiter(2) }); + const cookie = await gateCookie(); + const statuses: number[] = []; + let last: Response | undefined; + for (let i = 0; i < 3; i++) { + last = await handleRequest( + fromIp("203.0.113.7", "/", { headers: { Cookie: cookie } }), + env, + makeDeps(), + ); + statuses.push(last.status); + } + expect(statuses).toEqual([200, 200, 429]); + expect(last!.headers.get("Retry-After")).toBe("60"); + expect(last!.headers.get("Content-Type")).toContain("text/plain"); + expect(last!.headers.get("Cache-Control")).toBe("no-store"); + expect(last!.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(await last!.text()).toContain("Too many requests"); + expect(assets.calls).toHaveLength(2); + }); + + it("counts each client separately", async () => { + const { env } = makeEnv({ RATE_LIMIT: fakeLimiter(1) }); + expect((await handleRequest(fromIp("203.0.113.7"), env, makeDeps())).status).toBe(200); + expect((await handleRequest(fromIp("203.0.113.7"), env, makeDeps())).status).toBe(429); + expect((await handleRequest(fromIp("198.51.100.2"), env, makeDeps())).status).toBe(200); + }); + + it("counts a whole IPv6 /64 as one client", async () => { + const { env } = makeEnv({ RATE_LIMIT: fakeLimiter(1) }); + expect((await handleRequest(fromIp("2001:db8:1:2::1"), env, makeDeps())).status).toBe(200); + expect((await handleRequest(fromIp("2001:db8:1:2:aa::9"), env, makeDeps())).status).toBe(429); + expect((await handleRequest(fromIp("2001:db8:1:3::1"), env, makeDeps())).status).toBe(200); + }); + + it("covers the public routes and other methods too", async () => { + const { env, assets } = makeEnv({ RATE_LIMIT: fakeLimiter(0) }); + for (const request of [ + fromIp("203.0.113.7", "/legal/privacy"), + fromIp("203.0.113.7", "/__gate/status"), + fromIp("203.0.113.7", "/", { method: "PUT" }), + ]) { + expect((await handleRequest(request, env, makeDeps())).status).toBe(429); + } + expect(assets.calls).toEqual([]); + }); + + it("answers a verify flood with JSON 429 and no siteverify call", async () => { + const { env } = makeEnv({ VERIFY_RATE_LIMIT: fakeLimiter(1) }); + const deps = makeDeps({ + success: true, + hostname: "ytgenerator.stream", + action: "enter", + challenge_ts: "2026-09-30T11:59:30Z", + }); + expect((await handleRequest(verifyFrom("203.0.113.7"), env, deps)).status).toBe(200); + const res = await handleRequest(verifyFrom("203.0.113.7"), env, deps); + expect(res.status).toBe(429); + expect(res.headers.get("Retry-After")).toBe("60"); + expect(await res.json()).toEqual({ ok: false, error: "rate-limited" }); + expect(res.headers.getSetCookie()).toEqual([]); + expect(deps.fetch).toHaveBeenCalledTimes(1); + }); + + it("spends the verify budget on verify requests only", async () => { + const verifyLimiter = fakeLimiter(); + const { env } = makeEnv({ RATE_LIMIT: fakeLimiter(1), VERIFY_RATE_LIMIT: verifyLimiter }); + await handleRequest(fromIp("203.0.113.7", "/legal"), env, makeDeps()); + expect(verifyLimiter.keys).toEqual([]); + // Over the general limit: refused as JSON, without touching the verify one. + const res = await handleRequest(verifyFrom("203.0.113.7"), env, makeDeps()); + expect(res.status).toBe(429); + expect(res.headers.get("Content-Type")).toContain("application/json"); + expect(verifyLimiter.keys).toEqual([]); + }); +}); diff --git a/worker/env.ts b/worker/env.ts index 3db0191..4be8f4a 100644 --- a/worker/env.ts +++ b/worker/env.ts @@ -13,8 +13,18 @@ export interface AssetsBinding { fetch(request: Request): Promise; } +/** A `ratelimits` binding from `wrangler.jsonc` (Workers Rate Limiting API). */ +export interface RateLimitBinding { + limit(options: { key: string }): Promise<{ success: boolean }>; +} + export interface Env { ASSETS: AssetsBinding; + /** Every request the Worker handles, per client. Optional: a missing + * binding lets traffic through (see ./rate-limit.ts). */ + RATE_LIMIT?: RateLimitBinding; + /** `POST /__gate/verify` only — each one costs a siteverify call. */ + VERIFY_RATE_LIMIT?: RateLimitBinding; /** Public Turnstile sitekey (`vars` in wrangler.jsonc). */ TURNSTILE_SITE_KEY: string; /** Turnstile secret — a dashboard Secret, never committed. */ @@ -43,6 +53,10 @@ export const SITEVERIFY_URL = "https://challenges.cloudflare.com/turnstile/v0/si /** Turnstile tokens are valid for 300 s; anything older is refused. */ export const MAX_TOKEN_AGE_SECONDS = 300; +/** `Retry-After` on a 429: the `period` of both rate limiters in + * wrangler.jsonc (a test keeps them equal). */ +export const RATE_LIMIT_PERIOD_SECONDS = 60; + const DEFAULT_GATE_TTL_SECONDS = 86_400; const MIN_GATE_TTL_SECONDS = 300; const MAX_GATE_TTL_SECONDS = 30 * 86_400; diff --git a/worker/rate-limit.ts b/worker/rate-limit.ts new file mode 100644 index 0000000..4f95174 --- /dev/null +++ b/worker/rate-limit.ts @@ -0,0 +1,97 @@ +import { RATE_LIMIT_PERIOD_SECONDS, type Env, type RateLimitBinding } from "./env"; +import { jsonResponse, plainResponse } from "./headers"; + +/** + * Per-client rate limiting with the Workers Rate Limiting API — the second + * layer, behind the zone's WAF rate limiting rule (docs/HOSTING.md). + * + * The limits live in `wrangler.jsonc` (`ratelimits`). Cloudflare keeps the + * counters per location and they are approximate: they turn a flood from one + * client into cheap 429s before any siteverify call, HMAC check or asset + * fetch; they are not a quota. + */ + +const IPV4_MAPPED_PREFIX = "::ffff:"; +const HEX_GROUP = /^[0-9a-f]{1,4}$/; + +let warnedMissingBinding = false; +let warnedFailedLimit = false; + +/** The first four groups of an IPv6 address, or null when it doesn't parse. */ +function ipv6Prefix64(address: string): string[] | null { + const halves = address.split("::"); + if (halves.length > 2) return null; + const [before = "", after] = halves; + const head = before === "" ? [] : before.split(":"); + const tail = after === undefined || after === "" ? [] : after.split(":"); + // A trailing dotted quad ("…:1.2.3.4") fills two groups. + const width = head.length + tail.length + (address.includes(".") ? 1 : 0); + const missing = 8 - width; + if (after === undefined ? missing !== 0 : missing < 1) return null; + const groups = [...head, ...Array(missing).fill("0"), ...tail].slice(0, 4); + if (!groups.every((group) => HEX_GROUP.test(group))) return null; + return groups.map((group) => Number.parseInt(group, 16).toString(16)); +} + +/** + * The counter key for a client (`CF-Connecting-IP`): an IPv4 address as is, + * an IPv6 address as its /64 — one connection usually holds a whole /64, so + * counting single addresses would let a client rotate past the limit. + */ +export function rateLimitKey(ip: string | null): string { + const raw = (ip ?? "").trim().toLowerCase(); + if (raw === "") return "unknown"; + if (!raw.includes(":")) return raw; + if (raw.startsWith(IPV4_MAPPED_PREFIX) && raw.includes(".")) { + return raw.slice(IPV4_MAPPED_PREFIX.length); + } + const prefix = ipv6Prefix64(raw); + return prefix === null ? raw : `${prefix.join(":")}::/64`; +} + +/** + * Has this client gone over the limit? Fails open — with no binding, or when + * the call throws, the request goes through: a broken limiter must not take + * the site down. Each problem is logged once per isolate, not per request. + */ +async function overLimit(binding: RateLimitBinding | undefined, key: string): Promise { + if (!binding) { + if (!warnedMissingBinding) { + console.warn("rate-limit: a ratelimits binding is missing — not limiting"); + warnedMissingBinding = true; + } + return false; + } + try { + const { success } = await binding.limit({ key }); + return !success; + } catch (err) { + if (!warnedFailedLimit) { + console.error("rate-limit: limit() failed — letting requests through", err); + warnedFailedLimit = true; + } + return false; + } +} + +/** + * Every request counts against `RATE_LIMIT`; a verify request also against + * `VERIFY_RATE_LIMIT` (only while the first one still lets it through). + */ +export async function isRateLimited( + request: Request, + env: Env, + isVerify: boolean, +): Promise { + const key = rateLimitKey(request.headers.get("CF-Connecting-IP")); + if (await overLimit(env.RATE_LIMIT, key)) return true; + return isVerify && (await overLimit(env.VERIFY_RATE_LIMIT, key)); +} + +/** The 429 itself: JSON for the gate page's verify call, plain text otherwise. */ +export function tooManyRequests(isVerify: boolean): Response { + const extra = { "Retry-After": String(RATE_LIMIT_PERIOD_SECONDS) }; + return isVerify + ? jsonResponse(429, { ok: false, error: "rate-limited" }, extra) + : plainResponse(429, "Too many requests. Wait a minute, then reload the page.", extra); +} diff --git a/worker/router.ts b/worker/router.ts index 2d63d53..98890cf 100644 --- a/worker/router.ts +++ b/worker/router.ts @@ -12,13 +12,16 @@ import { plainResponse, withSecurityHeaders, } from "./headers"; +import { isRateLimited, tooManyRequests } from "./rate-limit"; import { evaluateSiteverify, siteverify } from "./turnstile"; /** * Request routing for ytgenerator.stream. * * `wrangler.jsonc` sends everything except hashed build output and a handful - * of public files (icons, robots, security.txt …) through here first: + * of public files (icons, robots, security.txt …) through here first. + * Every request is rate-limited per client before anything else + * (./rate-limit.ts); then: * * /__gate/verify POST: check a Turnstile token, set the gate cookie * /__gate/status 204 if the gate cookie is valid (cookie-blocked check) @@ -193,8 +196,11 @@ function serveGate(request: Request, env: Env, deps: Deps, url: URL): Response { export async function handleRequest(request: Request, env: Env, deps: Deps): Promise { const url = new URL(request.url); + const isVerify = url.pathname === VERIFY_PATH; - if (url.pathname === VERIFY_PATH) { + if (await isRateLimited(request, env, isVerify)) return tooManyRequests(isVerify); + + if (isVerify) { if (!env.TURNSTILE_SECRET) return jsonResponse(503, { ok: false, error: "not-configured" }); return handleVerify(request, env, deps, url); } diff --git a/wrangler.jsonc b/wrangler.jsonc index 8980e3a..a70e478 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -49,6 +49,27 @@ ] }, + // Per-client rate limits (Workers Rate Limiting API): the second layer, + // behind the zone's WAF rate limiting rule — see docs/HOSTING.md. Counted + // per Cloudflare location, approximately, by IPv4 address or IPv6 /64. + // Over the limit → 429 with Retry-After. `period` may be 10 or 60 and must + // equal RATE_LIMIT_PERIOD_SECONDS in worker/env.ts; `namespace_id` is an + // integer (as a string) unique within the Cloudflare account. + "ratelimits": [ + // Every request the Worker handles: the gate, the app shell, legal pages. + { + "name": "RATE_LIMIT", + "namespace_id": "1001", + "simple": { "limit": 120, "period": 60 } + }, + // POST /__gate/verify on top of that — each one costs a siteverify call. + { + "name": "VERIFY_RATE_LIMIT", + "namespace_id": "1002", + "simple": { "limit": 10, "period": 60 } + } + ], + "vars": { // Public by design — the widget embeds it in the page. The matching // secret is the dashboard Secret TURNSTILE_SECRET (never committed).