feat(worker): review queue, admin API, ingest and suggest endpoints (v4 Phase 4)

Adds the server side of the admin and the automatic game discovery. The
catalog stays JSON in this repo; the Worker writes it as a GitHub App
(GraphQL createCommitOnBranch with expectedHeadOid: verified commits,
retried on a race) and keeps only the review queue in D1.

- worker/admin.ts: /api/admin/* (queue list, approve/reject, add, edit the
  three maintainer fields, remove with reason + R2 cover cleanup, restore)
  and the /admin shell with its own CSP. Every request re-verifies the
  Cloudflare Access JWT (aud + maintainer email + same-origin writes).
  Approving commits to scripts/temp_link.json, which starts update.yml.
  The maintainer's email goes to the D1 audit log only, never a commit.
- worker/ingest.ts: POST /api/ingest for the browser extension (Access
  service token, per-client rate limit, Idempotency-Key replay, <= 150
  items / 64 KB), replacing its PAT commits.
- worker/suggest.ts: public POST /api/suggest (same-origin, per-IP rate
  limit, Turnstile siteverify, canonical itch.io URL) into the queue.
- worker/discover.ts + scheduled(): every 4 h, one itch.io RSS feed
  (round-robin, 6 h back-off on 429) into the queue, then reconcile
  (queued -> ingested / failed) and retention (idempotency 7 d, form
  notes 180 d).
- worker/queue.ts: URL classification (catalog, pending, deleted,
  blocklist) and the D1 store; worker/repo.ts builds each admin commit
  from the files at the head of main, byte-compatible with the Python
  pipeline (golden fixtures).
- wrangler.jsonc: D1 (bound by name; migrations applied by hand), rate
  limits, cron, non-secret settings as vars (empty until setup: features
  answer 503 until configured), staging env without the cron.
  `npm run cf:types` ignores .dev.vars so the committed types are stable.
- Tests: 159 passing; the SQL runs against a real local D1 (workerd via
  wrangler's platform proxy).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5 committed 2026-09-18 17:55:01 +07:00
1 parent ce177d8f27
commit 8f121a8e9c
38 files changed
+3276 -17

No files matched your search

+3 -3
View File
@@ -40,7 +40,7 @@ jobs:
run: npm ci
- name: Worker types are up to date
run: npx wrangler types --check
run: npm run cf:types -- --check
- name: Lint
run: npm run lint
@@ -48,11 +48,11 @@ jobs:
- name: Unused code (knip)
run: npm run knip
- name: Unit tests (Worker)
- name: Unit tests (Worker, app; D1 tests run a local workerd)
run: npm test
- name: Type-check and build
run: npm run build
- name: Validate Workers config (dry run)
run: npx wrangler deploy --dry-run
run: npx wrangler deploy --dry-run --env=""
+10
View File
@@ -28,6 +28,7 @@
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"idb-keyval": "^6.3.0",
"jose": "^6.2.12",
"lucide-react": "^1.47.0",
"react": "^19.3.0",
"react-dom": "^19.3.0",
@@ -6189,6 +6190,15 @@
"jiti": "bin/jiti.js"
}
},
"node_modules/jose": {
"version": "6.2.12",
"resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz",
"integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
+2 -1
View File
@@ -16,7 +16,7 @@
"tauri:dev": "tauri dev",
"tauri:build": "tauri build",
"test": "vitest run",
"cf:types": "wrangler types",
"cf:types": "wrangler types --env-file=types.env --strict-vars=false",
"cf:dev": "wrangler dev"
},
"dependencies": {
@@ -40,6 +40,7 @@
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"idb-keyval": "^6.3.0",
"jose": "^6.2.12",
"lucide-react": "^1.47.0",
"react": "^19.3.0",
"react-dom": "^19.3.0",
+1
View File
@@ -152,6 +152,7 @@ export const THIRD_PARTY: ThirdParty[] = [
{ name: 'idb-keyval', version: '6.3', license: 'Apache-2.0', url: 'https://github.com/jakearchibald/idb-keyval', category: 'data' },
// Hosting / tooling (Cloudflare)
{ name: 'jose', version: '6.2', license: 'MIT', url: 'https://github.com/panva/jose', category: 'dev' },
{ name: 'Wrangler', version: '4.134', license: 'MIT OR Apache-2.0', url: 'https://developers.cloudflare.com/workers/wrangler/', category: 'dev' },
{ name: 'Vitest', version: '5.0', license: 'MIT', url: 'https://vitest.dev', category: 'dev' },
+1
View File
@@ -5,6 +5,7 @@
"lib": ["ES2024"],
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"types": [],
"skipLibCheck": true,
"strict": true,
+2
View File
@@ -0,0 +1,2 @@
# Empty on purpose: `npm run cf:types` points wrangler here so secrets in a
# local .dev.vars never end up in the committed worker-configuration.d.ts.
+31 -1
View File
@@ -1,9 +1,21 @@
/* eslint-disable */
// Generated by Wrangler by running `wrangler types` (hash: b477d9057afc335e7b8bb1ffae6b449c)
// Generated by Wrangler by running `wrangler types --env-file=types.env --strict-vars=false` (hash: f483b1f0658395bde79d44bd8ede839a)
// Runtime types generated with workerd@1.20260917.1 2026-09-15
interface __BaseEnv_Env {
THUMBS: R2Bucket;
DB: D1Database;
RL_SUGGEST: RateLimit;
RL_INGEST: RateLimit;
ASSETS: Fetcher;
SITE_ORIGIN: string;
GITHUB_REPO: string;
BOT_UA: string;
GH_APP_ID: string;
GH_APP_INSTALLATION_ID: string;
ACCESS_TEAM_DOMAIN: string;
ACCESS_AUD_ADMIN: string;
ACCESS_AUD_INGEST: string;
TURNSTILE_SITEKEY: string;
}
declare namespace Cloudflare {
interface GlobalProps {
@@ -11,11 +23,29 @@ declare namespace Cloudflare {
}
interface StagingEnv {
THUMBS: R2Bucket;
DB: D1Database;
RL_SUGGEST: RateLimit;
RL_INGEST: RateLimit;
ASSETS: Fetcher;
SITE_ORIGIN: string;
GITHUB_REPO: string;
BOT_UA: string;
GH_APP_ID: string;
GH_APP_INSTALLATION_ID: string;
ACCESS_TEAM_DOMAIN: string;
ACCESS_AUD_ADMIN: string;
ACCESS_AUD_INGEST: string;
TURNSTILE_SITEKEY: string;
}
interface Env extends __BaseEnv_Env {}
}
interface Env extends __BaseEnv_Env {}
type StringifyValues<EnvType extends Record<string, unknown>> = {
[Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string;
};
declare namespace NodeJS {
interface ProcessEnv extends StringifyValues<Pick<Cloudflare.Env, "SITE_ORIGIN" | "GITHUB_REPO" | "BOT_UA" | "GH_APP_ID" | "GH_APP_INSTALLATION_ID" | "ACCESS_TEAM_DOMAIN" | "ACCESS_AUD_ADMIN" | "ACCESS_AUD_INGEST" | "TURNSTILE_SITEKEY">> {}
}
// Begin runtime types
/*! *****************************************************************************
+97
View File
@@ -0,0 +1,97 @@
import { SignJWT, exportJWK, generateKeyPair } from 'jose'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { AccessError, requireMaintainer, requireService, type AccessEnv } from './access'
const TEAM = 'https://example.cloudflareaccess.com'
const env: AccessEnv = {
ACCESS_TEAM_DOMAIN: TEAM,
ACCESS_AUD_ADMIN: 'aud-admin',
ACCESS_AUD_INGEST: 'aud-ingest',
ADMIN_EMAILS: 'Owner@Example.com, second@example.com',
SITE_ORIGIN: 'https://freeitchgames.win',
}
let privateKey: CryptoKey
beforeAll(async () => {
const pair = await generateKeyPair('RS256')
privateKey = pair.privateKey
const jwk = { ...(await exportJWK(pair.publicKey)), kid: 'k1', alg: 'RS256', use: 'sig' }
vi.stubGlobal('fetch', async (url: string) => {
if (String(url) === `${TEAM}/cdn-cgi/access/certs`) return Response.json({ keys: [jwk] })
return new Response(null, { status: 404 })
})
})
afterAll(() => vi.unstubAllGlobals())
async function token(claims: Record<string, unknown>, aud: string, iss = TEAM) {
return new SignJWT(claims)
.setProtectedHeader({ alg: 'RS256', kid: 'k1' })
.setIssuer(iss)
.setAudience(aud)
.setIssuedAt()
.setExpirationTime('5m')
.sign(privateKey)
}
function request(jwt: string | null, method = 'GET', origin?: string) {
const headers: Record<string, string> = {}
if (jwt) headers['cf-access-jwt-assertion'] = jwt
if (origin) headers.origin = origin
return new Request('https://freeitchgames.win/api/admin/me', { method, headers })
}
describe('requireMaintainer', () => {
it('accepts a listed email (case-insensitive)', async () => {
const jwt = await token({ email: 'owner@example.com' }, 'aud-admin')
await expect(requireMaintainer(request(jwt), env)).resolves.toEqual({
kind: 'maintainer',
id: 'owner@example.com',
})
})
it('rejects missing, forged, wrong-audience and wrong-issuer tokens', async () => {
await expect(requireMaintainer(request(null), env)).rejects.toBeInstanceOf(AccessError)
const other = await generateKeyPair('RS256')
const forged = await new SignJWT({ email: 'owner@example.com' })
.setProtectedHeader({ alg: 'RS256', kid: 'k1' })
.setIssuer(TEAM)
.setAudience('aud-admin')
.setExpirationTime('5m')
.sign(other.privateKey)
await expect(requireMaintainer(request(forged), env)).rejects.toThrow('invalid')
const wrongAud = await token({ email: 'owner@example.com' }, 'aud-ingest')
await expect(requireMaintainer(request(wrongAud), env)).rejects.toThrow('invalid')
const wrongIss = await token({ email: 'owner@example.com' }, 'aud-admin', 'https://evil.example')
await expect(requireMaintainer(request(wrongIss), env)).rejects.toThrow('invalid')
})
it('rejects emails that are not maintainers', async () => {
const jwt = await token({ email: 'someone@example.com' }, 'aud-admin')
await expect(requireMaintainer(request(jwt), env)).rejects.toThrow('not a maintainer')
})
it('requires the site origin on writes', async () => {
const jwt = await token({ email: 'owner@example.com' }, 'aud-admin')
await expect(requireMaintainer(request(jwt, 'POST'), env)).rejects.toThrow('bad origin')
await expect(requireMaintainer(request(jwt, 'POST', 'https://evil.example'), env)).rejects.toThrow('bad origin')
await expect(requireMaintainer(request(jwt, 'POST', env.SITE_ORIGIN), env)).resolves.toBeTruthy()
})
})
describe('requireService', () => {
it('accepts a service token and names it by client id', async () => {
const jwt = await token({ common_name: 'client-123.access' }, 'aud-ingest')
await expect(requireService(request(jwt, 'POST'), env)).resolves.toEqual({
kind: 'service',
id: 'client-123.access',
})
})
it('rejects user tokens and admin-audience tokens', async () => {
const user = await token({ email: 'owner@example.com', common_name: 'x' }, 'aud-ingest')
await expect(requireService(request(user), env)).rejects.toThrow('not a service token')
const adminAud = await token({ common_name: 'client-123.access' }, 'aud-admin')
await expect(requireService(request(adminAud), env)).rejects.toThrow('invalid')
})
})
+80
View File
@@ -0,0 +1,80 @@
/**
* Cloudflare Access verification for /admin and /api/admin/* (maintainer) and
* /api/ingest (service tokens, e.g. the browser extension).
*
* Access sits in front of those paths, but a Worker with static assets does
* not receive `ctx.access`, and requests that bypass the zone (workers.dev,
* previews — both disabled in wrangler.jsonc) would skip Access entirely.
* So every protected request re-verifies the `Cf-Access-Jwt-Assertion` JWT
* here: signature against the team's JWKS, issuer, and the application's AUD.
*/
import { createRemoteJWKSet, jwtVerify, type JWTPayload } from 'jose'
export interface AccessEnv {
ACCESS_TEAM_DOMAIN: string // https://<team>.cloudflareaccess.com
ACCESS_AUD_ADMIN: string
ACCESS_AUD_INGEST: string
ADMIN_EMAILS: string // comma-separated; a secret, not in the repo
SITE_ORIGIN: string
}
export interface Identity {
kind: 'maintainer' | 'service'
/** Email for people, service-token client id (common_name) for services. */
id: string
}
export class AccessError extends Error {}
const jwksByTeam = new Map<string, ReturnType<typeof createRemoteJWKSet>>()
function jwks(teamDomain: string) {
let set = jwksByTeam.get(teamDomain)
if (!set) {
set = createRemoteJWKSet(new URL(`${teamDomain}/cdn-cgi/access/certs`))
jwksByTeam.set(teamDomain, set)
}
return set
}
async function verify(request: Request, env: AccessEnv, audience: string): Promise<JWTPayload> {
const token = request.headers.get('cf-access-jwt-assertion')
if (!token) throw new AccessError('missing Access token')
try {
const { payload } = await jwtVerify(token, jwks(env.ACCESS_TEAM_DOMAIN), {
issuer: env.ACCESS_TEAM_DOMAIN,
audience,
})
return payload
} catch {
throw new AccessError('invalid Access token')
}
}
const SAFE_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])
/** The maintainer, signed in through the "Admin" Access application. */
export async function requireMaintainer(request: Request, env: AccessEnv): Promise<Identity> {
const payload = await verify(request, env, env.ACCESS_AUD_ADMIN)
const email = String(payload.email ?? '').toLowerCase()
const allowed = env.ADMIN_EMAILS.toLowerCase()
.split(',')
.map((s) => s.trim())
.filter(Boolean)
if (!email || !allowed.includes(email)) throw new AccessError('not a maintainer')
// Cross-site request forgery guard for writes (the Access cookie is sent
// with top-level cross-site requests too).
if (!SAFE_METHODS.has(request.method) && request.headers.get('origin') !== env.SITE_ORIGIN) {
throw new AccessError('bad origin')
}
return { kind: 'maintainer', id: email }
}
/** A service token of the "Ingest" Access application (Service Auth policy). */
export async function requireService(request: Request, env: AccessEnv): Promise<Identity> {
const payload = await verify(request, env, env.ACCESS_AUD_INGEST)
// Service-token JWTs carry the client id as common_name and no email.
const id = String(payload.common_name ?? '')
if (!id || payload.email) throw new AccessError('not a service token')
return { kind: 'service', id }
}
+323
View File
@@ -0,0 +1,323 @@
/**
* Maintainer-only admin: the JSON API under /api/admin/* and the admin app
* shell at /admin. Both sit behind the Cloudflare Access "Admin" application
* and re-verify its JWT here (access.ts): the edge login alone is not trusted.
*
* GET /api/admin/me who am I, what is configured
* GET /api/admin/queue?status=… review queue (+ counts per status)
* POST /api/admin/queue/decide {urls, action: approve|reject, override?}
* POST /api/admin/add {urls, override?} — add + approve at once
* PATCH /api/admin/games {url, edit: {safe_virus?, notes?, nsfw?}}
* DELETE /api/admin/games {url, reason}
* POST /api/admin/games/restore {url} — un-delete: unblock + queue again
*
* Approving commits the URLs to scripts/temp_link.json (GitHub App,
* verified commit); that push starts the ingest workflow, which scrapes and
* adds the games. The maintainer's email is recorded only in the D1 audit
* log, never in the public repository.
*/
import { AccessError, requireMaintainer, type AccessEnv } from './access'
import { canonicalize } from './canonical'
import { validateEdit, type FieldEdit } from './catalog'
import { catalogUrls, deletedGames, type DataSource } from './data'
import { ACCESS_CONFIG, GITHUB_CONFIG, missingConfig, type WorkerEnv } from './env'
import type { GitHubEnv } from './github'
import { HttpError, SECURITY_HEADERS, errorResponse, isObject, json, readJson } from './http'
import { r2Key } from './img'
import { autoFlags, classify, type CandidateRow, type CandidateStatus, type QueueStore } from './queue'
import { editCatalog, queueForIngest, type CatalogEditResult } from './repo'
const MAX_URLS = 200
const STATUSES: readonly CandidateStatus[] = ['pending', 'rejected', 'queued', 'ingested', 'failed']
const THUMB_WIDTHS = [160, 640]
// The admin app loads only its own bundle; cover images come straight from
// itch.io (candidates are not in the catalog yet, so /img won't serve them).
export const ADMIN_CSP = [
"default-src 'self'",
"script-src 'self'",
"style-src 'self' 'unsafe-inline'",
"img-src 'self' data: https://img.itch.zone",
"connect-src 'self'",
"font-src 'self'",
"object-src 'none'",
"base-uri 'self'",
"form-action 'self'",
"frame-ancestors 'none'",
].join('; ')
/** Writes to the repo; injectable so tests don't talk to GitHub. */
export interface RepoOps {
queueForIngest(urls: string[], unblock: string[]): Promise<{ sha: string | null; added: string[] }>
editCatalog(edits: Map<string, FieldEdit>, removals: { url: string; reason: string }[]): Promise<CatalogEditResult>
}
export interface AdminContext {
actor: string
store: QueueStore
data: DataSource
/** null when the GitHub App is not configured: read-only admin. */
repo: RepoOps | null
thumbs: Pick<R2Bucket, 'delete'>
now: Date
}
function githubRepo(env: WorkerEnv): RepoOps | null {
if (missingConfig(env, GITHUB_CONFIG).length > 0) return null
const gh = env as WorkerEnv & GitHubEnv
return {
queueForIngest: (urls, unblock) => queueForIngest(gh, urls, unblock),
editCatalog: (edits, removals) => editCatalog(gh, edits, removals),
}
}
async function maintainer(request: Request, env: WorkerEnv): Promise<string> {
const missing = missingConfig(env, [...ACCESS_CONFIG, 'ACCESS_AUD_ADMIN', 'ADMIN_EMAILS'])
if (missing.length > 0) {
console.error(`admin: not configured (${missing.join(', ')})`)
throw new HttpError(503, 'not_configured')
}
try {
return (await requireMaintainer(request, env as WorkerEnv & AccessEnv)).id
} catch (e) {
if (e instanceof AccessError) throw new HttpError(403, 'forbidden')
throw e
}
}
function urlList(value: unknown): string[] {
if (!Array.isArray(value) || value.length === 0) throw new HttpError(400, 'invalid_body', '`urls` must be a non-empty array')
if (value.length > MAX_URLS) throw new HttpError(413, 'too_many_urls', `At most ${MAX_URLS} URLs per request`)
if (!value.every((u) => typeof u === 'string' && u.length <= 500)) throw new HttpError(400, 'invalid_body')
return value as string[]
}
function gameUrl(value: unknown): string {
const url = typeof value === 'string' ? canonicalize(value) : null
if (!url) throw new HttpError(400, 'invalid_url')
return url
}
function needRepo(ctx: AdminContext): RepoOps {
if (!ctx.repo) throw new HttpError(503, 'github_not_configured')
return ctx.repo
}
function parseFlags(row: CandidateRow): string[] {
try {
const flags: unknown = JSON.parse(row.flags)
return Array.isArray(flags) ? flags.filter((f): f is string => typeof f === 'string') : []
} catch {
return []
}
}
interface Skipped {
url: string
reason: string
}
/**
* Queue candidate rows for ingest. A game that was removed from the catalog
* before (it is in the deleted log) needs `override`: approving it also
* unblocks it, or the ingest workflow would skip it again.
*/
async function approve(ctx: AdminContext, urls: string[], override: boolean) {
const rows = new Map((await ctx.store.get(urls)).map((r) => [r.url, r]))
const deleted = await deletedGames(ctx.data)
const approved: string[] = []
const skipped: Skipped[] = []
for (const url of urls) {
const row = rows.get(url)
if (!row) skipped.push({ url, reason: 'not_in_queue' })
else if (row.status === 'queued' || row.status === 'ingested') skipped.push({ url, reason: `already_${row.status}` })
else if (!override && (deleted.has(url) || parseFlags(row).includes('previously_deleted'))) {
skipped.push({ url, reason: 'needs_override' })
} else approved.push(url)
}
if (approved.length === 0) return { sha: null, approved, skipped }
const { sha } = await needRepo(ctx).queueForIngest(approved, override ? approved : [])
await ctx.store.setStatus(approved, 'queued', ctx.actor, ctx.now)
await ctx.store.audit(ctx.actor, override ? 'approve_override' : 'approve', approved.join(' '), sha)
return { sha, approved, skipped }
}
async function listQueue(url: URL, ctx: AdminContext): Promise<Response> {
const status = (url.searchParams.get('status') ?? 'pending') as CandidateStatus
if (!STATUSES.includes(status)) throw new HttpError(400, 'invalid_status')
const limit = Math.min(Math.max(Number(url.searchParams.get('limit')) || 200, 1), 500)
const [counts, rows] = await Promise.all([ctx.store.counts(), ctx.store.list(status, limit)])
return json({ counts, items: rows.map((r) => ({ ...r, flags: parseFlags(r) })) })
}
async function decide(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
const urls = urlList(body.urls).map(gameUrl)
if (body.action === 'reject') {
await ctx.store.setStatus(urls, 'rejected', ctx.actor, ctx.now)
await ctx.store.audit(ctx.actor, 'reject', urls.join(' '), null)
return json({ rejected: urls })
}
if (body.action !== 'approve') throw new HttpError(400, 'invalid_action')
return json(await approve(ctx, urls, body.override === true))
}
async function add(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
const inputs = urlList(body.urls)
const override = body.override === true
const canonical = inputs.map((u) => canonicalize(u)).filter((u): u is string => u !== null)
const [catalog, deleted, known] = await Promise.all([
catalogUrls(ctx.data),
deletedGames(ctx.data),
ctx.store.statuses(canonical),
])
const results = classify(inputs, { catalog: new Set(catalog.keys()), deleted, known })
await ctx.store.insert(
results
.filter((r) => r.store && r.canonical)
.map((r) => ({
url: r.canonical as string,
source: 'admin' as const,
submitter: 'admin',
flags: autoFlags(r.outcome),
})),
ctx.now,
)
// Rows that exist now and are waiting (new, pending, failed, or rejected
// and explicitly re-added) are approved right away.
const approvable = results
.filter(
(r) =>
r.canonical &&
(r.store ||
(r.outcome === 'duplicate_pending' && ['pending', 'failed'].includes(known.get(r.canonical) ?? '')) ||
(r.outcome === 'rejected' && override)),
)
.map((r) => r.canonical as string)
const decision = approvable.length > 0 ? await approve(ctx, approvable, override) : null
const approved = new Set(decision?.approved ?? [])
const skipped = new Map((decision?.skipped ?? []).map((s) => [s.url, s.reason]))
return json({
sha: decision?.sha ?? null,
results: results.map((r) => ({
input: r.input,
url: r.canonical,
outcome: r.outcome,
approved: r.canonical !== null && approved.has(r.canonical),
...(r.canonical && skipped.has(r.canonical) ? { reason: skipped.get(r.canonical) } : {}),
})),
})
}
async function patchGame(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
const url = gameUrl(body.url)
if (!isObject(body.edit) || Object.keys(body.edit).length === 0) throw new HttpError(400, 'invalid_edit')
const problem = validateEdit(body.edit)
if (problem) throw new HttpError(400, 'invalid_edit', problem)
const edit = body.edit as FieldEdit
const result = await needRepo(ctx).editCatalog(new Map([[url, edit]]), [])
if (result.sha) await ctx.store.audit(ctx.actor, 'edit', url, result.sha, JSON.stringify(edit))
return json({ sha: result.sha, edited: result.summary.edited })
}
async function deleteGame(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
const url = gameUrl(body.url)
const reason = typeof body.reason === 'string' ? body.reason.trim() : ''
if (!reason || reason.length > 200) throw new HttpError(400, 'invalid_reason', 'A reason (≤ 200 characters) is required')
const result = await needRepo(ctx).editCatalog(new Map(), [{ url, reason }])
if (result.summary.removed === 0) throw new HttpError(404, 'not_in_catalog')
// Resized covers of a removed game (content-removal requests included).
const keys = await Promise.all(
result.thumbnails.flatMap((thumb) => THUMB_WIDTHS.map((w) => r2Key(w, thumb))),
)
if (keys.length > 0) await ctx.thumbs.delete(keys)
await ctx.store.audit(ctx.actor, 'delete', url, result.sha, reason)
return json({ sha: result.sha, removed: result.summary.removed })
}
async function restoreGame(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
const url = gameUrl(body.url)
const { sha, added } = await needRepo(ctx).queueForIngest([url], [url])
await ctx.store.insert([{ url, source: 'admin', submitter: 'admin' }], ctx.now)
await ctx.store.setStatus([url], 'queued', ctx.actor, ctx.now)
await ctx.store.audit(ctx.actor, 'restore', url, sha)
return json({ sha, queued: added.length > 0 })
}
/** Route an authenticated admin API request. */
export async function adminApi(request: Request, url: URL, ctx: AdminContext): Promise<Response> {
try {
return await route(request, url, ctx)
} catch (e) {
return errorResponse(e)
}
}
async function route(request: Request, url: URL, ctx: AdminContext): Promise<Response> {
const route = `${request.method} ${url.pathname.replace(/\/+$/, '')}`
if (route === 'GET /api/admin/me') {
return json({ email: ctx.actor, writes: ctx.repo !== null })
}
if (route === 'GET /api/admin/queue') return listQueue(url, ctx)
const body = await readJson(request, 64 * 1024)
if (!isObject(body)) throw new HttpError(400, 'invalid_body')
switch (route) {
case 'POST /api/admin/queue/decide':
return decide(body, ctx)
case 'POST /api/admin/add':
return add(body, ctx)
case 'PATCH /api/admin/games':
return patchGame(body, ctx)
case 'DELETE /api/admin/games':
return deleteGame(body, ctx)
case 'POST /api/admin/games/restore':
return restoreGame(body, ctx)
default:
throw new HttpError(404, 'not_found')
}
}
export async function handleAdminApi(
request: Request,
url: URL,
env: WorkerEnv,
store: QueueStore | null,
): Promise<Response> {
try {
const actor = await maintainer(request, env)
if (!store) throw new HttpError(503, 'not_configured')
return await adminApi(request, url, {
actor,
store,
data: { assets: env.ASSETS, origin: url.origin },
repo: githubRepo(env),
thumbs: env.THUMBS,
now: new Date(),
})
} catch (e) {
return errorResponse(e)
}
}
/** /admin and /admin/* → the admin app shell (HashRouter), maintainer only. */
export async function serveAdminApp(request: Request, url: URL, env: WorkerEnv): Promise<Response> {
const text = (status: number, body: string) =>
new Response(body, {
status,
headers: { 'Content-Type': 'text/plain; charset=utf-8', 'Cache-Control': 'no-store', ...SECURITY_HEADERS },
})
if (request.method !== 'GET' && request.method !== 'HEAD') return text(405, 'Method not allowed')
if (/\.[a-z0-9]+$/i.test(url.pathname) && url.pathname !== '/admin/index.html') return text(404, 'Not found')
try {
await maintainer(request, env)
} catch (e) {
return e instanceof HttpError && e.status === 503 ? text(503, 'Admin is not configured.') : text(403, 'Forbidden')
}
const shell = await env.ASSETS.fetch(new Request(new URL('/admin/', url), { headers: request.headers }))
const headers = new Headers(shell.headers)
headers.set('Content-Security-Policy', ADMIN_CSP)
headers.set('Cache-Control', 'no-store')
headers.set('X-Robots-Tag', 'noindex, nofollow')
headers.set('X-Frame-Options', 'DENY')
for (const [k, v] of Object.entries(SECURITY_HEADERS)) headers.set(k, v)
return new Response(request.method === 'HEAD' ? null : shell.body, { status: shell.status, headers })
}
+15
View File
@@ -0,0 +1,15 @@
import { describe, expect, it } from 'vitest'
import { canonicalize } from './canonical'
// Shared with the Python pipeline (tests/test_canonical.py).
import vectors from '../../tests/fixtures/url_vectors.json'
describe('canonicalize (shared vectors)', () => {
it.each(vectors.map((v) => [v.input, v.expected] as const))('%j → %j', (input, expected) => {
expect(canonicalize(input)).toBe(expected)
})
it('rejects non-strings', () => {
expect(canonicalize(undefined)).toBeNull()
expect(canonicalize(42)).toBeNull()
})
})
+34
View File
@@ -0,0 +1,34 @@
/**
* One canonical form for itch.io game URLs — port of scripts/canonical.py.
* Both implementations are tested against tests/fixtures/url_vectors.json.
*
* Canonical form: https://{creator}.itch.io/{slug}
* - https only, host lower-cased, no port / userinfo / query / fragment
* - exactly one lower-cased path segment, no trailing slash
* - creator must be a *.itch.io subdomain (not itch.io / www.itch.io)
*/
const HOST = /^[a-z0-9][a-z0-9-]*\.itch\.io$/
const SLUG = /^[a-z0-9][a-z0-9_-]*$/
const RESERVED_HOSTS = new Set(['itch.io', 'www.itch.io'])
export function canonicalize(raw: unknown): string | null {
if (typeof raw !== 'string') return null
let text = raw.trim()
if (!text) return null
if (!text.includes('://')) text = `https://${text}`
let url: URL
try {
url = new URL(text)
} catch {
return null
}
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
if (url.username || url.password) return null
const host = url.hostname.toLowerCase()
if (RESERVED_HOSTS.has(host) || !HOST.test(host)) return null
const segments = url.pathname.split('/').filter(Boolean)
if (segments.length !== 1) return null
const slug = segments[0].toLowerCase()
if (!SLUG.test(slug)) return null
return `https://${host}/${slug}`
}
+116
View File
@@ -0,0 +1,116 @@
import { describe, expect, it } from 'vitest'
// Shared with the Python pipeline (tests/test_data_store.py).
import planCases from '../../tests/fixtures/golden/plan_chunks.json'
import serializeInput from '../../tests/fixtures/golden/serialize_input.json'
import serializeExpected from '../../tests/fixtures/golden/serialize_expected.txt?raw'
import {
applyAdminChange,
buildIndex,
dedupDeleted,
dumps,
planChunks,
upsertCountHistory,
validateEdit,
type CatalogFiles,
type Game,
} from './catalog'
const NOW = new Date('2026-09-20T10:00:00.123Z')
const SMALL = { chunkSize: 2, minFill: 1 }
function game(slug: string, extra: Record<string, unknown> = {}): Game {
return { url: `https://dev.itch.io/${slug}`, name: slug, safe_virus: '?', notes: '', nsfw: 'No', ...extra }
}
describe('golden fixtures shared with Python', () => {
it.each(planCases.map((c) => [c.name, c] as const))('planChunks: %s', (_name, c) => {
const old = c.old.map((chunk) => chunk.map((url) => ({ url })))
const games = c.games.map((url) => ({ url }))
const got = planChunks(old, games, c.chunk_size, c.min_fill)
expect(got.map((chunk) => chunk.map((g) => g.url))).toEqual(c.expected)
})
it('dumps matches json.dumps(indent=4, ensure_ascii=False) byte for byte', () => {
expect(dumps(serializeInput)).toBe(serializeExpected)
})
})
describe('index and history', () => {
it('keeps last_updated when nothing changed', () => {
const chunks = [[game('a')]]
const prev = buildIndex(chunks, null, true, new Date('2026-01-01T00:00:00Z'))
expect(prev.last_updated).toBe('2026-01-01T00:00:00Z')
expect(buildIndex(chunks, prev, false, NOW).last_updated).toBe('2026-01-01T00:00:00Z')
expect(buildIndex(chunks, prev, true, NOW).last_updated).toBe('2026-09-20T10:00:00Z')
})
it('adds a history row only when the total changes', () => {
const h = [{ date: '2026-09-01', total: 10 }]
expect(upsertCountHistory(h, 10, NOW)).toEqual(h)
expect(upsertCountHistory(h, 9, NOW)).toEqual([...h, { date: '2026-09-20', total: 9 }])
expect(upsertCountHistory([{ date: '2026-09-20', total: 5 }], 9, NOW)).toEqual([{ date: '2026-09-20', total: 9 }])
})
it('dedupDeleted keeps the earliest entry per url', () => {
const e = (url: string, at: string) => ({ url, name: url, reason: 'r', deleted_at: at })
expect(dedupDeleted([e('u1', '2026-02'), e('u1', '2026-01'), e('', '2025'), e('u2', '2025-12')])).toEqual([
e('u2', '2025-12'),
e('u1', '2026-01'),
])
})
})
describe('validateEdit', () => {
it('allows only the three editable fields with valid values', () => {
expect(validateEdit({ safe_virus: 'Yes', notes: 'ok', nsfw: 'No' })).toBeNull()
expect(validateEdit({ name: 'x' })).toMatch(/not editable/)
expect(validateEdit({ safe_virus: 'y' })).toMatch(/safe_virus/)
expect(validateEdit({ nsfw: 'maybe' })).toMatch(/nsfw/)
expect(validateEdit({ notes: 1 })).toMatch(/string/)
expect(validateEdit({ notes: 'x'.repeat(2001) })).toMatch(/2000/)
})
})
describe('applyAdminChange', () => {
function files(): CatalogFiles {
const chunks = [[game('a'), game('b')], [game('c')]]
return {
chunks,
index: buildIndex(chunks, null, true, new Date('2026-09-01T00:00:00Z')),
history: [{ date: '2026-09-01', total: 3 }],
deleted: [],
}
}
it('edits in place and only rewrites the touched chunk + index', () => {
const change = applyAdminChange(files(), new Map([['https://dev.itch.io/c', { safe_virus: 'Yes' }]]), [], NOW, SMALL)
expect([...change.writes.keys()].sort()).toEqual(['data_game/game_info_002.json', 'data_game/index.json'])
expect(JSON.parse(change.writes.get('data_game/game_info_002.json')!)[0].safe_virus).toBe('Yes')
expect(change.summary).toEqual({ edited: 1, removed: 0 })
})
it('removes games, logs them and updates index + history in one change', () => {
const change = applyAdminChange(
files(),
new Map(),
[{ url: 'https://dev.itch.io/c', reason: 'Removed at the creator’s request' }],
NOW,
SMALL,
)
expect(change.deletes).toEqual(['data_game/game_info_002.json'])
const index = JSON.parse(change.writes.get('data_game/index.json')!)
expect(index.total_games).toBe(2)
expect(index.files).toEqual([{ name: 'game_info_001.json', count: 2 }])
expect(JSON.parse(change.writes.get('data_game/count_history.json')!).at(-1)).toEqual({ date: '2026-09-20', total: 2 })
const deleted = JSON.parse(change.writes.get('scripts/deleted_games.json')!)
expect(deleted).toEqual([
{ url: 'https://dev.itch.io/c', name: 'c', reason: 'Removed at the creator’s request', deleted_at: '2026-09-20T10:00:00Z' },
])
})
it('writes nothing for a no-op edit', () => {
const change = applyAdminChange(files(), new Map([['https://dev.itch.io/a', { safe_virus: '?' }]]), [], NOW, SMALL)
expect(change.writes.size).toBe(0)
expect(change.deletes).toEqual([])
})
})
+234
View File
@@ -0,0 +1,234 @@
/**
* Catalog file logic for admin writes — a TypeScript port of
* scripts/data_store.py + scripts/json_io.py. Pure functions only: the
* GitHub layer reads the files at a commit, calls these, and commits the
* result, so the repo stays the single source of truth.
*
* Kept byte-for-byte compatible with the Python pipeline (same chunk
* placement policy, same `json.dumps(indent=4, ensure_ascii=False)` text);
* both sides are tested against tests/fixtures/golden/.
*/
const CHUNK_SIZE = 500
const MIN_FILL = 250
/** The only fields a maintainer edits; everything else is scraper-owned. */
const EDITABLE_FIELDS = ['safe_virus', 'notes', 'nsfw'] as const
type EditableField = (typeof EDITABLE_FIELDS)[number]
const SAFE_VIRUS_VALUES = ['?', 'Yes', 'No', 'Caution'] as const
const NSFW_VALUES = ['Yes', 'No'] as const
export interface Game {
url: string
name?: string
[field: string]: unknown
}
export interface CatalogIndex {
total_games: number
max_per_file: number
last_updated: string
files: { name: string; count: number }[]
}
export interface CountPoint {
date: string
total: number
}
export interface DeletedEntry {
url: string
name: string
reason: string
deleted_at: string
}
/** Same text as Python's json.dumps(data, ensure_ascii=False, indent=4). */
export function dumps(data: unknown): string {
return JSON.stringify(data, null, 4)
}
function chunkName(index: number): string {
return `game_info_${String(index).padStart(3, '0')}.json`
}
/**
* Place `games` into chunks, keeping each game in its current chunk (port of
* data_store.plan_chunks). New games go to the last chunk; the whole list is
* re-sliced only when a chunk overflows or a non-last chunk under-fills.
*/
export function planChunks(
oldChunks: Game[][],
games: Game[],
chunkSize = CHUNK_SIZE,
minFill = MIN_FILL,
): Game[][] {
const byUrl = new Map<string, Game>()
for (const g of games) if (!byUrl.has(g.url)) byUrl.set(g.url, g)
const placed = new Set<string>()
let chunks: Game[][] = oldChunks.map((old) => {
const chunk: Game[] = []
for (const g of old) {
const current = byUrl.get(g.url)
if (current && !placed.has(g.url)) {
chunk.push(current)
placed.add(g.url)
}
}
return chunk
})
const fresh = [...byUrl.values()].filter((g) => !placed.has(g.url))
if (fresh.length > 0) {
if (chunks.length === 0) chunks.push([])
chunks[chunks.length - 1].push(...fresh)
}
chunks = chunks.filter((c) => c.length > 0)
const needsReslice =
chunks.some((c) => c.length > chunkSize) || chunks.slice(0, -1).some((c) => c.length < minFill)
if (needsReslice) {
const flat = chunks.flat()
chunks = []
for (let i = 0; i < flat.length; i += chunkSize) chunks.push(flat.slice(i, i + chunkSize))
}
return chunks
}
/** index.json content; `last_updated` moves only when data or layout changed. */
export function buildIndex(
chunks: Game[][],
previous: Partial<CatalogIndex> | null,
dataChanged: boolean,
now: Date,
): CatalogIndex {
const files = chunks.map((c, i) => ({ name: chunkName(i + 1), count: c.length }))
const total = files.reduce((sum, f) => sum + f.count, 0)
const layoutChanged =
JSON.stringify(previous?.files) !== JSON.stringify(files) || previous?.total_games !== total
const lastUpdated =
dataChanged || layoutChanged || !previous?.last_updated
? now.toISOString().replace(/\.\d{3}Z$/, 'Z')
: previous.last_updated
return { total_games: total, max_per_file: CHUNK_SIZE, last_updated: lastUpdated, files }
}
/** Date-keyed upsert; a day whose total equals the last recorded one adds no row. */
export function upsertCountHistory(history: CountPoint[], total: number, now: Date): CountPoint[] {
const today = now.toISOString().slice(0, 10)
const rows = [...history].sort((a, b) => a.date.localeCompare(b.date))
const existing = rows.find((r) => r.date === today)
if (existing) {
existing.total = total
return rows
}
if (rows.length > 0 && rows[rows.length - 1].total === total) return rows
return [...rows, { date: today, total }]
}
/** Keep the earliest deleted_at per URL, sorted by deleted_at (dedup_deleted). */
export function dedupDeleted(entries: DeletedEntry[]): DeletedEntry[] {
const byUrl = new Map<string, DeletedEntry>()
for (const e of entries) {
if (!e.url) continue
const current = byUrl.get(e.url)
if (!current || e.deleted_at < current.deleted_at) byUrl.set(e.url, e)
}
return [...byUrl.values()].sort((a, b) => a.deleted_at.localeCompare(b.deleted_at))
}
export type FieldEdit = Partial<Record<EditableField, string>>
/** Validate an admin edit; returns an error message or null. */
export function validateEdit(edit: Record<string, unknown>): string | null {
for (const [key, value] of Object.entries(edit)) {
if (!(EDITABLE_FIELDS as readonly string[]).includes(key)) return `field '${key}' is not editable`
if (typeof value !== 'string') return `field '${key}' must be a string`
if (key === 'safe_virus' && !(SAFE_VIRUS_VALUES as readonly string[]).includes(value))
return `safe_virus must be one of ${SAFE_VIRUS_VALUES.join(', ')}`
if (key === 'nsfw' && !(NSFW_VALUES as readonly string[]).includes(value))
return `nsfw must be one of ${NSFW_VALUES.join(', ')}`
if (key === 'notes' && value.length > 2000) return 'notes is limited to 2000 characters'
}
return null
}
export interface CatalogFiles {
chunks: Game[][]
index: CatalogIndex | null
history: CountPoint[]
deleted: DeletedEntry[]
}
export interface CatalogChange {
/** path → new file text (only files whose text changed). */
writes: Map<string, string>
/** paths to delete (chunks that disappeared). */
deletes: string[]
summary: { edited: number; removed: number }
}
/**
* Apply edits and removals to the catalog as read at one commit. Produces
* exactly the file texts the Python pipeline would write for the same change.
*/
export function applyAdminChange(
files: CatalogFiles,
edits: Map<string, FieldEdit>,
removals: { url: string; reason: string }[],
now: Date,
sizes: { chunkSize: number; minFill: number } = { chunkSize: CHUNK_SIZE, minFill: MIN_FILL },
): CatalogChange {
const games = files.chunks.flat().map((g) => ({ ...g }))
let edited = 0
for (const g of games) {
const edit = edits.get(g.url)
if (!edit) continue
let changed = false
for (const [key, value] of Object.entries(edit)) {
if (g[key] !== value) {
g[key] = value
changed = true
}
}
if (changed) edited++
}
const removeUrls = new Map(removals.map((r) => [r.url, r.reason]))
const stamp = now.toISOString().replace(/\.\d{3}Z$/, 'Z')
const deletedEntries: DeletedEntry[] = []
const kept = games.filter((g) => {
const reason = removeUrls.get(g.url)
if (reason === undefined) return true
deletedEntries.push({ url: g.url, name: String(g.name ?? 'N/A'), reason, deleted_at: stamp })
return false
})
const writes = new Map<string, string>()
const newChunks = planChunks(files.chunks, kept, sizes.chunkSize, sizes.minFill)
const oldTexts = files.chunks.map((c) => dumps(c))
let dataChanged = false
newChunks.forEach((chunk, i) => {
const text = dumps(chunk)
if (oldTexts[i] !== text) {
writes.set(`data_game/${chunkName(i + 1)}`, text)
dataChanged = true
}
})
const deletes: string[] = []
for (let i = newChunks.length; i < files.chunks.length; i++) {
deletes.push(`data_game/${chunkName(i + 1)}`)
dataChanged = true
}
const index = buildIndex(newChunks, files.index, dataChanged, now)
if (!files.index || dumps(files.index) !== dumps(index)) writes.set('data_game/index.json', dumps(index))
const history = upsertCountHistory(files.history, index.total_games, now)
if (dumps(history) !== dumps(files.history)) writes.set('data_game/count_history.json', dumps(history))
if (deletedEntries.length > 0) {
writes.set('scripts/deleted_games.json', dumps(dedupDeleted([...files.deleted, ...deletedEntries])))
}
return { writes, deletes, summary: { edited, removed: deletedEntries.length } }
}
+21
View File
@@ -10,10 +10,17 @@ export interface DataSource {
origin: string
}
interface DeletedInfo {
reason: string
deleted_at: string
}
let urlMap: Map<string, string> | undefined
let deletedMap: Map<string, DeletedInfo> | undefined
export function resetDataCache(): void {
urlMap = undefined
deletedMap = undefined
}
async function fetchJson<T>(src: DataSource, file: string): Promise<T> {
@@ -46,3 +53,17 @@ export function catalogUrls(src: DataSource): Promise<Map<string, string>> {
)
}
/** Removed game URL → why and when (scripts/deleted_games.json). */
export function deletedGames(src: DataSource): Promise<Map<string, DeletedInfo>> {
return cachedLoad(
() => deletedMap,
(v) => (deletedMap = v),
async () =>
new Map(
(await fetchJson<(DeletedInfo & { url: string })[]>(src, 'deleted_games.json')).map((e) => [
e.url,
{ reason: e.reason, deleted_at: e.deleted_at },
]),
),
)
}
+176
View File
@@ -0,0 +1,176 @@
/**
* Scheduled discovery of new free games from itch.io RSS feeds. Each run
* polls ONE feed (the one polled longest ago that is not backing off), so
* itch.io sees at most one request per cron tick. Items go to the review
* queue as `source = 'rss'`; nothing is added to the catalog without a
* maintainer's approval.
*/
import { canonicalize } from './canonical'
import { catalogUrls, deletedGames, type DataSource } from './data'
import { autoFlags, classify, isoSeconds, type FeedState, type QueueStore } from './queue'
interface Feed {
id: string
url: string
genre: string | null
}
export const FEEDS: readonly Feed[] = [
{ id: 'free', url: 'https://itch.io/games/new-and-popular/free.xml', genre: null },
{
id: 'visual-novel',
url: 'https://itch.io/games/new-and-popular/free/genre-visual-novel.xml',
genre: 'Visual Novel',
},
]
const BACKOFF_MS = 6 * 3_600_000
const MAX_ITEMS = 60
export interface FeedItem {
url: string
title: string
image: string | null
genre: string | null
free: boolean
}
// Bracket tags itch.io appends to <title> that are not genres.
const NOT_GENRE = new Set(['free', 'windows', 'macos', 'linux', 'android', 'ios', 'html5', 'web', 'browser'])
function decode(text: string): string {
return text
.replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, '$1')
.replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n)))
.replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(parseInt(n, 16)))
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&amp;/g, '&')
.trim()
}
function tag(block: string, name: string): string | null {
const m = new RegExp(`<${name}>([\\s\\S]*?)</${name}>`).exec(block)
return m ? decode(m[1]) : null
}
/** Items of an itch.io browse feed (`<item>` with plainTitle / price / imageurl). */
export function parseFeed(xml: string): FeedItem[] {
const items: FeedItem[] = []
for (const [, block] of xml.matchAll(/<item>([\s\S]*?)<\/item>/g)) {
const url = tag(block, 'link') ?? tag(block, 'guid')
if (!url) continue
const rawTitle = tag(block, 'title') ?? ''
const brackets = [...rawTitle.matchAll(/\[([^\]]+)\]/g)].map((m) => m[1].trim())
const genre = brackets.find((b) => !NOT_GENRE.has(b.toLowerCase())) ?? null
const price = tag(block, 'price') ?? ''
const image = tag(block, 'imageurl')
items.push({
url,
title: tag(block, 'plainTitle') ?? rawTitle.replace(/\s*\[[^\]]*\]/g, '').trim(),
image: image?.startsWith('https://img.itch.zone/') ? image : null,
genre,
free: /^[^0-9]*0+(?:[.,]0+)?$/.test(price) || brackets.some((b) => b.toLowerCase() === 'free'),
})
}
return items
}
/** The feed to poll now: not backing off, least recently polled first. */
export function pickFeed(feeds: readonly Feed[], states: Map<string, FeedState>, now: Date): Feed | null {
const stamp = isoSeconds(now)
const ready = feeds.filter((f) => (states.get(f.id)?.backoff_until ?? '') <= stamp)
ready.sort((a, b) => (states.get(a.id)?.last_polled_at ?? '').localeCompare(states.get(b.id)?.last_polled_at ?? ''))
return ready[0] ?? null
}
function retryAfterMs(header: string | null, now: Date): number {
if (!header) return 0
const seconds = Number(header)
if (Number.isFinite(seconds)) return seconds * 1000
const at = Date.parse(header)
return Number.isNaN(at) ? 0 : at - now.getTime()
}
export interface DiscoverDeps {
store: QueueStore
data: DataSource
fetch: typeof fetch
userAgent: string
now: Date
}
export interface DiscoverResult {
feed: string | null
status: number | null
items: number
added: number
}
export async function discover(deps: DiscoverDeps): Promise<DiscoverResult> {
const { store, now } = deps
const feed = pickFeed(FEEDS, await store.feeds(), now)
if (!feed) return { feed: null, status: null, items: 0, added: 0 }
const save = (status: number, backoffMs = 0) =>
store.saveFeed({
feed: feed.id,
last_polled_at: isoSeconds(now),
backoff_until: backoffMs > 0 ? isoSeconds(new Date(now.getTime() + backoffMs)) : null,
last_status: status,
})
let res: Response
try {
res = await deps.fetch(feed.url, {
headers: { 'User-Agent': deps.userAgent, Accept: 'application/rss+xml, application/xml;q=0.9' },
})
} catch {
await save(0)
return { feed: feed.id, status: 0, items: 0, added: 0 }
}
if (res.status === 429 || res.status === 503) {
res.body?.cancel().catch(() => {})
await save(res.status, Math.max(BACKOFF_MS, retryAfterMs(res.headers.get('retry-after'), now)))
return { feed: feed.id, status: res.status, items: 0, added: 0 }
}
if (!res.ok) {
res.body?.cancel().catch(() => {})
await save(res.status)
return { feed: feed.id, status: res.status, items: 0, added: 0 }
}
const items = parseFeed(await res.text())
.filter((i) => i.free)
.slice(0, MAX_ITEMS)
const canonical = items.map((i) => canonicalize(i.url)).filter((u): u is string => u !== null)
const [catalog, deleted, known] = await Promise.all([
catalogUrls(deps.data),
deletedGames(deps.data),
store.statuses(canonical),
])
const results = classify(
items.map((i) => i.url),
{ catalog: new Set(catalog.keys()), deleted, known },
)
const fresh = results.flatMap((r, i) =>
r.store && r.canonical
? [
{
url: r.canonical,
source: 'rss' as const,
title: items[i].title.slice(0, 200),
image_url: items[i].image,
genre_hint: items[i].genre ?? feed.genre,
flags: autoFlags(r.outcome, items[i].title),
submitter: `rss:${feed.id}`,
},
]
: [],
)
await store.insert(fresh, now)
await save(200)
return { feed: feed.id, status: 200, items: items.length, added: fresh.length }
}
+22
View File
@@ -0,0 +1,22 @@
/**
* Secrets live only in Cloudflare (`wrangler secret put`, or .dev.vars for
* `wrangler dev`) and are typed here instead of in the generated
* worker-configuration.d.ts (`npm run cf:types` ignores .dev.vars, so the
* committed file never depends on who generated it). They are optional at
* runtime: a route whose configuration is incomplete answers 503.
*/
interface Secrets {
GH_APP_PRIVATE_KEY?: string // PKCS#8 PEM of the GitHub App
ADMIN_EMAILS?: string // comma-separated maintainer emails
TURNSTILE_SECRET?: string
}
export type WorkerEnv = Env & Secrets
/** Names of the settings in `keys` that are empty or unset. */
export function missingConfig<K extends keyof WorkerEnv>(env: WorkerEnv, keys: readonly K[]): K[] {
return keys.filter((k) => !env[k])
}
export const ACCESS_CONFIG = ['ACCESS_TEAM_DOMAIN', 'SITE_ORIGIN'] as const
export const GITHUB_CONFIG = ['GITHUB_REPO', 'GH_APP_ID', 'GH_APP_INSTALLATION_ID', 'GH_APP_PRIVATE_KEY'] as const
+129
View File
@@ -0,0 +1,129 @@
import { exportPKCS8, generateKeyPair, jwtVerify } from 'jose'
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import {
ConflictError,
commit,
commitWithRetry,
readFile,
resetTokenCache,
utf8ToBase64,
type GitHubEnv,
} from './github'
let env: GitHubEnv
let publicKey: CryptoKey
beforeAll(async () => {
const pair = await generateKeyPair('RS256', { extractable: true })
publicKey = pair.publicKey
env = {
GITHUB_REPO: 'owner/repo',
GH_APP_ID: '12345',
GH_APP_INSTALLATION_ID: '678',
GH_APP_PRIVATE_KEY: await exportPKCS8(pair.privateKey),
}
})
type Handler = (url: string, init: RequestInit) => Response | Promise<Response>
function mockFetch(handler: Handler) {
const calls: { url: string; init: RequestInit }[] = []
vi.stubGlobal('fetch', async (input: string, init: RequestInit = {}) => {
calls.push({ url: String(input), init })
return handler(String(input), init)
})
return calls
}
const tokenResponse = () =>
Response.json({ token: 'inst-token', expires_at: new Date(Date.now() + 3600_000).toISOString() })
beforeEach(() => resetTokenCache())
afterEach(() => vi.unstubAllGlobals())
describe('utf8ToBase64', () => {
it('encodes non-ASCII text as UTF-8', () => {
expect(atob(utf8ToBase64('Trò chơi'))).toBe(String.fromCharCode(...new TextEncoder().encode('Trò chơi')))
expect(utf8ToBase64('x'.repeat(100_000))).toHaveLength(Math.ceil(100_000 / 3) * 4)
})
})
describe('GitHub App auth + reads', () => {
it('signs an app JWT, caches the installation token and reads raw files', async () => {
const calls = mockFetch(async (url) => {
if (url.endsWith('/access_tokens')) return tokenResponse()
if (url.includes('/contents/data_game/index.json')) return new Response('{"total_games": 1}')
return new Response(null, { status: 404 })
})
expect(await readFile(env, 'data_game/index.json', 'abc')).toBe('{"total_games": 1}')
expect(await readFile(env, 'missing.json', 'abc')).toBeNull()
const tokenCalls = calls.filter((c) => c.url.endsWith('/access_tokens'))
expect(tokenCalls).toHaveLength(1) // cached for the second read
const jwt = String((tokenCalls[0].init.headers as Record<string, string>).Authorization).slice(7)
const { payload } = await jwtVerify(jwt, publicKey)
expect(payload.iss).toBe('12345')
const read = calls.find((c) => c.url.includes('index.json'))!
const headers = read.init.headers as Record<string, string>
expect(headers.Accept).toBe('application/vnd.github.raw+json')
expect(headers.Authorization).toBe('Bearer inst-token')
expect(headers['User-Agent']).toBeTruthy()
expect(read.url).toContain('ref=abc')
})
})
describe('commit', () => {
it('sends createCommitOnBranch with the expected head and base64 contents', async () => {
const calls = mockFetch(async (url) => {
if (url.endsWith('/access_tokens')) return tokenResponse()
return Response.json({ data: { createCommitOnBranch: { commit: { oid: 'new-oid' } } } })
})
const oid = await commit(env, {
expectedHeadOid: 'head1',
headline: 'admin: test',
writes: new Map([['scripts/temp_link.json', '[]']]),
deletes: ['data_game/game_info_009.json'],
})
expect(oid).toBe('new-oid')
const body = JSON.parse(String(calls.at(-1)!.init.body))
const input = body.variables.input
expect(input.expectedHeadOid).toBe('head1')
expect(input.branch).toEqual({ repositoryNameWithOwner: 'owner/repo', branchName: 'main' })
expect(input.fileChanges.additions).toEqual([{ path: 'scripts/temp_link.json', contents: btoa('[]') }])
expect(input.fileChanges.deletions).toEqual([{ path: 'data_game/game_info_009.json' }])
})
it('retries on a moved head, then succeeds', async () => {
let heads = 0
let commits = 0
mockFetch(async (url) => {
if (url.endsWith('/access_tokens')) return tokenResponse()
if (url.includes('/git/ref/heads/main')) return Response.json({ object: { sha: `head${++heads}` } })
commits++
if (commits === 1) {
return Response.json({ errors: [{ message: 'Expected branch to point to "head1" but it did not.' }] })
}
return Response.json({ data: { createCommitOnBranch: { commit: { oid: 'ok' } } } })
})
const seen: string[] = []
const oid = await commitWithRetry(env, async (head) => {
seen.push(head)
return { headline: 'x', writes: new Map([['a.json', '{}']]) }
})
expect(oid).toBe('ok')
expect(seen).toEqual(['head1', 'head2'])
})
it('gives up after the attempt limit and skips no-op changes', async () => {
mockFetch(async (url) => {
if (url.endsWith('/access_tokens')) return tokenResponse()
if (url.includes('/git/ref/heads/main')) return Response.json({ object: { sha: 'h' } })
return Response.json({ errors: [{ message: 'expectedHeadOid mismatch' }] })
})
await expect(
commitWithRetry(env, async () => ({ headline: 'x', writes: new Map([['a', 'b']]) }), 2),
).rejects.toBeInstanceOf(ConflictError)
expect(await commitWithRetry(env, async () => null)).toBeNull()
})
})
+164
View File
@@ -0,0 +1,164 @@
/**
* GitHub access for admin writes, as a GitHub App (no personal token).
*
* - Installation token: RS256 app JWT → POST /app/installations/:id/access_tokens,
* cached per isolate until 5 minutes before expiry.
* - Reads: the Contents API with the raw media type at an exact commit
* (no base64 / UTF-8 mojibake — CLAUDE.md gotcha #8).
* - Writes: GraphQL `createCommitOnBranch` with `expectedHeadOid`. Commits made
* by an app through the API are signed by GitHub ("Verified"), and the
* expected head makes a write fail instead of silently overwriting a commit
* another writer (pipeline, extension) pushed in between; callers retry.
*/
import { SignJWT, importPKCS8 } from 'jose'
export interface GitHubEnv {
GITHUB_REPO: string // "owner/name"
GH_APP_ID: string
GH_APP_INSTALLATION_ID: string
GH_APP_PRIVATE_KEY: string // PKCS#8 PEM
}
const API = 'https://api.github.com'
const BRANCH = 'main'
const BASE_HEADERS = {
Accept: 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28',
// GitHub rejects API requests without a User-Agent.
'User-Agent': 'freeitchgames-admin',
}
export class ConflictError extends Error {}
let cached: { token: string; expiresAt: number } | undefined
export function resetTokenCache(): void {
cached = undefined
}
async function appJwt(env: GitHubEnv): Promise<string> {
const key = await importPKCS8(env.GH_APP_PRIVATE_KEY, 'RS256')
const now = Math.floor(Date.now() / 1000)
return new SignJWT({})
.setProtectedHeader({ alg: 'RS256' })
.setIssuer(env.GH_APP_ID)
.setIssuedAt(now - 60) // clock skew allowance
.setExpirationTime(now + 9 * 60)
.sign(key)
}
async function installationToken(env: GitHubEnv): Promise<string> {
if (cached && cached.expiresAt - Date.now() > 5 * 60_000) return cached.token
const res = await fetch(`${API}/app/installations/${env.GH_APP_INSTALLATION_ID}/access_tokens`, {
method: 'POST',
headers: { ...BASE_HEADERS, Authorization: `Bearer ${await appJwt(env)}` },
})
if (!res.ok) throw new Error(`GitHub App token: HTTP ${res.status}`)
const body = (await res.json()) as { token: string; expires_at: string }
cached = { token: body.token, expiresAt: Date.parse(body.expires_at) }
return body.token
}
async function gh(env: GitHubEnv, path: string, init: RequestInit = {}): Promise<Response> {
const token = await installationToken(env)
return fetch(`${API}${path}`, {
...init,
headers: { ...BASE_HEADERS, Authorization: `Bearer ${token}`, ...(init.headers ?? {}) },
})
}
async function headOid(env: GitHubEnv): Promise<string> {
const res = await gh(env, `/repos/${env.GITHUB_REPO}/git/ref/heads/${BRANCH}`)
if (!res.ok) throw new Error(`read ref: HTTP ${res.status}`)
return ((await res.json()) as { object: { sha: string } }).object.sha
}
/** File text at a commit, or null when it does not exist. */
export async function readFile(env: GitHubEnv, path: string, ref: string): Promise<string | null> {
const res = await gh(env, `/repos/${env.GITHUB_REPO}/contents/${path}?ref=${ref}`, {
headers: { Accept: 'application/vnd.github.raw+json' },
})
if (res.status === 404) return null
if (!res.ok) throw new Error(`read ${path}: HTTP ${res.status}`)
return res.text()
}
/** Names of the files directly in a directory at a commit. */
export async function listDir(env: GitHubEnv, path: string, ref: string): Promise<string[]> {
const res = await gh(env, `/repos/${env.GITHUB_REPO}/contents/${path}?ref=${ref}`)
if (!res.ok) throw new Error(`list ${path}: HTTP ${res.status}`)
const items = (await res.json()) as { name: string; type: string }[]
return items.filter((i) => i.type === 'file').map((i) => i.name)
}
/** Base64 of the UTF-8 bytes of `text` (chunked: large files overflow apply()). */
export function utf8ToBase64(text: string): string {
const bytes = new TextEncoder().encode(text)
let binary = ''
for (let i = 0; i < bytes.length; i += 0x8000) {
binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000))
}
return btoa(binary)
}
export interface CommitInput {
expectedHeadOid: string
headline: string
body?: string
writes: Map<string, string>
deletes?: string[]
}
/** One atomic, GitHub-signed commit on main. Throws ConflictError if main moved. */
export async function commit(env: GitHubEnv, input: CommitInput): Promise<string> {
const query = `mutation($input: CreateCommitOnBranchInput!) {
createCommitOnBranch(input: $input) { commit { oid } }
}`
const variables = {
input: {
branch: { repositoryNameWithOwner: env.GITHUB_REPO, branchName: BRANCH },
expectedHeadOid: input.expectedHeadOid,
message: { headline: input.headline, ...(input.body ? { body: input.body } : {}) },
fileChanges: {
additions: [...input.writes].map(([path, text]) => ({ path, contents: utf8ToBase64(text) })),
deletions: (input.deletes ?? []).map((path) => ({ path })),
},
},
}
const res = await gh(env, '/graphql', { method: 'POST', body: JSON.stringify({ query, variables }) })
const body = (await res.json()) as {
data?: { createCommitOnBranch?: { commit: { oid: string } } }
errors?: { message: string }[]
}
const errors = body.errors ?? []
if (errors.some((e) => /expected|head|oid/i.test(e.message))) {
throw new ConflictError(errors.map((e) => e.message).join('; '))
}
const oid = body.data?.createCommitOnBranch?.commit.oid
if (!res.ok || errors.length > 0 || !oid) {
throw new Error(`commit: HTTP ${res.status} ${errors.map((e) => e.message).join('; ')}`)
}
return oid
}
/**
* Read → change → commit, retried when another writer moved main meanwhile.
* `build` receives the head it must read from and returns the commit to make
* (or null when there is nothing to change).
*/
export async function commitWithRetry(
env: GitHubEnv,
build: (head: string) => Promise<Omit<CommitInput, 'expectedHeadOid'> | null>,
attempts = 3,
): Promise<string | null> {
for (let i = 1; ; i++) {
const head = await headOid(env)
const change = await build(head)
if (!change) return null
try {
return await commit(env, { ...change, expectedHeadOid: head })
} catch (e) {
if (!(e instanceof ConflictError) || i >= attempts) throw e
}
}
}
+45
View File
@@ -17,3 +17,48 @@ export function json(body: unknown, status = 200, headers: Record<string, string
},
})
}
/** A request the handler rejects; `status` + `error` become the JSON response. */
export class HttpError extends Error {
readonly status: number
readonly error: string
constructor(status: number, error: string, message?: string) {
super(message ?? error)
this.status = status
this.error = error
}
}
export function errorResponse(e: unknown): Response {
if (e instanceof HttpError) {
return json({ error: e.error, ...(e.message !== e.error ? { message: e.message } : {}) }, e.status)
}
console.error(e)
return json({ error: 'internal_error' }, 500)
}
/** Parse a JSON request body of at most `maxBytes`. */
export async function readJson(request: Request, maxBytes: number): Promise<unknown> {
if (!(request.headers.get('content-type') ?? '').includes('application/json')) {
throw new HttpError(415, 'unsupported_media_type', 'Send application/json')
}
const declared = Number(request.headers.get('content-length') ?? '0')
if (declared > maxBytes) throw new HttpError(413, 'payload_too_large')
const bytes = new Uint8Array(await request.arrayBuffer())
if (bytes.byteLength > maxBytes) throw new HttpError(413, 'payload_too_large')
try {
return JSON.parse(new TextDecoder().decode(bytes))
} catch {
throw new HttpError(400, 'invalid_json')
}
}
export async function sha256Hex(text: string): Promise<string> {
const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text))
return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join('')
}
export function isObject(v: unknown): v is Record<string, unknown> {
return typeof v === 'object' && v !== null && !Array.isArray(v)
}
+67 -7
View File
@@ -3,16 +3,37 @@
* served by the assets layer without invoking this script. It runs for the
* paths in wrangler.jsonc `assets.run_worker_first`, and for every request
* that matches no asset (`not_found_handling: "none"`, see spa.ts).
*
* /img/* resized covers (img.ts)
* /api/suggest public suggestion form (suggest.ts)
* /api/ingest browser extension, Access service token (ingest.ts)
* /api/admin/* maintainer API, Access + JWT check (admin.ts)
* /admin, /admin/* admin app shell, maintainer only (admin.ts)
*
* Cron: one RSS feed per run into the review queue, then queue bookkeeping.
*/
import { handleImg } from './img'
import { handleAdminApi, serveAdminApp } from './admin'
import { catalogUrls } from './data'
import { discover } from './discover'
import type { WorkerEnv } from './env'
import { json } from './http'
import { handleImg } from './img'
import { accessService, handleIngest } from './ingest'
import { QueueStore } from './queue'
import { handleMiss } from './spa'
import { handleSuggest } from './suggest'
function queueStore(env: WorkerEnv): QueueStore | null {
return env.DB ? new QueueStore(env.DB) : null
}
export default {
async fetch(request, env, ctx): Promise<Response> {
async fetch(request, env: WorkerEnv, ctx): Promise<Response> {
const url = new URL(request.url)
const path = url.pathname
const data = { assets: env.ASSETS, origin: url.origin }
if (url.pathname.startsWith('/img/')) {
if (path.startsWith('/img/')) {
return handleImg(request, url, {
assets: env.ASSETS,
bucket: env.THUMBS,
@@ -21,10 +42,49 @@ export default {
waitUntil: (p) => ctx.waitUntil(p),
})
}
if (url.pathname === '/api/health') return json({ ok: true })
if (url.pathname.startsWith('/api/')) return json({ error: 'not_found' }, 404)
if (path === '/api/health') return json({ ok: true })
if (path === '/api/suggest') {
return handleSuggest(request, env, {
store: queueStore(env),
data,
fetch: (input, init) => fetch(input, init),
limiter: env.RL_SUGGEST,
})
}
if (path === '/api/ingest') {
return handleIngest(request, {
store: queueStore(env),
data,
limiter: env.RL_INGEST,
authenticate: accessService(env),
})
}
if (path.startsWith('/api/admin/')) return handleAdminApi(request, url, env, queueStore(env))
if (path.startsWith('/api/')) return json({ error: 'not_found' }, 404)
if (path === '/admin' || path.startsWith('/admin/')) return serveAdminApp(request, url, env)
return handleMiss(request, url, env.ASSETS)
},
} satisfies ExportedHandler<Env>
async scheduled(_controller, env: WorkerEnv): Promise<void> {
const store = queueStore(env)
if (!store) return
const data = { assets: env.ASSETS, origin: env.SITE_ORIGIN }
const now = new Date()
try {
const found = await discover({
store,
data,
fetch: (input, init) => fetch(input, init),
userAgent: env.BOT_UA,
now,
})
console.log('discover', found)
} catch (e) {
console.error('discover failed', e)
}
const catalog = new Set((await catalogUrls(data)).keys())
console.log('reconcile', await store.reconcile(catalog, now))
await store.purge(now)
},
} satisfies ExportedHandler<WorkerEnv>
+135
View File
@@ -0,0 +1,135 @@
/**
* POST /api/ingest — machine submissions (the browser extension) into the
* review queue. Replaces the extension's direct commits with a GitHub PAT.
*
* Auth: a Cloudflare Access service token of the "Ingest" application
* (the client sends CF-Access-Client-Id/Secret; Access adds the JWT that
* requireService verifies). The submitter recorded is the token's client
* id, never something from the body.
*
* Body (≤ 64 KB, ≤ 150 items), with a required Idempotency-Key header:
* {"items": [{"url": "...", "title"?: "...", "note"?: "..."}]} or {"urls": ["..."]}
* A retry with the same key and body gets the stored answer; the same key
* with a different body is refused. Kept for 7 days.
*/
import { AccessError, requireService, type AccessEnv } from './access'
import { canonicalize } from './canonical'
import { catalogUrls, deletedGames, type DataSource } from './data'
import { ACCESS_CONFIG, missingConfig, type WorkerEnv } from './env'
import { HttpError, errorResponse, isObject, json, sha256Hex } from './http'
import { autoFlags, classify, type QueueStore } from './queue'
const MAX_BYTES = 64 * 1024
const MAX_ITEMS = 150
const KEY = /^[A-Za-z0-9_-]{8,128}$/
interface Item {
url: string
title: string | null
note: string | null
}
function text(value: unknown, max: number): string | null {
return typeof value === 'string' && value.trim() ? value.trim().slice(0, max) : null
}
function parseItems(body: unknown): Item[] {
if (!isObject(body)) throw new HttpError(400, 'invalid_body')
const raw = Array.isArray(body.items) ? body.items : Array.isArray(body.urls) ? body.urls : null
if (!raw || raw.length === 0) throw new HttpError(400, 'invalid_body', 'Send `items` or `urls`')
if (raw.length > MAX_ITEMS) throw new HttpError(413, 'too_many_items', `At most ${MAX_ITEMS} items`)
return raw.map((entry: unknown) => {
if (typeof entry === 'string') return { url: entry, title: null, note: null }
if (isObject(entry) && typeof entry.url === 'string') {
return { url: entry.url, title: text(entry.title, 200), note: text(entry.note, 500) }
}
throw new HttpError(400, 'invalid_item')
})
}
export interface IngestDeps {
store: QueueStore | null
data: DataSource
limiter: Pick<RateLimit, 'limit'> | undefined
/** Verifies the Access JWT; returns the service-token client id. */
authenticate: (request: Request) => Promise<string>
}
export function accessService(env: WorkerEnv) {
return async (request: Request): Promise<string> => {
if (missingConfig(env, [...ACCESS_CONFIG, 'ACCESS_AUD_INGEST']).length > 0) {
throw new HttpError(503, 'not_configured')
}
try {
return (await requireService(request, env as WorkerEnv & AccessEnv)).id
} catch (e) {
if (e instanceof AccessError) throw new HttpError(403, 'forbidden')
throw e
}
}
}
export async function handleIngest(request: Request, deps: IngestDeps): Promise<Response> {
try {
if (request.method !== 'POST') throw new HttpError(405, 'method_not_allowed')
const client = await deps.authenticate(request)
if (!deps.store) throw new HttpError(503, 'not_configured')
if (deps.limiter && !(await deps.limiter.limit({ key: client })).success) {
throw new HttpError(429, 'rate_limited')
}
const key = request.headers.get('idempotency-key') ?? ''
if (!KEY.test(key)) throw new HttpError(400, 'idempotency_key_required')
if (Number(request.headers.get('content-length') ?? '0') > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
const raw = await request.text()
if (new TextEncoder().encode(raw).byteLength > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
const bodyHash = await sha256Hex(raw)
const previous = await deps.store.idempotent(client, key)
if (previous) {
if (previous.body_hash !== bodyHash) throw new HttpError(422, 'idempotency_key_reused')
return json(JSON.parse(previous.response), 200, { 'Idempotent-Replayed': 'true' })
}
let body: unknown
try {
body = JSON.parse(raw)
} catch {
throw new HttpError(400, 'invalid_json')
}
const items = parseItems(body)
const canonical = items.map((i) => canonicalize(i.url)).filter((u): u is string => u !== null)
const [catalog, deleted, known] = await Promise.all([
catalogUrls(deps.data),
deletedGames(deps.data),
deps.store.statuses(canonical),
])
const results = classify(
items.map((i) => i.url),
{ catalog: new Set(catalog.keys()), deleted, known },
)
const fresh = results.flatMap((r, i) =>
r.store && r.canonical
? [
{
url: r.canonical,
source: 'ext' as const,
title: items[i].title,
note: items[i].note,
submitter: client,
flags: autoFlags(r.outcome, items[i].title),
},
]
: [],
)
await deps.store.insert(fresh)
const response = JSON.stringify({
received: fresh.length,
results: results.map((r) => ({ url: r.input, canonical: r.canonical, outcome: r.outcome })),
})
await deps.store.remember(client, key, bodyHash, response)
return json(JSON.parse(response))
} catch (e) {
return errorResponse(e)
}
}
+53
View File
@@ -0,0 +1,53 @@
-- Review queue + bookkeeping for the admin Worker. The catalog itself stays
-- JSON in the GitHub repo; D1 only holds what is waiting for a decision.
-- Migrations are additive only (never edit or drop in a later file).
-- Games proposed for the catalog (RSS discovery, the public Suggest form, the
-- browser extension via /api/ingest, or the maintainer). `rejected` rows are
-- kept as a permanent blocklist (URL only is needed for that).
CREATE TABLE candidates (
url TEXT PRIMARY KEY,
source TEXT NOT NULL CHECK (source IN ('rss', 'form', 'ext', 'admin')),
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'rejected', 'queued', 'ingested', 'failed')),
title TEXT,
image_url TEXT,
genre_hint TEXT,
flags TEXT NOT NULL DEFAULT '[]', -- JSON array: previously_deleted, demo, nsfw_keyword
note TEXT, -- Suggest form / extension note (<= 500 chars)
submitter TEXT, -- service-token client id or 'form' / 'rss:<feed>'
discovered_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
decided_at TEXT,
decided_by TEXT
);
CREATE INDEX idx_candidates_status ON candidates (status, discovered_at);
-- Round-robin state for itch.io RSS discovery (one feed per cron run).
CREATE TABLE feed_state (
feed TEXT PRIMARY KEY,
last_polled_at TEXT,
backoff_until TEXT,
last_status INTEGER
);
-- Who changed what (admin actions, with the resulting commit).
CREATE TABLE audit_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
actor TEXT NOT NULL,
action TEXT NOT NULL,
target TEXT,
commit_sha TEXT,
detail TEXT
);
-- Replay protection for /api/ingest (Idempotency-Key header), kept 7 days.
CREATE TABLE idempotency (
client TEXT NOT NULL, -- service-token client id
key TEXT NOT NULL, -- Idempotency-Key header
body_hash TEXT NOT NULL,
response TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
PRIMARY KEY (client, key)
);
CREATE INDEX idx_idempotency_created ON idempotency (created_at);
+57
View File
@@ -0,0 +1,57 @@
import { describe, expect, it } from 'vitest'
import { classify, type ClassifyContext } from './queue'
const ctx = (over: Partial<ClassifyContext> = {}): ClassifyContext => ({
catalog: new Set(['https://dev.itch.io/in-catalog']),
deleted: new Map([['https://dev.itch.io/gone', { reason: 'paid', deleted_at: '2026-06-19T03:00:00Z' }]]),
known: new Map([
['https://dev.itch.io/waiting', 'pending'],
['https://dev.itch.io/approved', 'queued'],
['https://dev.itch.io/blocked', 'rejected'],
['https://dev.itch.io/broke', 'failed'],
]),
...over,
})
describe('classify', () => {
it('assigns each outcome in precedence order', () => {
const out = classify(
[
'not a url',
'https://dev.itch.io/new',
'HTTPS://DEV.itch.io/new/', // same game after canonicalization
'https://dev.itch.io/in-catalog',
'https://dev.itch.io/blocked',
'https://dev.itch.io/waiting',
'https://dev.itch.io/approved',
'https://dev.itch.io/gone',
'https://dev.itch.io/broke',
],
ctx(),
)
expect(out.map((c) => [c.outcome, c.store])).toEqual([
['invalid_url', false],
['queued', true],
['duplicate_in_request', false],
['duplicate_catalog', false],
['rejected', false],
['duplicate_pending', false],
['duplicate_pending', false],
['previously_deleted', true],
['queued', true], // failed rows are reopened
])
expect(out[1].canonical).toBe('https://dev.itch.io/new')
expect(out[7].reason).toBe('paid (2026-06-19)')
})
it('checks the catalog before the blocklist and the deleted log', () => {
const url = 'https://dev.itch.io/in-catalog'
const out = classify([url], ctx({ known: new Map([[url, 'rejected']]), deleted: new Map() }))
expect(out[0].outcome).toBe('duplicate_catalog')
})
it('keeps the caller input verbatim', () => {
const [c] = classify([' https://dev.itch.io/new?ref=x '], ctx())
expect(c.input).toBe(' https://dev.itch.io/new?ref=x ')
})
})
+275
View File
@@ -0,0 +1,275 @@
/**
* Review queue: classification of proposed URLs (pure) and the D1 store.
*
* Order of checks for each proposed URL:
* invalid_url → duplicate_in_request → duplicate_catalog → rejected (blocklist)
* → duplicate_pending (already waiting / queued) → previously_deleted
* (kept for review, flagged; approving needs an explicit override) → queued.
*/
import { canonicalize } from './canonical'
type CandidateSource = 'rss' | 'form' | 'ext' | 'admin'
export type CandidateStatus = 'pending' | 'rejected' | 'queued' | 'ingested' | 'failed'
export type Outcome =
| 'queued'
| 'invalid_url'
| 'duplicate_in_request'
| 'duplicate_catalog'
| 'duplicate_pending'
| 'previously_deleted'
| 'rejected'
export interface ClassifyContext {
catalog: Set<string>
deleted: Map<string, { reason: string; deleted_at: string }>
/** Status of URLs already in D1 (only the ones asked about). */
known: Map<string, CandidateStatus>
}
export interface Classified {
input: string
canonical: string | null
outcome: Outcome
/** Whether a new candidate row should be stored. */
store: boolean
reason?: string
}
export function classify(inputs: string[], ctx: ClassifyContext): Classified[] {
const seen = new Set<string>()
return inputs.map((input) => {
const canonical = canonicalize(input)
if (!canonical) return { input, canonical, outcome: 'invalid_url', store: false }
if (seen.has(canonical)) return { input, canonical, outcome: 'duplicate_in_request', store: false }
seen.add(canonical)
if (ctx.catalog.has(canonical)) return { input, canonical, outcome: 'duplicate_catalog', store: false }
const status = ctx.known.get(canonical)
if (status === 'rejected') return { input, canonical, outcome: 'rejected', store: false }
if (status && status !== 'failed') return { input, canonical, outcome: 'duplicate_pending', store: false }
const deleted = ctx.deleted.get(canonical)
if (deleted) {
return {
input,
canonical,
outcome: 'previously_deleted',
store: true,
reason: `${deleted.reason} (${deleted.deleted_at.slice(0, 10)})`,
}
}
return { input, canonical, outcome: 'queued', store: true }
})
}
const NSFW_WORDS = /(?:^|[^a-z0-9])(nsfw|18\+|hentai|porn|sex|adult|lewd|erotic)(?![a-z0-9])/i
/** Review hints stored with a new candidate (never decisions by themselves). */
export function autoFlags(outcome: Outcome, title?: string | null): string[] {
const flags: string[] = []
if (outcome === 'previously_deleted') flags.push('previously_deleted')
if (title && NSFW_WORDS.test(title)) flags.push('nsfw_keyword')
if (title && /(?:^|[^a-z0-9])demo(?![a-z0-9])/i.test(title)) flags.push('demo')
return flags
}
export interface CandidateRow {
url: string
source: CandidateSource
status: CandidateStatus
title: string | null
image_url: string | null
genre_hint: string | null
flags: string
note: string | null
submitter: string | null
discovered_at: string
decided_at: string | null
decided_by: string | null
}
export interface NewCandidate {
url: string
source: CandidateSource
title?: string | null
image_url?: string | null
genre_hint?: string | null
flags?: string[]
note?: string | null
submitter?: string | null
}
export interface FeedState {
feed: string
last_polled_at: string | null
backoff_until: string | null
last_status: number | null
}
/** D1 binds at most 100 parameters per statement. */
const PARAM_SLICE = 90
const QUEUED_TIMEOUT_MS = 3 * 86_400_000
const IDEMPOTENCY_TTL_MS = 7 * 86_400_000
const FORM_NOTE_TTL_MS = 180 * 86_400_000
export function isoSeconds(date: Date): string {
return date.toISOString().replace(/\.\d{3}Z$/, 'Z')
}
function slices<T>(items: T[]): T[][] {
const out: T[][] = []
for (let i = 0; i < items.length; i += PARAM_SLICE) out.push(items.slice(i, i + PARAM_SLICE))
return out
}
export class QueueStore {
private readonly db: D1Database
constructor(db: D1Database) {
this.db = db
}
async statuses(urls: string[]): Promise<Map<string, CandidateStatus>> {
const out = new Map<string, CandidateStatus>()
for (const slice of slices(urls)) {
const { results } = await this.db
.prepare(`SELECT url, status FROM candidates WHERE url IN (${slice.map(() => '?').join(',')})`)
.bind(...slice)
.all<{ url: string; status: CandidateStatus }>()
for (const r of results) out.set(r.url, r.status)
}
return out
}
/** Insert new candidates; an existing `failed` row is reopened as pending. */
async insert(items: NewCandidate[], now = new Date()): Promise<void> {
if (items.length === 0) return
const stmt = this.db.prepare(
`INSERT INTO candidates (url, source, title, image_url, genre_hint, flags, note, submitter, discovered_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (url) DO UPDATE SET status = 'pending', source = excluded.source,
title = coalesce(excluded.title, candidates.title),
image_url = coalesce(excluded.image_url, candidates.image_url),
flags = excluded.flags, note = excluded.note, submitter = excluded.submitter,
discovered_at = excluded.discovered_at, decided_at = NULL, decided_by = NULL
WHERE candidates.status = 'failed'`,
)
await this.db.batch(
items.map((c) =>
stmt.bind(
c.url,
c.source,
c.title ?? null,
c.image_url ?? null,
c.genre_hint ?? null,
JSON.stringify(c.flags ?? []),
c.note ?? null,
c.submitter ?? null,
isoSeconds(now),
),
),
)
}
async list(status: CandidateStatus, limit = 200): Promise<CandidateRow[]> {
const { results } = await this.db
.prepare('SELECT * FROM candidates WHERE status = ? ORDER BY discovered_at DESC, url LIMIT ?')
.bind(status, limit)
.all<CandidateRow>()
return results
}
async counts(): Promise<Record<CandidateStatus, number>> {
const out: Record<CandidateStatus, number> = { pending: 0, rejected: 0, queued: 0, ingested: 0, failed: 0 }
const { results } = await this.db
.prepare('SELECT status, count(*) AS n FROM candidates GROUP BY status')
.all<{ status: CandidateStatus; n: number }>()
for (const r of results) out[r.status] = r.n
return out
}
async get(urls: string[]): Promise<CandidateRow[]> {
const out: CandidateRow[] = []
for (const slice of slices(urls)) {
const { results } = await this.db
.prepare(`SELECT * FROM candidates WHERE url IN (${slice.map(() => '?').join(',')})`)
.bind(...slice)
.all<CandidateRow>()
out.push(...results)
}
return out
}
async setStatus(urls: string[], status: CandidateStatus, actor: string, now = new Date()): Promise<void> {
if (urls.length === 0) return
const stmt = this.db.prepare('UPDATE candidates SET status = ?, decided_by = ?, decided_at = ? WHERE url = ?')
await this.db.batch(urls.map((u) => stmt.bind(status, actor, isoSeconds(now), u)))
}
/**
* queued → ingested once the game is in the published catalog; queued for
* more than 3 days without showing up (the ingest workflow gave up on it)
* → failed, so it comes back for review.
*/
async reconcile(catalog: Set<string>, now = new Date()): Promise<{ ingested: number; failed: number }> {
const { results } = await this.db
.prepare("SELECT url, decided_at FROM candidates WHERE status = 'queued'")
.all<{ url: string; decided_at: string | null }>()
const cutoff = isoSeconds(new Date(now.getTime() - QUEUED_TIMEOUT_MS))
const ingested = results.filter((c) => catalog.has(c.url)).map((c) => c.url)
const failed = results.filter((c) => !catalog.has(c.url) && (c.decided_at ?? '') < cutoff).map((c) => c.url)
const set = this.db.prepare('UPDATE candidates SET status = ? WHERE url = ?')
const updates = [...ingested.map((u) => set.bind('ingested', u)), ...failed.map((u) => set.bind('failed', u))]
if (updates.length > 0) await this.db.batch(updates)
return { ingested: ingested.length, failed: failed.length }
}
/** Retention: idempotency records after 7 days, Suggest-form notes after 180. */
async purge(now = new Date()): Promise<void> {
await this.db.batch([
this.db
.prepare('DELETE FROM idempotency WHERE created_at < ?')
.bind(isoSeconds(new Date(now.getTime() - IDEMPOTENCY_TTL_MS))),
this.db
.prepare("UPDATE candidates SET note = NULL WHERE source = 'form' AND note IS NOT NULL AND discovered_at < ?")
.bind(isoSeconds(new Date(now.getTime() - FORM_NOTE_TTL_MS))),
])
}
async idempotent(client: string, key: string): Promise<{ body_hash: string; response: string } | null> {
return this.db
.prepare('SELECT body_hash, response FROM idempotency WHERE client = ? AND key = ?')
.bind(client, key)
.first<{ body_hash: string; response: string }>()
}
async remember(client: string, key: string, bodyHash: string, response: string, now = new Date()): Promise<void> {
await this.db
.prepare('INSERT OR IGNORE INTO idempotency (client, key, body_hash, response, created_at) VALUES (?, ?, ?, ?, ?)')
.bind(client, key, bodyHash, response, isoSeconds(now))
.run()
}
async feeds(): Promise<Map<string, FeedState>> {
const { results } = await this.db.prepare('SELECT * FROM feed_state').all<FeedState>()
return new Map(results.map((f) => [f.feed, f]))
}
async saveFeed(state: FeedState): Promise<void> {
await this.db
.prepare(
`INSERT INTO feed_state (feed, last_polled_at, backoff_until, last_status) VALUES (?, ?, ?, ?)
ON CONFLICT (feed) DO UPDATE SET last_polled_at = excluded.last_polled_at,
backoff_until = excluded.backoff_until, last_status = excluded.last_status`,
)
.bind(state.feed, state.last_polled_at, state.backoff_until, state.last_status)
.run()
}
async audit(actor: string, action: string, target: string, commitSha: string | null, detail?: string) {
await this.db
.prepare('INSERT INTO audit_log (actor, action, target, commit_sha, detail) VALUES (?, ?, ?, ?, ?)')
.bind(actor, action, target, commitSha, detail ?? null)
.run()
}
}
+139
View File
@@ -0,0 +1,139 @@
/**
* Admin writes to the repo, each as one commit built from the files at the
* current head of main (commitWithRetry re-reads and rebuilds on a race).
* Pure builders are exported for tests; the GitHub reads stay thin.
* Commit messages never name the maintainer (their email stays in the
* private D1 audit log): the repo and its history are public.
*/
import { canonicalize } from './canonical'
import {
applyAdminChange,
dumps,
type CatalogFiles,
type CatalogIndex,
type CountPoint,
type DeletedEntry,
type FieldEdit,
type Game,
} from './catalog'
import { commitWithRetry, listDir, readFile, type CommitInput, type GitHubEnv } from './github'
const TEMP_LINK = 'scripts/temp_link.json'
const DELETED_LOG = 'scripts/deleted_games.json'
const CHUNK_FILE = /^game_info_\d+\.json$/
type Change = Omit<CommitInput, 'expectedHeadOid'>
async function readJsonFile<T>(env: GitHubEnv, path: string, head: string, fallback: T): Promise<T> {
const text = await readFile(env, path, head)
return text === null ? fallback : (JSON.parse(text) as T)
}
/** The catalog files at `head`, chunks in the same order as data_store.py. */
async function readCatalog(env: GitHubEnv, head: string): Promise<CatalogFiles> {
const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort()
const [chunks, index, history, deleted] = await Promise.all([
Promise.all(names.map((n) => readJsonFile<Game[]>(env, `data_game/${n}`, head, []))),
readJsonFile<CatalogIndex | null>(env, 'data_game/index.json', head, null),
readJsonFile<CountPoint[]>(env, 'data_game/count_history.json', head, []),
readJsonFile<DeletedEntry[]>(env, DELETED_LOG, head, []),
])
return { chunks, index, history, deleted }
}
/**
* temp_link.json with `urls` appended (skipping ones already queued in any
* spelling). Unknown entries are kept as they are; apply_patch.py cleans them.
*/
export function withQueued(queueText: string | null, urls: string[]): { text: string; added: string[] } {
const parsed: unknown = queueText === null ? [] : JSON.parse(queueText)
const queue: unknown[] = Array.isArray(parsed) ? parsed : []
const present = new Set(queue.map((u) => (typeof u === 'string' ? (canonicalize(u) ?? u) : '')))
const added: string[] = []
for (const url of urls) {
if (present.has(url)) continue
present.add(url)
added.push(url)
}
return { text: dumps([...queue, ...added]), added }
}
/** deleted_games.json without `urls` (a game the maintainer allows back in). */
export function withoutDeleted(deletedText: string | null, urls: Set<string>): { text: string; removed: number } {
const log = deletedText === null ? [] : (JSON.parse(deletedText) as DeletedEntry[])
const kept = log.filter((e) => !urls.has(e.url))
return { text: dumps(kept), removed: log.length - kept.length }
}
/**
* Queue `urls` for the ingest workflow (the push to temp_link.json triggers
* update.yml). With `unblock`, also drop those URLs from the deleted log,
* which update_info.py otherwise refuses to re-add.
*/
export async function queueForIngest(
env: GitHubEnv,
urls: string[],
unblock: string[],
): Promise<{ sha: string | null; added: string[] }> {
let added: string[] = []
const sha = await commitWithRetry(env, async (head): Promise<Change | null> => {
const writes = new Map<string, string>()
const queued = withQueued(await readFile(env, TEMP_LINK, head), urls)
added = queued.added
if (queued.added.length > 0) writes.set(TEMP_LINK, queued.text)
if (unblock.length > 0) {
const log = withoutDeleted(await readFile(env, DELETED_LOG, head), new Set(unblock))
if (log.removed > 0) writes.set(DELETED_LOG, log.text)
}
if (writes.size === 0) return null
return {
headline: `Queue ${urls.length} game(s) for ingest [admin]`,
body: `Via the admin page.\n\n${urls.join('\n')}`,
writes,
}
})
return { sha, added }
}
export interface CatalogEditResult {
sha: string | null
summary: { edited: number; removed: number }
/** Thumbnails of removed games (their resized copies are deleted from R2). */
thumbnails: string[]
}
/** Edit maintainer fields and/or remove games in one commit. */
export async function editCatalog(
env: GitHubEnv,
edits: Map<string, FieldEdit>,
removals: { url: string; reason: string }[],
now: () => Date = () => new Date(),
): Promise<CatalogEditResult> {
let result: CatalogEditResult = { sha: null, summary: { edited: 0, removed: 0 }, thumbnails: [] }
const sha = await commitWithRetry(env, async (head): Promise<Change | null> => {
const files = await readCatalog(env, head)
const removing = new Set(removals.map((r) => r.url))
const thumbnails = files.chunks
.flat()
.filter((g) => removing.has(g.url) && typeof g.thumbnail === 'string' && g.thumbnail.startsWith('https://'))
.map((g) => g.thumbnail as string)
const change = applyAdminChange(files, edits, removals, now())
result = { sha: null, summary: change.summary, thumbnails }
if (change.writes.size === 0 && change.deletes.length === 0) return null
const parts = [
change.summary.edited ? `edit ${change.summary.edited}` : '',
change.summary.removed ? `remove ${change.summary.removed}` : '',
].filter(Boolean)
const lines = [
...[...edits].map(([url, edit]) => `edit ${url} ${JSON.stringify(edit)}`),
...removals.map((r) => `remove ${r.url} (${r.reason})`),
]
return {
headline: `Catalog: ${parts.join(', ')} [admin]`,
body: `Via the admin page.\n\n${lines.join('\n')}`,
writes: change.writes,
deletes: change.deletes,
}
})
return { ...result, sha }
}
+97
View File
@@ -0,0 +1,97 @@
/**
* Public "suggest a game" form: GET /api/suggest (is it enabled + Turnstile
* site key), POST /api/suggest {url, note?, token}.
*
* Guards: same-origin POST, per-IP rate limit (RL_SUGGEST), Turnstile
* siteverify (the only thing a bot cannot forge), canonical itch.io URL. A
* suggestion only enters the review queue; the answer never says more than
* the public catalog already shows.
*/
import { canonicalize } from './canonical'
import { catalogUrls, deletedGames, type DataSource } from './data'
import { missingConfig, type WorkerEnv } from './env'
import { HttpError, errorResponse, isObject, json, readJson } from './http'
import { autoFlags, classify, type Outcome, type QueueStore } from './queue'
const SITEVERIFY = 'https://challenges.cloudflare.com/turnstile/v0/siteverify'
const MAX_NOTE = 500
type SuggestStatus = 'received' | 'already_listed' | 'already_suggested' | 'not_accepted'
const PUBLIC_STATUS: Record<Exclude<Outcome, 'invalid_url' | 'duplicate_in_request'>, SuggestStatus> = {
queued: 'received',
previously_deleted: 'received', // flagged for the maintainer
duplicate_catalog: 'already_listed',
duplicate_pending: 'already_suggested',
rejected: 'not_accepted',
}
export interface SuggestDeps {
store: QueueStore | null
data: DataSource
fetch: typeof fetch
limiter: Pick<RateLimit, 'limit'> | undefined
}
function enabled(env: WorkerEnv, deps: SuggestDeps): boolean {
return missingConfig(env, ['TURNSTILE_SECRET', 'TURNSTILE_SITEKEY', 'SITE_ORIGIN']).length === 0 && deps.store !== null
}
async function verifyTurnstile(env: WorkerEnv, deps: SuggestDeps, token: string, ip: string): Promise<boolean> {
const form = new FormData()
form.set('secret', env.TURNSTILE_SECRET ?? '')
form.set('response', token)
if (ip) form.set('remoteip', ip)
try {
const res = await deps.fetch(SITEVERIFY, { method: 'POST', body: form })
const outcome = (await res.json()) as { success?: boolean; hostname?: string }
if (!outcome.success) return false
const expected = new URL(env.SITE_ORIGIN).hostname
// Turnstile's test keys answer with a placeholder hostname; only local dev uses them.
return !outcome.hostname || outcome.hostname === expected || expected === 'localhost'
} catch {
return false
}
}
export async function handleSuggest(request: Request, env: WorkerEnv, deps: SuggestDeps): Promise<Response> {
try {
if (request.method === 'GET') {
const on = enabled(env, deps)
return json({ enabled: on, sitekey: on ? env.TURNSTILE_SITEKEY : null })
}
if (request.method !== 'POST') throw new HttpError(405, 'method_not_allowed')
if (!enabled(env, deps) || !deps.store) throw new HttpError(503, 'unavailable')
if (request.headers.get('origin') !== env.SITE_ORIGIN) throw new HttpError(403, 'bad_origin')
const ip = request.headers.get('cf-connecting-ip') ?? ''
if (deps.limiter && !(await deps.limiter.limit({ key: ip || 'unknown' })).success) {
throw new HttpError(429, 'rate_limited')
}
const body = await readJson(request, 4096)
if (!isObject(body) || typeof body.url !== 'string' || typeof body.token !== 'string') {
throw new HttpError(400, 'invalid_body')
}
const note = typeof body.note === 'string' ? body.note.trim() : ''
if (note.length > MAX_NOTE) throw new HttpError(400, 'note_too_long')
const url = canonicalize(body.url)
if (!url) throw new HttpError(400, 'invalid_url')
if (!(await verifyTurnstile(env, deps, body.token, ip))) throw new HttpError(403, 'challenge_failed')
const [catalog, deleted, known] = await Promise.all([
catalogUrls(deps.data),
deletedGames(deps.data),
deps.store.statuses([url]),
])
const [result] = classify([url], { catalog: new Set(catalog.keys()), deleted, known })
if (result.store) {
await deps.store.insert([
{ url, source: 'form', note: note || null, submitter: 'form', flags: autoFlags(result.outcome) },
])
}
const status = PUBLIC_STATUS[result.outcome as keyof typeof PUBLIC_STATUS]
return json({ status })
} catch (e) {
return errorResponse(e)
}
}
+5
View File
@@ -0,0 +1,5 @@
// Vite/Vitest `?raw` imports (used by tests for golden text fixtures).
declare module '*?raw' {
const text: string
export default text
}
+82
View File
@@ -0,0 +1,82 @@
import { SignJWT, exportJWK, generateKeyPair } from 'jose'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { ADMIN_CSP, handleAdminApi, serveAdminApp } from '../admin'
import type { WorkerEnv } from '../env'
import { ORIGIN } from './fakes'
const TEAM = 'https://team.cloudflareaccess.com'
let privateKey: CryptoKey
const ADMIN_SHELL = '<!doctype html><title>admin</title>'
const env = {
SITE_ORIGIN: ORIGIN,
ACCESS_TEAM_DOMAIN: TEAM,
ACCESS_AUD_ADMIN: 'aud-admin',
ACCESS_AUD_INGEST: 'aud-ingest',
ADMIN_EMAILS: 'owner@example.com',
ASSETS: {
fetch: async (req: Request) =>
new URL(req.url).pathname === '/admin/'
? new Response(ADMIN_SHELL, { headers: { 'content-type': 'text/html', 'content-security-policy': "default-src 'self'; script-src https://static.cloudflareinsights.com" } })
: new Response(null, { status: 404 }),
},
} as unknown as WorkerEnv
beforeAll(async () => {
const pair = await generateKeyPair('RS256')
privateKey = pair.privateKey
const jwk = { ...(await exportJWK(pair.publicKey)), kid: 'k1', alg: 'RS256', use: 'sig' }
vi.stubGlobal('fetch', async (url: string) =>
String(url) === `${TEAM}/cdn-cgi/access/certs` ? Response.json({ keys: [jwk] }) : new Response(null, { status: 404 }),
)
})
afterAll(() => vi.unstubAllGlobals())
const jwt = (email: string, aud = 'aud-admin') =>
new SignJWT({ email })
.setProtectedHeader({ alg: 'RS256', kid: 'k1' })
.setIssuer(TEAM)
.setAudience(aud)
.setExpirationTime('5m')
.sign(privateKey)
async function get(path: string, token?: string, e: WorkerEnv = env) {
const url = new URL(path, ORIGIN)
const headers: Record<string, string> = token ? { 'cf-access-jwt-assertion': token } : {}
const request = new Request(url, { headers })
return path.startsWith('/api/') ? handleAdminApi(request, url, e, null) : serveAdminApp(request, url, e)
}
describe('admin authentication', () => {
it('serves the admin shell only to the maintainer, with its own CSP', async () => {
const res = await get('/admin/', await jwt('owner@example.com'))
expect(res.status).toBe(200)
expect(await res.text()).toBe(ADMIN_SHELL)
expect(res.headers.get('content-security-policy')).toBe(ADMIN_CSP)
expect(res.headers.get('cache-control')).toBe('no-store')
expect(res.headers.get('x-robots-tag')).toContain('noindex')
})
it.each([
['no token', undefined],
['someone else', 'stranger@example.com'],
])('refuses %s', async (_name, email) => {
const token = email ? await jwt(email) : undefined
expect((await get('/admin', token)).status).toBe(403)
expect((await get('/api/admin/me', token)).status).toBe(403)
})
it('refuses a token minted for the ingest application', async () => {
expect((await get('/admin/', await jwt('owner@example.com', 'aud-ingest'))).status).toBe(403)
})
it('answers 503 until Access is configured, never serving the shell', async () => {
const bare = { ...env, ACCESS_AUD_ADMIN: '' } as WorkerEnv
expect((await get('/admin/', await jwt('owner@example.com'), bare)).status).toBe(503)
expect((await get('/api/admin/me', await jwt('owner@example.com'), bare)).status).toBe(503)
})
it('does not serve files from under /admin', async () => {
expect((await get('/admin/secrets.json', await jwt('owner@example.com'))).status).toBe(404)
})
})
+195
View File
@@ -0,0 +1,195 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { adminApi, type AdminContext, type RepoOps } from '../admin'
import { resetDataCache } from '../data'
import { r2Key } from '../img'
import { QueueStore } from '../queue'
import { withoutDeleted, withQueued } from '../repo'
import { testDb } from './d1'
import { fakeData, ORIGIN } from './fakes'
let db: D1Database
let dispose: () => Promise<void>
let store: QueueStore
beforeAll(async () => {
;({ db, dispose } = await testDb())
}, 60_000)
afterAll(() => dispose())
beforeEach(async () => {
resetDataCache()
await db.batch(['candidates', 'audit_log'].map((t) => db.prepare(`DELETE FROM ${t}`)))
store = new QueueStore(db)
})
const NOW = new Date('2026-09-18T12:00:00Z')
const THUMB = 'https://img.itch.zone/aW1n/original/cover.png'
function fakeRepo() {
const repo = {
queueForIngest: vi.fn(async (urls: string[]) => ({ sha: 'sha-queue', added: urls })),
editCatalog: vi.fn(async (edits: Map<string, unknown>, removals: { url: string }[]) => ({
sha: 'sha-edit',
summary: { edited: edits.size, removed: removals.length },
thumbnails: removals.length ? [THUMB] : [],
})),
}
return repo satisfies RepoOps
}
function context(over: Partial<AdminContext> = {}) {
const thumbs = { delete: vi.fn(async () => undefined) }
const ctx: AdminContext = {
actor: 'owner@example.com',
store,
data: fakeData(['https://dev.itch.io/listed'], [{ url: 'https://dev.itch.io/gone' }]),
repo: fakeRepo(),
thumbs,
now: NOW,
...over,
}
return { ctx, thumbs, repo: ctx.repo as ReturnType<typeof fakeRepo> }
}
async function call(ctx: AdminContext, method: string, path: string, body?: unknown) {
const url = new URL(path, ORIGIN)
const init: RequestInit = { method }
if (body !== undefined) {
init.body = JSON.stringify(body)
init.headers = { 'content-type': 'application/json' }
}
const res = await adminApi(new Request(url, init), url, ctx)
return { status: res.status, body: (await res.json()) as Record<string, unknown> }
}
describe('admin API', () => {
it('reports the maintainer and whether writes are possible', async () => {
expect((await call(context().ctx, 'GET', '/api/admin/me')).body).toEqual({ email: 'owner@example.com', writes: true })
expect((await call(context({ repo: null }).ctx, 'GET', '/api/admin/me')).body.writes).toBe(false)
})
it('lists the queue with parsed flags and counts', async () => {
await store.insert([{ url: 'https://dev.itch.io/a', source: 'rss', flags: ['demo'] }], NOW)
const { body } = await call(context().ctx, 'GET', '/api/admin/queue?status=pending')
expect(body.counts).toMatchObject({ pending: 1 })
expect(body.items).toEqual([expect.objectContaining({ url: 'https://dev.itch.io/a', flags: ['demo'] })])
expect((await call(context().ctx, 'GET', '/api/admin/queue?status=bogus')).status).toBe(400)
})
it('approves pending rows into one commit and records the decision', async () => {
await store.insert(['a', 'b'].map((s) => ({ url: `https://dev.itch.io/${s}`, source: 'rss' as const })), NOW)
const { ctx, repo } = context()
const { body } = await call(ctx, 'POST', '/api/admin/queue/decide', {
action: 'approve',
urls: ['https://dev.itch.io/a', 'https://dev.itch.io/b', 'https://dev.itch.io/unknown'],
})
expect(body).toEqual({
sha: 'sha-queue',
approved: ['https://dev.itch.io/a', 'https://dev.itch.io/b'],
skipped: [{ url: 'https://dev.itch.io/unknown', reason: 'not_in_queue' }],
})
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/a', 'https://dev.itch.io/b'], [])
expect((await store.statuses(['https://dev.itch.io/a'])).get('https://dev.itch.io/a')).toBe('queued')
const audit = await db.prepare('SELECT actor, action, commit_sha FROM audit_log').first()
expect(audit).toEqual({ actor: 'owner@example.com', action: 'approve', commit_sha: 'sha-queue' })
})
it('needs an override to bring back a game that was removed', async () => {
await store.insert([{ url: 'https://dev.itch.io/gone', source: 'form', flags: ['previously_deleted'] }], NOW)
const { ctx, repo } = context()
const refused = await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'approve', urls: ['https://dev.itch.io/gone'] })
expect(refused.body.skipped).toEqual([{ url: 'https://dev.itch.io/gone', reason: 'needs_override' }])
expect(repo.queueForIngest).not.toHaveBeenCalled()
await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'approve', urls: ['https://dev.itch.io/gone'], override: true })
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/gone'], ['https://dev.itch.io/gone'])
})
it('rejects rows (a permanent blocklist) without touching the repo', async () => {
await store.insert([{ url: 'https://dev.itch.io/spam', source: 'form' }], NOW)
const { ctx, repo } = context()
await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'reject', urls: ['https://dev.itch.io/spam'] })
expect((await store.statuses(['https://dev.itch.io/spam'])).get('https://dev.itch.io/spam')).toBe('rejected')
expect(repo.queueForIngest).not.toHaveBeenCalled()
})
it('adds and approves pasted URLs in one step', async () => {
await store.insert([{ url: 'https://dev.itch.io/waiting', source: 'rss' }], NOW)
await store.insert([{ url: 'https://dev.itch.io/blocked', source: 'form' }], NOW)
await store.setStatus(['https://dev.itch.io/blocked'], 'rejected', 'x', NOW)
const { ctx, repo } = context()
const { body } = await call(ctx, 'POST', '/api/admin/add', {
urls: ['https://dev.itch.io/new', 'https://dev.itch.io/listed', 'https://dev.itch.io/waiting', 'https://dev.itch.io/blocked', 'x'],
})
const results = body.results as { url: string | null; outcome: string; approved: boolean }[]
expect(results.map((r) => [r.outcome, r.approved])).toEqual([
['queued', true],
['duplicate_catalog', false],
['duplicate_pending', true],
['rejected', false],
['invalid_url', false],
])
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/new', 'https://dev.itch.io/waiting'], [])
expect((await store.get(['https://dev.itch.io/new']))[0]).toMatchObject({ source: 'admin', status: 'queued' })
})
it('edits only maintainer fields', async () => {
const { ctx, repo } = context()
const ok = await call(ctx, 'PATCH', '/api/admin/games', { url: 'https://dev.itch.io/listed', edit: { nsfw: 'Yes', notes: 'x' } })
expect(ok.body).toEqual({ sha: 'sha-edit', edited: 1 })
expect(repo.editCatalog).toHaveBeenCalledWith(new Map([['https://dev.itch.io/listed', { nsfw: 'Yes', notes: 'x' }]]), [])
expect((await call(ctx, 'PATCH', '/api/admin/games', { url: 'https://dev.itch.io/listed', edit: { rating: '5' } })).status).toBe(400)
expect((await call(ctx, 'PATCH', '/api/admin/games', { url: 'https://dev.itch.io/listed', edit: { nsfw: 'maybe' } })).status).toBe(400)
})
it('removes a game with a reason and deletes its resized covers', async () => {
const { ctx, repo, thumbs } = context()
expect((await call(ctx, 'DELETE', '/api/admin/games', { url: 'https://dev.itch.io/listed' })).status).toBe(400)
const { body } = await call(ctx, 'DELETE', '/api/admin/games', { url: 'https://dev.itch.io/listed', reason: 'Takedown request' })
expect(body).toEqual({ sha: 'sha-edit', removed: 1 })
expect(repo.editCatalog).toHaveBeenCalledWith(new Map(), [{ url: 'https://dev.itch.io/listed', reason: 'Takedown request' }])
expect(thumbs.delete).toHaveBeenCalledWith([await r2Key(160, THUMB), await r2Key(640, THUMB)])
})
it('restores a removed game: unblock + queue again', async () => {
const { ctx, repo } = context()
const { body } = await call(ctx, 'POST', '/api/admin/games/restore', { url: 'https://dev.itch.io/gone' })
expect(body).toEqual({ sha: 'sha-queue', queued: true })
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/gone'], ['https://dev.itch.io/gone'])
expect((await store.statuses(['https://dev.itch.io/gone'])).get('https://dev.itch.io/gone')).toBe('queued')
})
it('stays read-only without the GitHub App', async () => {
await store.insert([{ url: 'https://dev.itch.io/a', source: 'rss' }], NOW)
const { ctx } = context({ repo: null })
const res = await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'approve', urls: ['https://dev.itch.io/a'] })
expect(res.status).toBe(503)
expect((await store.statuses(['https://dev.itch.io/a'])).get('https://dev.itch.io/a')).toBe('pending')
})
it('validates bodies and routes', async () => {
const { ctx } = context()
expect((await call(ctx, 'POST', '/api/admin/add', { urls: [] })).status).toBe(400)
expect((await call(ctx, 'POST', '/api/admin/add', { urls: Array(201).fill('https://dev.itch.io/a') })).status).toBe(413)
expect((await call(ctx, 'POST', '/api/admin/nope', {})).status).toBe(404)
})
})
describe('repo builders', () => {
it('appends queued URLs once, in any spelling, keeping odd entries', () => {
const current = JSON.stringify(['https://DEV.itch.io/a/', { url: 'https://dev.itch.io/obj' }], null, 4)
const { text, added } = withQueued(current, ['https://dev.itch.io/a', 'https://dev.itch.io/b', 'https://dev.itch.io/b'])
expect(added).toEqual(['https://dev.itch.io/b'])
expect(JSON.parse(text)).toEqual(['https://DEV.itch.io/a/', { url: 'https://dev.itch.io/obj' }, 'https://dev.itch.io/b'])
expect(text).toBe(JSON.stringify(JSON.parse(text), null, 4)) // json_io.dumps format
expect(withQueued(null, ['https://dev.itch.io/x']).text).toBe('[\n "https://dev.itch.io/x"\n]')
})
it('drops unblocked URLs from the deleted log', () => {
const log = JSON.stringify([
{ url: 'https://dev.itch.io/a', name: 'A', reason: 'r', deleted_at: 't' },
{ url: 'https://dev.itch.io/b', name: 'B', reason: 'r', deleted_at: 't' },
])
const { text, removed } = withoutDeleted(log, new Set(['https://dev.itch.io/a']))
expect(removed).toBe(1)
expect(JSON.parse(text).map((e: { url: string }) => e.url)).toEqual(['https://dev.itch.io/b'])
})
})
+20
View File
@@ -0,0 +1,20 @@
/**
* A real (local, in-memory) D1 database with the migrations applied, for
* tests of the SQL in queue.ts. Runs workerd through wrangler's platform proxy.
*/
import { getPlatformProxy } from 'wrangler'
import init from '../migrations/0001_init.sql?raw'
export async function testDb(): Promise<{ db: D1Database; dispose: () => Promise<void> }> {
const proxy = await getPlatformProxy<{ DB: D1Database }>({
configPath: new URL('./wrangler.test.jsonc', import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'),
persist: false,
})
const statements = init
.replace(/--.*$/gm, '')
.split(';')
.map((s) => s.trim())
.filter(Boolean)
for (const sql of statements) await proxy.env.DB.prepare(sql).run()
return { db: proxy.env.DB, dispose: proxy.dispose }
}
+121
View File
@@ -0,0 +1,121 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { resetDataCache } from '../data'
import { discover, FEEDS, parseFeed, pickFeed } from '../discover'
import { QueueStore, type FeedState } from '../queue'
import { testDb } from './d1'
import { fakeData } from './fakes'
// Same shape as itch.io's browse feeds; made-up games.
const item = (slug: string, title: string, tags: string, price = '$0.00', image = true) => `<item>
<guid>https://studio.itch.io/${slug}</guid><title>${title} ${tags}</title><plainTitle>${title}</plainTitle>
${image ? `<imageurl>https://img.itch.zone/aW1n/315x250%23c/${slug}.png</imageurl>` : '<imageurl>https://evil.example/x.png</imageurl>'}
<price>${price}</price><currency>USD</currency><link>https://studio.itch.io/${slug}</link>
<description><![CDATA[Short blurb <img src="x"/>]]></description><pubDate>Mon, 14 Sep 2026 10:00:00 GMT</pubDate>
</item>`
const FEED = `<?xml version="1.0" encoding="UTF-8" ?><rss version="2.0"><channel><title>t</title>
${item('moon-garden', 'Moon Garden', '[Free] [Puzzle] [Windows] [macOS]')}
${item('ink-and-rain', 'Ink &amp; Rain', '[Free] [Visual Novel] [HTML5]')}
${item('paid-thing', 'Paid Thing', '[$4.99] [Action]', '$4.99')}
${item('lewd-demo', 'Lewd Demo', '[Free] [Simulation]', '$0.00', false)}
${item('already-listed', 'Already Listed', '[Free] [Platformer]')}
</channel></rss>`
describe('parseFeed', () => {
it('reads link, plain title, genre, price and a safe cover', () => {
const items = parseFeed(FEED)
expect(items).toHaveLength(5)
expect(items[0]).toEqual({
url: 'https://studio.itch.io/moon-garden',
title: 'Moon Garden',
image: 'https://img.itch.zone/aW1n/315x250%23c/moon-garden.png',
genre: 'Puzzle',
free: true,
})
expect(items[1]).toMatchObject({ title: 'Ink & Rain', genre: 'Visual Novel' })
expect(items[2].free).toBe(false)
expect(items[3].image).toBeNull() // only img.itch.zone covers are kept
})
})
describe('pickFeed', () => {
const now = new Date('2026-09-18T12:00:00Z')
const state = (feed: string, polled: string | null, backoff: string | null = null): [string, FeedState] => [
feed,
{ feed, last_polled_at: polled, backoff_until: backoff, last_status: 200 },
]
it('takes the feed polled longest ago that is not backing off', () => {
expect(pickFeed(FEEDS, new Map(), now)?.id).toBe('free')
expect(pickFeed(FEEDS, new Map([state('free', '2026-09-18T08:00:00Z')]), now)?.id).toBe('visual-novel')
const both = new Map([state('free', '2026-09-18T04:00:00Z'), state('visual-novel', '2026-09-18T08:00:00Z')])
expect(pickFeed(FEEDS, both, now)?.id).toBe('free')
both.set(...state('free', '2026-09-18T04:00:00Z', '2026-09-18T14:00:00Z'))
expect(pickFeed(FEEDS, both, now)?.id).toBe('visual-novel')
both.set(...state('visual-novel', '2026-09-18T08:00:00Z', '2026-09-18T13:00:00Z'))
expect(pickFeed(FEEDS, both, now)).toBeNull()
})
})
describe('discover', () => {
let db: D1Database
let dispose: () => Promise<void>
let store: QueueStore
const now = new Date('2026-09-18T12:00:00Z')
beforeAll(async () => {
;({ db, dispose } = await testDb())
}, 60_000)
afterAll(() => dispose())
beforeEach(async () => {
resetDataCache()
await db.batch(['candidates', 'feed_state'].map((t) => db.prepare(`DELETE FROM ${t}`)))
store = new QueueStore(db)
})
it('queues new free games from one feed, with hints and flags', async () => {
const fetch = vi.fn(async () => new Response(FEED, { headers: { 'content-type': 'application/xml' } }))
const result = await discover({
store,
data: fakeData(['https://studio.itch.io/already-listed']),
fetch: fetch as unknown as typeof globalThis.fetch,
userAgent: 'FreeItchGamesBot/test',
now,
})
expect(result).toEqual({ feed: 'free', status: 200, items: 4, added: 3 })
const [, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]
expect((init.headers as Record<string, string>)['User-Agent']).toBe('FreeItchGamesBot/test')
const rows = new Map((await store.list('pending')).map((r) => [r.url, r]))
expect([...rows.keys()].sort()).toEqual([
'https://studio.itch.io/ink-and-rain',
'https://studio.itch.io/lewd-demo',
'https://studio.itch.io/moon-garden',
])
expect(rows.get('https://studio.itch.io/moon-garden')).toMatchObject({
source: 'rss',
submitter: 'rss:free',
genre_hint: 'Puzzle',
title: 'Moon Garden',
})
expect(JSON.parse(rows.get('https://studio.itch.io/lewd-demo')!.flags)).toEqual(['nsfw_keyword', 'demo'])
expect((await store.feeds()).get('free')).toMatchObject({ last_status: 200, backoff_until: null })
})
it('backs off for at least 6 hours on 429, longer if asked', async () => {
const fetch = vi.fn(async () => new Response(null, { status: 429, headers: { 'retry-after': '43200' } }))
const result = await discover({ store, data: fakeData(), fetch: fetch as unknown as typeof globalThis.fetch, userAgent: 'x', now })
expect(result.status).toBe(429)
expect((await store.feeds()).get('free')?.backoff_until).toBe('2026-09-19T00:00:00Z')
// Next run moves on to the other feed.
const ok = vi.fn(async () => new Response('<rss></rss>'))
expect((await discover({ store, data: fakeData(), fetch: ok as unknown as typeof globalThis.fetch, userAgent: 'x', now })).feed).toBe(
'visual-novel',
)
})
it('records network errors without throwing', async () => {
const fetch = vi.fn(async () => {
throw new TypeError('Network connection lost.')
})
const result = await discover({ store, data: fakeData(), fetch: fetch as unknown as typeof globalThis.fetch, userAgent: 'x', now })
expect(result).toMatchObject({ feed: 'free', status: 0, added: 0 })
})
})
+43
View File
@@ -0,0 +1,43 @@
/** Shared fakes for the Worker handler tests. */
import type { DataSource } from '../data'
export const ORIGIN = 'https://freeitchgames.win'
/** Static assets holding a catalog (/data/urls.json) and a deleted log. */
export function fakeData(
catalog: string[] = [],
deleted: { url: string; reason?: string; deleted_at?: string }[] = [],
): DataSource {
const urls = Object.fromEntries(catalog.map((u) => [u, 'https://img.itch.zone/x/original/y.png']))
const log = deleted.map((d) => ({
url: d.url,
name: 'N',
reason: d.reason ?? 'Game became paid',
deleted_at: d.deleted_at ?? '2026-06-19T03:00:00Z',
}))
return {
origin: ORIGIN,
assets: {
fetch: async (input: RequestInfo | URL) => {
const path = new URL(input instanceof Request ? input.url : String(input)).pathname
if (path === '/data/urls.json') return Response.json(urls)
if (path === '/data/deleted_games.json') return Response.json(log)
return new Response(null, { status: 404 })
},
} as unknown as Fetcher,
}
}
/** A rate limiter that allows `allowed` calls, then refuses. */
export function fakeLimiter(allowed = Infinity) {
let calls = 0
const keys: string[] = []
return {
keys,
limit: async ({ key }: { key: string }) => {
keys.push(key)
calls += 1
return { success: calls <= allowed }
},
}
}
+106
View File
@@ -0,0 +1,106 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
import { QueueStore } from '../queue'
import { testDb } from './d1'
let db: D1Database
let dispose: () => Promise<void>
let store: QueueStore
beforeAll(async () => {
;({ db, dispose } = await testDb())
}, 60_000)
afterAll(() => dispose())
beforeEach(async () => {
await db.batch(['candidates', 'feed_state', 'audit_log', 'idempotency'].map((t) => db.prepare(`DELETE FROM ${t}`)))
store = new QueueStore(db)
})
const T0 = new Date('2026-09-18T00:00:00Z')
const hours = (h: number) => new Date(T0.getTime() + h * 3_600_000)
const url = (slug: string) => `https://dev.itch.io/${slug}`
describe('QueueStore', () => {
it('stores candidates, reports statuses and lists newest first', async () => {
await store.insert([{ url: url('a'), source: 'rss', title: 'A', flags: ['demo'] }], T0)
await store.insert([{ url: url('b'), source: 'form', note: 'hi' }], hours(1))
expect(await store.statuses([url('a'), url('b'), url('zzz')])).toEqual(
new Map([
[url('a'), 'pending'],
[url('b'), 'pending'],
]),
)
const rows = await store.list('pending')
expect(rows.map((r) => r.url)).toEqual([url('b'), url('a')])
expect(JSON.parse(rows[1].flags)).toEqual(['demo'])
expect((await store.counts()).pending).toBe(2)
})
it('never overwrites a live row, but reopens a failed one', async () => {
await store.insert([{ url: url('a'), source: 'rss', title: 'Old' }], T0)
await store.setStatus([url('a')], 'rejected', 'me@example.com', T0)
await store.insert([{ url: url('a'), source: 'form', title: 'New' }], hours(1))
expect((await store.get([url('a')]))[0]).toMatchObject({ status: 'rejected', source: 'rss', title: 'Old' })
await store.setStatus([url('a')], 'failed', 'reconcile', hours(2))
await store.insert([{ url: url('a'), source: 'ext', note: 'again' }], hours(3))
expect((await store.get([url('a')]))[0]).toMatchObject({
status: 'pending',
source: 'ext',
title: 'Old', // kept when the new submission has none
note: 'again',
decided_at: null,
discovered_at: '2026-09-18T03:00:00Z',
})
})
it('queries more URLs than one statement can bind', async () => {
const urls = Array.from({ length: 250 }, (_, i) => url(`g${i}`))
await store.insert(urls.map((u) => ({ url: u, source: 'rss' as const })), T0)
expect((await store.statuses(urls)).size).toBe(250)
expect((await store.get(urls)).length).toBe(250)
})
it('reconciles queued rows: ingested when in the catalog, failed after 3 days', async () => {
await store.insert([url('in'), url('slow'), url('fresh')].map((u) => ({ url: u, source: 'admin' as const })), T0)
await store.setStatus([url('in'), url('slow')], 'queued', 'me', T0)
await store.setStatus([url('fresh')], 'queued', 'me', hours(70))
expect(await store.reconcile(new Set([url('in')]), hours(73))).toEqual({ ingested: 1, failed: 1 })
expect(await store.statuses([url('in'), url('slow'), url('fresh')])).toEqual(
new Map([
[url('in'), 'ingested'],
[url('slow'), 'failed'],
[url('fresh'), 'queued'],
]),
)
})
it('purges old idempotency records and old Suggest-form notes', async () => {
await store.remember('ext', 'key-old-0001', 'h', '{}', T0)
await store.remember('ext', 'key-new-0001', 'h', '{}', hours(24 * 6))
await store.insert([{ url: url('form'), source: 'form', note: 'contact me' }], T0)
await store.insert([{ url: url('ext'), source: 'ext', note: 'kept' }], T0)
await store.purge(hours(24 * 181))
expect(await store.idempotent('ext', 'key-old-0001')).toBeNull()
expect(await store.idempotent('ext', 'key-new-0001')).toBeNull() // 175 days old by then
const notes = new Map((await store.get([url('form'), url('ext')])).map((r) => [r.url, r.note]))
expect(notes.get(url('form'))).toBeNull()
expect(notes.get(url('ext'))).toBe('kept')
})
it('keeps idempotency records per client and ignores a duplicate insert', async () => {
await store.remember('ext-a', 'same-key-01', 'h1', '{"n":1}', T0)
await store.remember('ext-a', 'same-key-01', 'h2', '{"n":2}', T0)
await store.remember('ext-b', 'same-key-01', 'h3', '{"n":3}', T0)
expect(await store.idempotent('ext-a', 'same-key-01')).toEqual({ body_hash: 'h1', response: '{"n":1}' })
expect(await store.idempotent('ext-b', 'same-key-01')).toEqual({ body_hash: 'h3', response: '{"n":3}' })
})
it('upserts feed state and writes the audit log', async () => {
await store.saveFeed({ feed: 'free', last_polled_at: 'a', backoff_until: null, last_status: 200 })
await store.saveFeed({ feed: 'free', last_polled_at: 'b', backoff_until: 'c', last_status: 429 })
expect((await store.feeds()).get('free')).toEqual({ feed: 'free', last_polled_at: 'b', backoff_until: 'c', last_status: 429 })
await store.audit('me@example.com', 'approve', url('a'), 'abc123')
const row = await db.prepare('SELECT actor, action, target, commit_sha FROM audit_log').first()
expect(row).toEqual({ actor: 'me@example.com', action: 'approve', target: url('a'), commit_sha: 'abc123' })
})
})
+134
View File
@@ -0,0 +1,134 @@
import { exportPKCS8, generateKeyPair } from 'jose'
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { dumps } from '../catalog'
import { resetTokenCache, type GitHubEnv } from '../github'
import { editCatalog, queueForIngest } from '../repo'
let env: GitHubEnv
beforeAll(async () => {
const pair = await generateKeyPair('RS256', { extractable: true })
env = {
GITHUB_REPO: 'owner/repo',
GH_APP_ID: '1',
GH_APP_INSTALLATION_ID: '2',
GH_APP_PRIVATE_KEY: await exportPKCS8(pair.privateKey),
}
})
const game = (slug: string, extra: Record<string, unknown> = {}) => ({
url: `https://dev.itch.io/${slug}`,
name: slug,
thumbnail: `https://img.itch.zone/aW1n/original/${slug}.png`,
nsfw: 'No',
notes: '',
safe_virus: '?',
...extra,
})
/** A tiny in-memory GitHub: files at one head, captures the commit mutation. */
function fakeGitHub(files: Record<string, string>) {
const commits: { headline: string; body?: string; additions: Record<string, string>; deletions: string[] }[] = []
vi.stubGlobal('fetch', async (input: string, init: RequestInit = {}) => {
const url = new URL(String(input))
const path = url.pathname
if (path.endsWith('/access_tokens')) {
return Response.json({ token: 't', expires_at: new Date(Date.now() + 3_600_000).toISOString() })
}
if (path === '/repos/owner/repo/git/ref/heads/main') return Response.json({ object: { sha: 'head1' } })
if (path === '/graphql') {
const input = JSON.parse(String(init.body)).variables.input
const decode = (b64: string) => new TextDecoder().decode(Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)))
commits.push({
headline: input.message.headline,
body: input.message.body,
additions: Object.fromEntries(input.fileChanges.additions.map((a: { path: string; contents: string }) => [a.path, decode(a.contents)])),
deletions: input.fileChanges.deletions.map((d: { path: string }) => d.path),
})
return Response.json({ data: { createCommitOnBranch: { commit: { oid: 'new-sha' } } } })
}
const file = path.replace('/repos/owner/repo/contents/', '')
if (url.searchParams.get('ref') !== 'head1') return new Response(null, { status: 404 })
if (file === 'data_game') {
return Response.json(
Object.keys(files)
.filter((f) => f.startsWith('data_game/'))
.map((f) => ({ name: f.slice('data_game/'.length), type: 'file' })),
)
}
return file in files ? new Response(files[file]) : new Response(null, { status: 404 })
})
return commits
}
beforeEach(() => resetTokenCache())
afterEach(() => vi.unstubAllGlobals())
describe('editCatalog', () => {
const chunk = [game('a'), game('b'), game('c')]
const files = {
'data_game/game_info_001.json': dumps(chunk),
'data_game/index.json': dumps({
total_games: 3,
max_per_file: 500,
last_updated: '2026-09-01T00:00:00Z',
files: [{ name: 'game_info_001.json', count: 3 }],
}),
'data_game/count_history.json': dumps([{ date: '2026-09-01', total: 3 }]),
'scripts/deleted_games.json': dumps([]),
}
it('removes a game in one commit: chunk, index, history, deleted log', async () => {
const commits = fakeGitHub(files)
const result = await editCatalog(env, new Map(), [{ url: 'https://dev.itch.io/b', reason: 'Takedown request' }], () => new Date('2026-09-18T12:00:00Z'))
expect(result).toEqual({
sha: 'new-sha',
summary: { edited: 0, removed: 1 },
thumbnails: ['https://img.itch.zone/aW1n/original/b.png'],
})
const [c] = commits
expect(Object.keys(c.additions).sort()).toEqual([
'data_game/count_history.json',
'data_game/game_info_001.json',
'data_game/index.json',
'scripts/deleted_games.json',
])
expect(JSON.parse(c.additions['data_game/game_info_001.json']).map((g: { url: string }) => g.url)).toEqual([
'https://dev.itch.io/a',
'https://dev.itch.io/c',
])
expect(JSON.parse(c.additions['scripts/deleted_games.json'])).toEqual([
{ url: 'https://dev.itch.io/b', name: 'b', reason: 'Takedown request', deleted_at: '2026-09-18T12:00:00Z' },
])
expect(`${c.headline}\n${c.body}`).not.toMatch(/@/) // never the maintainer's email
})
it('edits one field and makes no commit when nothing changes', async () => {
const commits = fakeGitHub(files)
await editCatalog(env, new Map([['https://dev.itch.io/a', { nsfw: 'Yes' }]]), [])
expect(Object.keys(commits[0].additions)).toEqual(['data_game/game_info_001.json', 'data_game/index.json'])
const unchanged = await editCatalog(env, new Map([['https://dev.itch.io/a', { nsfw: 'No' }]]), [])
expect(unchanged.sha).toBeNull()
expect(commits).toHaveLength(1)
})
})
describe('queueForIngest', () => {
it('appends to temp_link.json and, when unblocking, prunes the deleted log', async () => {
const commits = fakeGitHub({
'scripts/temp_link.json': dumps(['https://dev.itch.io/queued']),
'scripts/deleted_games.json': dumps([{ url: 'https://dev.itch.io/b', name: 'b', reason: 'r', deleted_at: 't' }]),
})
const { sha, added } = await queueForIngest(env, ['https://dev.itch.io/queued', 'https://dev.itch.io/b'], ['https://dev.itch.io/b'])
expect(sha).toBe('new-sha')
expect(added).toEqual(['https://dev.itch.io/b'])
expect(JSON.parse(commits[0].additions['scripts/temp_link.json'])).toEqual(['https://dev.itch.io/queued', 'https://dev.itch.io/b'])
expect(JSON.parse(commits[0].additions['scripts/deleted_games.json'])).toEqual([])
})
it('makes no commit when everything is already queued', async () => {
const commits = fakeGitHub({ 'scripts/temp_link.json': dumps(['https://dev.itch.io/queued']) })
expect((await queueForIngest(env, ['https://dev.itch.io/queued'], [])).sha).toBeNull()
expect(commits).toHaveLength(0)
})
})
+176
View File
@@ -0,0 +1,176 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { resetDataCache } from '../data'
import type { WorkerEnv } from '../env'
import { HttpError } from '../http'
import { handleIngest } from '../ingest'
import { QueueStore } from '../queue'
import { handleSuggest } from '../suggest'
import { testDb } from './d1'
import { fakeData, fakeLimiter, ORIGIN } from './fakes'
let db: D1Database
let dispose: () => Promise<void>
let store: QueueStore
beforeAll(async () => {
;({ db, dispose } = await testDb())
}, 60_000)
afterAll(() => dispose())
beforeEach(async () => {
resetDataCache()
await db.batch(['candidates', 'idempotency'].map((t) => db.prepare(`DELETE FROM ${t}`)))
store = new QueueStore(db)
})
const env = {
SITE_ORIGIN: ORIGIN,
TURNSTILE_SITEKEY: '0x4AAA-site',
TURNSTILE_SECRET: 'secret',
} as unknown as WorkerEnv
function siteverify(success: boolean, hostname = 'freeitchgames.win') {
return vi.fn(async () => Response.json({ success, hostname }))
}
function suggestRequest(body: unknown, headers: Record<string, string> = {}) {
return new Request(`${ORIGIN}/api/suggest`, {
method: 'POST',
headers: { 'content-type': 'application/json', origin: ORIGIN, 'cf-connecting-ip': '203.0.113.9', ...headers },
body: JSON.stringify(body),
})
}
describe('POST /api/suggest', () => {
const deps = (over: Partial<Parameters<typeof handleSuggest>[2]> = {}) => ({
store,
data: fakeData(['https://dev.itch.io/listed'], [{ url: 'https://dev.itch.io/gone' }]),
fetch: siteverify(true) as unknown as typeof fetch,
limiter: fakeLimiter(),
...over,
})
it('tells the page whether the form is enabled', async () => {
const on = await handleSuggest(new Request(`${ORIGIN}/api/suggest`), env, deps())
expect(await on.json()).toEqual({ enabled: true, sitekey: '0x4AAA-site' })
const off = await handleSuggest(new Request(`${ORIGIN}/api/suggest`), { ...env, TURNSTILE_SECRET: undefined }, deps())
expect(await off.json()).toEqual({ enabled: false, sitekey: null })
})
it('queues a new game with its note', async () => {
const fetch = siteverify(true)
const res = await handleSuggest(
suggestRequest({ url: 'https://Dev.itch.io/New-Game/', note: ' cozy puzzle ', token: 't' }),
env,
deps({ fetch: fetch as unknown as typeof globalThis.fetch }),
)
expect(await res.json()).toEqual({ status: 'received' })
const [row] = await store.get(['https://dev.itch.io/new-game'])
expect(row).toMatchObject({ source: 'form', note: 'cozy puzzle', submitter: 'form', status: 'pending' })
const form = (fetch.mock.calls[0] as unknown as [string, RequestInit])[1].body as FormData
expect(form.get('remoteip')).toBe('203.0.113.9')
})
it('answers what is already known without storing it again', async () => {
const listed = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/listed', token: 't' }), env, deps())
expect(await listed.json()).toEqual({ status: 'already_listed' })
await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/twice', token: 't' }), env, deps())
const again = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/twice', token: 't' }), env, deps())
expect(await again.json()).toEqual({ status: 'already_suggested' })
const gone = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/gone', token: 't' }), env, deps())
expect(await gone.json()).toEqual({ status: 'received' })
expect(JSON.parse((await store.get(['https://dev.itch.io/gone']))[0].flags)).toEqual(['previously_deleted'])
})
it.each([
['a failed challenge', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), { fetch: siteverify(false) }, 403],
['a token for another site', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), { fetch: siteverify(true, 'evil.example') }, 403],
['a cross-site post', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }, { origin: 'https://evil.example' }), {}, 403],
['a non-itch URL', suggestRequest({ url: 'https://example.com/game', token: 't' }), {}, 400],
['an over-long note', suggestRequest({ url: 'https://dev.itch.io/x', note: 'x'.repeat(501), token: 't' }), {}, 400],
['the rate limit', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), { limiter: fakeLimiter(0) }, 429],
])('refuses %s', async (_name, request, over, status) => {
const res = await handleSuggest(request, env, deps(over as Parameters<typeof deps>[0]))
expect(res.status).toBe(status)
expect(await store.counts()).toMatchObject({ pending: 0 })
})
it('is unavailable until configured', async () => {
const res = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), env, deps({ store: null }))
expect(res.status).toBe(503)
})
})
describe('POST /api/ingest', () => {
const deps = (over: Partial<Parameters<typeof handleIngest>[1]> = {}) => ({
store,
data: fakeData(['https://dev.itch.io/listed']),
limiter: fakeLimiter(),
authenticate: async () => 'ext-client.access',
...over,
})
const ingestRequest = (body: unknown, key = 'key-0000-0001') =>
new Request(`${ORIGIN}/api/ingest`, {
method: 'POST',
headers: { 'content-type': 'application/json', 'idempotency-key': key },
body: typeof body === 'string' ? body : JSON.stringify(body),
})
it('queues items and reports each outcome, attributed to the token', async () => {
const res = await handleIngest(
ingestRequest({
items: [
{ url: 'https://dev.itch.io/fresh', title: 'Fresh Demo', note: 'from a jam' },
{ url: 'https://dev.itch.io/listed' },
'https://DEV.itch.io/fresh/',
'nope',
],
}),
deps(),
)
expect(res.status).toBe(200)
expect(await res.json()).toEqual({
received: 1,
results: [
{ url: 'https://dev.itch.io/fresh', canonical: 'https://dev.itch.io/fresh', outcome: 'queued' },
{ url: 'https://dev.itch.io/listed', canonical: 'https://dev.itch.io/listed', outcome: 'duplicate_catalog' },
{ url: 'https://DEV.itch.io/fresh/', canonical: 'https://dev.itch.io/fresh', outcome: 'duplicate_in_request' },
{ url: 'nope', canonical: null, outcome: 'invalid_url' },
],
})
const [row] = await store.get(['https://dev.itch.io/fresh'])
expect(row).toMatchObject({ source: 'ext', submitter: 'ext-client.access', title: 'Fresh Demo', note: 'from a jam' })
expect(JSON.parse(row.flags)).toEqual(['demo'])
})
it('replays a retry with the same key and refuses the key with another body', async () => {
const first = await handleIngest(ingestRequest({ urls: ['https://dev.itch.io/a'] }), deps())
const again = await handleIngest(ingestRequest({ urls: ['https://dev.itch.io/a'] }), deps())
expect(again.headers.get('idempotent-replayed')).toBe('true')
expect(await again.json()).toEqual(await first.json())
const other = await handleIngest(ingestRequest({ urls: ['https://dev.itch.io/b'] }), deps())
expect(other.status).toBe(422)
})
it.each([
['no idempotency key', ingestRequest({ urls: ['https://dev.itch.io/a'] }, ''), {}, 400],
['too many items', ingestRequest({ urls: Array.from({ length: 151 }, (_, i) => `https://dev.itch.io/g${i}`) }), {}, 413],
['a body over 64 KB', ingestRequest({ urls: ['https://dev.itch.io/a'], pad: 'x'.repeat(70_000) }), {}, 413],
['invalid JSON', ingestRequest('{nope'), {}, 400],
['an empty list', ingestRequest({ items: [] }), {}, 400],
['the rate limit', ingestRequest({ urls: ['https://dev.itch.io/a'] }), { limiter: fakeLimiter(0) }, 429],
[
'a failed Access check',
ingestRequest({ urls: ['https://dev.itch.io/a'] }),
{
authenticate: async () => {
throw new HttpError(403, 'forbidden')
},
},
403,
],
])('refuses %s', async (_name, request, over, status) => {
const res = await handleIngest(request, deps(over as Parameters<typeof deps>[0]))
expect(res.status).toBe(status)
expect(await store.counts()).toMatchObject({ pending: 0 })
})
})
+6
View File
@@ -0,0 +1,6 @@
// Bindings for the D1 tests only (worker/test/d1.ts, in-memory, never deployed).
{
"name": "freeitchgames-test",
"compatibility_date": "2026-09-15",
"d1_databases": [{ "binding": "DB", "database_name": "test", "database_id": "00000000-0000-0000-0000-000000000000" }]
}
+59 -5
View File
@@ -3,17 +3,23 @@
// Root directory: webapp · Build command: npm run build · Deploy command: npx wrangler deploy
// Custom Domains (freeitchgames.win, www) are attached in the dashboard and are
// intentionally not declared here yet, so a deploy never touches them.
//
// Secrets are NOT in this file: `npx wrangler secret put <NAME>` (and
// webapp/.dev.vars for `wrangler dev`) — GH_APP_PRIVATE_KEY, ADMIN_EMAILS,
// TURNSTILE_SECRET. Until a feature's settings are filled in, its routes
// answer 503 (see worker/env.ts); the public site works without any of them.
// After editing this file run `npm run cf:types`.
{
"$schema": "./node_modules/wrangler/config-schema.json",
// MUST match the Worker name in the dashboard or Workers Builds fails.
"name": "free-games-itchio-list",
"main": "./worker/index.ts",
"compatibility_date": "2026-09-15",
// workers.dev / preview URLs would bypass Cloudflare Access (admin, Phase 4).
// workers.dev / preview URLs would bypass Cloudflare Access (admin, ingest).
"workers_dev": false,
"preview_urls": false,
"assets": {
// Vite build output (app + bundled catalog under /data). Never a source dir.
// Vite build output (app + admin + bundled catalog under /data). Never a source dir.
"directory": "./dist",
"binding": "ASSETS",
// Misses go to the Worker, not to an asset-layer SPA fallback: that would
@@ -23,22 +29,70 @@
// (worker/spa.ts).
"not_found_handling": "none",
"html_handling": "auto-trailing-slash",
// Existing assets never invoke the Worker (free, cached); these paths always do.
"run_worker_first": ["/img/*", "/api/*"]
// Existing assets never invoke the Worker (free, cached); these paths always
// do — /admin so its shell is only ever served after the Access JWT check.
"run_worker_first": ["/img/*", "/api/*", "/admin", "/admin/*"]
},
// Resized cover images, written once per image + width by worker/img.ts.
"r2_buckets": [{ "binding": "THUMBS", "bucket_name": "freeitchgames-thumbs" }],
// Review queue (worker/migrations). No database_id: `wrangler deploy` binds the
// database by name (and creates it on the first deploy if it does not exist).
// Migrations are applied by hand, never by the build:
// npx wrangler d1 migrations apply freeitchgames --remote
"d1_databases": [
{ "binding": "DB", "database_name": "freeitchgames", "migrations_dir": "./worker/migrations" }
],
// Per-key limits (IP for the Suggest form, service-token id for ingest).
"ratelimits": [
{ "name": "RL_SUGGEST", "namespace_id": "1001", "simple": { "limit": 5, "period": 60 } },
{ "name": "RL_INGEST", "namespace_id": "1002", "simple": { "limit": 20, "period": 60 } }
],
// RSS discovery (one feed per run) + queue bookkeeping, worker/index.ts scheduled().
"triggers": { "crons": ["23 */4 * * *"] },
"vars": {
"SITE_ORIGIN": "https://freeitchgames.win",
"GITHUB_REPO": "poli0981/free-games-itchio-list",
"BOT_UA": "FreeItchGamesBot/4.0 (+https://freeitchgames.win/about)",
// Filled in during setup (none of these are secret):
"GH_APP_ID": "",
"GH_APP_INSTALLATION_ID": "",
"ACCESS_TEAM_DOMAIN": "", // https://<team>.cloudflareaccess.com
"ACCESS_AUD_ADMIN": "", // AUD tag of the "Admin" Access application
"ACCESS_AUD_INGEST": "", // AUD tag of the "Ingest" Access application
"TURNSTILE_SITEKEY": ""
},
"observability": { "enabled": true, "head_sampling_rate": 0.1 },
"upload_source_maps": false,
"env": {
// Pre-release checks of the v4 branch: Worker "free-games-itchio-list-staging",
// deployed with `npx wrangler deploy --env staging`, whole host behind Access.
// Bindings and vars are not inherited, so every one is repeated here.
"staging": {
"workers_dev": false,
"preview_urls": false,
"routes": [{ "pattern": "staging.freeitchgames.win", "custom_domain": true }],
"r2_buckets": [{ "binding": "THUMBS", "bucket_name": "freeitchgames-thumbs-staging" }]
"r2_buckets": [{ "binding": "THUMBS", "bucket_name": "freeitchgames-thumbs-staging" }],
"d1_databases": [
{ "binding": "DB", "database_name": "freeitchgames-staging", "migrations_dir": "./worker/migrations" }
],
"ratelimits": [
{ "name": "RL_SUGGEST", "namespace_id": "2001", "simple": { "limit": 5, "period": 60 } },
{ "name": "RL_INGEST", "namespace_id": "2002", "simple": { "limit": 20, "period": 60 } }
],
// No RSS polling from staging: production is the only itch.io client.
"triggers": { "crons": [] },
"vars": {
"SITE_ORIGIN": "https://staging.freeitchgames.win",
"GITHUB_REPO": "poli0981/free-games-itchio-list",
"BOT_UA": "FreeItchGamesBot/4.0 (+https://freeitchgames.win/about)",
"GH_APP_ID": "",
"GH_APP_INSTALLATION_ID": "",
"ACCESS_TEAM_DOMAIN": "",
"ACCESS_AUD_ADMIN": "",
"ACCESS_AUD_INGEST": "",
"TURNSTILE_SITEKEY": ""
}
}
}
}