feat(worker): review queue, admin API, ingest and suggest endpoints (v4 Phase 4)
Adds the server side of the admin and the automatic game discovery. The catalog stays JSON in this repo; the Worker writes it as a GitHub App (GraphQL createCommitOnBranch with expectedHeadOid: verified commits, retried on a race) and keeps only the review queue in D1. - worker/admin.ts: /api/admin/* (queue list, approve/reject, add, edit the three maintainer fields, remove with reason + R2 cover cleanup, restore) and the /admin shell with its own CSP. Every request re-verifies the Cloudflare Access JWT (aud + maintainer email + same-origin writes). Approving commits to scripts/temp_link.json, which starts update.yml. The maintainer's email goes to the D1 audit log only, never a commit. - worker/ingest.ts: POST /api/ingest for the browser extension (Access service token, per-client rate limit, Idempotency-Key replay, <= 150 items / 64 KB), replacing its PAT commits. - worker/suggest.ts: public POST /api/suggest (same-origin, per-IP rate limit, Turnstile siteverify, canonical itch.io URL) into the queue. - worker/discover.ts + scheduled(): every 4 h, one itch.io RSS feed (round-robin, 6 h back-off on 429) into the queue, then reconcile (queued -> ingested / failed) and retention (idempotency 7 d, form notes 180 d). - worker/queue.ts: URL classification (catalog, pending, deleted, blocklist) and the D1 store; worker/repo.ts builds each admin commit from the files at the head of main, byte-compatible with the Python pipeline (golden fixtures). - wrangler.jsonc: D1 (bound by name; migrations applied by hand), rate limits, cron, non-secret settings as vars (empty until setup: features answer 503 until configured), staging env without the cron. `npm run cf:types` ignores .dev.vars so the committed types are stable. - Tests: 159 passing; the SQL runs against a real local D1 (workerd via wrangler's platform proxy). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
38 files changed
+3276
-17
No files matched your search
@@ -40,7 +40,7 @@ jobs:
|
||||
run: npm ci
|
||||
|
||||
- name: Worker types are up to date
|
||||
run: npx wrangler types --check
|
||||
run: npm run cf:types -- --check
|
||||
|
||||
- name: Lint
|
||||
run: npm run lint
|
||||
@@ -48,11 +48,11 @@ jobs:
|
||||
- name: Unused code (knip)
|
||||
run: npm run knip
|
||||
|
||||
- name: Unit tests (Worker)
|
||||
- name: Unit tests (Worker, app; D1 tests run a local workerd)
|
||||
run: npm test
|
||||
|
||||
- name: Type-check and build
|
||||
run: npm run build
|
||||
|
||||
- name: Validate Workers config (dry run)
|
||||
run: npx wrangler deploy --dry-run
|
||||
run: npx wrangler deploy --dry-run --env=""
|
||||
Generated
+10
@@ -28,6 +28,7 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"idb-keyval": "^6.3.0",
|
||||
"jose": "^6.2.12",
|
||||
"lucide-react": "^1.47.0",
|
||||
"react": "^19.3.0",
|
||||
"react-dom": "^19.3.0",
|
||||
@@ -6189,6 +6190,15 @@
|
||||
"jiti": "bin/jiti.js"
|
||||
}
|
||||
},
|
||||
"node_modules/jose": {
|
||||
"version": "6.2.12",
|
||||
"resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz",
|
||||
"integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/panva"
|
||||
}
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
|
||||
+2
-1
@@ -16,7 +16,7 @@
|
||||
"tauri:dev": "tauri dev",
|
||||
"tauri:build": "tauri build",
|
||||
"test": "vitest run",
|
||||
"cf:types": "wrangler types",
|
||||
"cf:types": "wrangler types --env-file=types.env --strict-vars=false",
|
||||
"cf:dev": "wrangler dev"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -40,6 +40,7 @@
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"idb-keyval": "^6.3.0",
|
||||
"jose": "^6.2.12",
|
||||
"lucide-react": "^1.47.0",
|
||||
"react": "^19.3.0",
|
||||
"react-dom": "^19.3.0",
|
||||
|
||||
@@ -152,6 +152,7 @@ export const THIRD_PARTY: ThirdParty[] = [
|
||||
{ name: 'idb-keyval', version: '6.3', license: 'Apache-2.0', url: 'https://github.com/jakearchibald/idb-keyval', category: 'data' },
|
||||
|
||||
// Hosting / tooling (Cloudflare)
|
||||
{ name: 'jose', version: '6.2', license: 'MIT', url: 'https://github.com/panva/jose', category: 'dev' },
|
||||
{ name: 'Wrangler', version: '4.134', license: 'MIT OR Apache-2.0', url: 'https://developers.cloudflare.com/workers/wrangler/', category: 'dev' },
|
||||
{ name: 'Vitest', version: '5.0', license: 'MIT', url: 'https://vitest.dev', category: 'dev' },
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
"lib": ["ES2024"],
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"types": [],
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# Empty on purpose: `npm run cf:types` points wrangler here so secrets in a
|
||||
# local .dev.vars never end up in the committed worker-configuration.d.ts.
|
||||
+31
-1
@@ -1,9 +1,21 @@
|
||||
/* eslint-disable */
|
||||
// Generated by Wrangler by running `wrangler types` (hash: b477d9057afc335e7b8bb1ffae6b449c)
|
||||
// Generated by Wrangler by running `wrangler types --env-file=types.env --strict-vars=false` (hash: f483b1f0658395bde79d44bd8ede839a)
|
||||
// Runtime types generated with workerd@1.20260917.1 2026-09-15
|
||||
interface __BaseEnv_Env {
|
||||
THUMBS: R2Bucket;
|
||||
DB: D1Database;
|
||||
RL_SUGGEST: RateLimit;
|
||||
RL_INGEST: RateLimit;
|
||||
ASSETS: Fetcher;
|
||||
SITE_ORIGIN: string;
|
||||
GITHUB_REPO: string;
|
||||
BOT_UA: string;
|
||||
GH_APP_ID: string;
|
||||
GH_APP_INSTALLATION_ID: string;
|
||||
ACCESS_TEAM_DOMAIN: string;
|
||||
ACCESS_AUD_ADMIN: string;
|
||||
ACCESS_AUD_INGEST: string;
|
||||
TURNSTILE_SITEKEY: string;
|
||||
}
|
||||
declare namespace Cloudflare {
|
||||
interface GlobalProps {
|
||||
@@ -11,11 +23,29 @@ declare namespace Cloudflare {
|
||||
}
|
||||
interface StagingEnv {
|
||||
THUMBS: R2Bucket;
|
||||
DB: D1Database;
|
||||
RL_SUGGEST: RateLimit;
|
||||
RL_INGEST: RateLimit;
|
||||
ASSETS: Fetcher;
|
||||
SITE_ORIGIN: string;
|
||||
GITHUB_REPO: string;
|
||||
BOT_UA: string;
|
||||
GH_APP_ID: string;
|
||||
GH_APP_INSTALLATION_ID: string;
|
||||
ACCESS_TEAM_DOMAIN: string;
|
||||
ACCESS_AUD_ADMIN: string;
|
||||
ACCESS_AUD_INGEST: string;
|
||||
TURNSTILE_SITEKEY: string;
|
||||
}
|
||||
interface Env extends __BaseEnv_Env {}
|
||||
}
|
||||
interface Env extends __BaseEnv_Env {}
|
||||
type StringifyValues<EnvType extends Record<string, unknown>> = {
|
||||
[Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string;
|
||||
};
|
||||
declare namespace NodeJS {
|
||||
interface ProcessEnv extends StringifyValues<Pick<Cloudflare.Env, "SITE_ORIGIN" | "GITHUB_REPO" | "BOT_UA" | "GH_APP_ID" | "GH_APP_INSTALLATION_ID" | "ACCESS_TEAM_DOMAIN" | "ACCESS_AUD_ADMIN" | "ACCESS_AUD_INGEST" | "TURNSTILE_SITEKEY">> {}
|
||||
}
|
||||
|
||||
// Begin runtime types
|
||||
/*! *****************************************************************************
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { SignJWT, exportJWK, generateKeyPair } from 'jose'
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
import { AccessError, requireMaintainer, requireService, type AccessEnv } from './access'
|
||||
|
||||
const TEAM = 'https://example.cloudflareaccess.com'
|
||||
const env: AccessEnv = {
|
||||
ACCESS_TEAM_DOMAIN: TEAM,
|
||||
ACCESS_AUD_ADMIN: 'aud-admin',
|
||||
ACCESS_AUD_INGEST: 'aud-ingest',
|
||||
ADMIN_EMAILS: 'Owner@Example.com, second@example.com',
|
||||
SITE_ORIGIN: 'https://freeitchgames.win',
|
||||
}
|
||||
|
||||
let privateKey: CryptoKey
|
||||
|
||||
beforeAll(async () => {
|
||||
const pair = await generateKeyPair('RS256')
|
||||
privateKey = pair.privateKey
|
||||
const jwk = { ...(await exportJWK(pair.publicKey)), kid: 'k1', alg: 'RS256', use: 'sig' }
|
||||
vi.stubGlobal('fetch', async (url: string) => {
|
||||
if (String(url) === `${TEAM}/cdn-cgi/access/certs`) return Response.json({ keys: [jwk] })
|
||||
return new Response(null, { status: 404 })
|
||||
})
|
||||
})
|
||||
afterAll(() => vi.unstubAllGlobals())
|
||||
|
||||
async function token(claims: Record<string, unknown>, aud: string, iss = TEAM) {
|
||||
return new SignJWT(claims)
|
||||
.setProtectedHeader({ alg: 'RS256', kid: 'k1' })
|
||||
.setIssuer(iss)
|
||||
.setAudience(aud)
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('5m')
|
||||
.sign(privateKey)
|
||||
}
|
||||
|
||||
function request(jwt: string | null, method = 'GET', origin?: string) {
|
||||
const headers: Record<string, string> = {}
|
||||
if (jwt) headers['cf-access-jwt-assertion'] = jwt
|
||||
if (origin) headers.origin = origin
|
||||
return new Request('https://freeitchgames.win/api/admin/me', { method, headers })
|
||||
}
|
||||
|
||||
describe('requireMaintainer', () => {
|
||||
it('accepts a listed email (case-insensitive)', async () => {
|
||||
const jwt = await token({ email: 'owner@example.com' }, 'aud-admin')
|
||||
await expect(requireMaintainer(request(jwt), env)).resolves.toEqual({
|
||||
kind: 'maintainer',
|
||||
id: 'owner@example.com',
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects missing, forged, wrong-audience and wrong-issuer tokens', async () => {
|
||||
await expect(requireMaintainer(request(null), env)).rejects.toBeInstanceOf(AccessError)
|
||||
const other = await generateKeyPair('RS256')
|
||||
const forged = await new SignJWT({ email: 'owner@example.com' })
|
||||
.setProtectedHeader({ alg: 'RS256', kid: 'k1' })
|
||||
.setIssuer(TEAM)
|
||||
.setAudience('aud-admin')
|
||||
.setExpirationTime('5m')
|
||||
.sign(other.privateKey)
|
||||
await expect(requireMaintainer(request(forged), env)).rejects.toThrow('invalid')
|
||||
const wrongAud = await token({ email: 'owner@example.com' }, 'aud-ingest')
|
||||
await expect(requireMaintainer(request(wrongAud), env)).rejects.toThrow('invalid')
|
||||
const wrongIss = await token({ email: 'owner@example.com' }, 'aud-admin', 'https://evil.example')
|
||||
await expect(requireMaintainer(request(wrongIss), env)).rejects.toThrow('invalid')
|
||||
})
|
||||
|
||||
it('rejects emails that are not maintainers', async () => {
|
||||
const jwt = await token({ email: 'someone@example.com' }, 'aud-admin')
|
||||
await expect(requireMaintainer(request(jwt), env)).rejects.toThrow('not a maintainer')
|
||||
})
|
||||
|
||||
it('requires the site origin on writes', async () => {
|
||||
const jwt = await token({ email: 'owner@example.com' }, 'aud-admin')
|
||||
await expect(requireMaintainer(request(jwt, 'POST'), env)).rejects.toThrow('bad origin')
|
||||
await expect(requireMaintainer(request(jwt, 'POST', 'https://evil.example'), env)).rejects.toThrow('bad origin')
|
||||
await expect(requireMaintainer(request(jwt, 'POST', env.SITE_ORIGIN), env)).resolves.toBeTruthy()
|
||||
})
|
||||
})
|
||||
|
||||
describe('requireService', () => {
|
||||
it('accepts a service token and names it by client id', async () => {
|
||||
const jwt = await token({ common_name: 'client-123.access' }, 'aud-ingest')
|
||||
await expect(requireService(request(jwt, 'POST'), env)).resolves.toEqual({
|
||||
kind: 'service',
|
||||
id: 'client-123.access',
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects user tokens and admin-audience tokens', async () => {
|
||||
const user = await token({ email: 'owner@example.com', common_name: 'x' }, 'aud-ingest')
|
||||
await expect(requireService(request(user), env)).rejects.toThrow('not a service token')
|
||||
const adminAud = await token({ common_name: 'client-123.access' }, 'aud-admin')
|
||||
await expect(requireService(request(adminAud), env)).rejects.toThrow('invalid')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* Cloudflare Access verification for /admin and /api/admin/* (maintainer) and
|
||||
* /api/ingest (service tokens, e.g. the browser extension).
|
||||
*
|
||||
* Access sits in front of those paths, but a Worker with static assets does
|
||||
* not receive `ctx.access`, and requests that bypass the zone (workers.dev,
|
||||
* previews — both disabled in wrangler.jsonc) would skip Access entirely.
|
||||
* So every protected request re-verifies the `Cf-Access-Jwt-Assertion` JWT
|
||||
* here: signature against the team's JWKS, issuer, and the application's AUD.
|
||||
*/
|
||||
import { createRemoteJWKSet, jwtVerify, type JWTPayload } from 'jose'
|
||||
|
||||
export interface AccessEnv {
|
||||
ACCESS_TEAM_DOMAIN: string // https://<team>.cloudflareaccess.com
|
||||
ACCESS_AUD_ADMIN: string
|
||||
ACCESS_AUD_INGEST: string
|
||||
ADMIN_EMAILS: string // comma-separated; a secret, not in the repo
|
||||
SITE_ORIGIN: string
|
||||
}
|
||||
|
||||
export interface Identity {
|
||||
kind: 'maintainer' | 'service'
|
||||
/** Email for people, service-token client id (common_name) for services. */
|
||||
id: string
|
||||
}
|
||||
|
||||
export class AccessError extends Error {}
|
||||
|
||||
const jwksByTeam = new Map<string, ReturnType<typeof createRemoteJWKSet>>()
|
||||
|
||||
function jwks(teamDomain: string) {
|
||||
let set = jwksByTeam.get(teamDomain)
|
||||
if (!set) {
|
||||
set = createRemoteJWKSet(new URL(`${teamDomain}/cdn-cgi/access/certs`))
|
||||
jwksByTeam.set(teamDomain, set)
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
async function verify(request: Request, env: AccessEnv, audience: string): Promise<JWTPayload> {
|
||||
const token = request.headers.get('cf-access-jwt-assertion')
|
||||
if (!token) throw new AccessError('missing Access token')
|
||||
try {
|
||||
const { payload } = await jwtVerify(token, jwks(env.ACCESS_TEAM_DOMAIN), {
|
||||
issuer: env.ACCESS_TEAM_DOMAIN,
|
||||
audience,
|
||||
})
|
||||
return payload
|
||||
} catch {
|
||||
throw new AccessError('invalid Access token')
|
||||
}
|
||||
}
|
||||
|
||||
const SAFE_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])
|
||||
|
||||
/** The maintainer, signed in through the "Admin" Access application. */
|
||||
export async function requireMaintainer(request: Request, env: AccessEnv): Promise<Identity> {
|
||||
const payload = await verify(request, env, env.ACCESS_AUD_ADMIN)
|
||||
const email = String(payload.email ?? '').toLowerCase()
|
||||
const allowed = env.ADMIN_EMAILS.toLowerCase()
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
if (!email || !allowed.includes(email)) throw new AccessError('not a maintainer')
|
||||
// Cross-site request forgery guard for writes (the Access cookie is sent
|
||||
// with top-level cross-site requests too).
|
||||
if (!SAFE_METHODS.has(request.method) && request.headers.get('origin') !== env.SITE_ORIGIN) {
|
||||
throw new AccessError('bad origin')
|
||||
}
|
||||
return { kind: 'maintainer', id: email }
|
||||
}
|
||||
|
||||
/** A service token of the "Ingest" Access application (Service Auth policy). */
|
||||
export async function requireService(request: Request, env: AccessEnv): Promise<Identity> {
|
||||
const payload = await verify(request, env, env.ACCESS_AUD_INGEST)
|
||||
// Service-token JWTs carry the client id as common_name and no email.
|
||||
const id = String(payload.common_name ?? '')
|
||||
if (!id || payload.email) throw new AccessError('not a service token')
|
||||
return { kind: 'service', id }
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
/**
|
||||
* Maintainer-only admin: the JSON API under /api/admin/* and the admin app
|
||||
* shell at /admin. Both sit behind the Cloudflare Access "Admin" application
|
||||
* and re-verify its JWT here (access.ts): the edge login alone is not trusted.
|
||||
*
|
||||
* GET /api/admin/me who am I, what is configured
|
||||
* GET /api/admin/queue?status=… review queue (+ counts per status)
|
||||
* POST /api/admin/queue/decide {urls, action: approve|reject, override?}
|
||||
* POST /api/admin/add {urls, override?} — add + approve at once
|
||||
* PATCH /api/admin/games {url, edit: {safe_virus?, notes?, nsfw?}}
|
||||
* DELETE /api/admin/games {url, reason}
|
||||
* POST /api/admin/games/restore {url} — un-delete: unblock + queue again
|
||||
*
|
||||
* Approving commits the URLs to scripts/temp_link.json (GitHub App,
|
||||
* verified commit); that push starts the ingest workflow, which scrapes and
|
||||
* adds the games. The maintainer's email is recorded only in the D1 audit
|
||||
* log, never in the public repository.
|
||||
*/
|
||||
import { AccessError, requireMaintainer, type AccessEnv } from './access'
|
||||
import { canonicalize } from './canonical'
|
||||
import { validateEdit, type FieldEdit } from './catalog'
|
||||
import { catalogUrls, deletedGames, type DataSource } from './data'
|
||||
import { ACCESS_CONFIG, GITHUB_CONFIG, missingConfig, type WorkerEnv } from './env'
|
||||
import type { GitHubEnv } from './github'
|
||||
import { HttpError, SECURITY_HEADERS, errorResponse, isObject, json, readJson } from './http'
|
||||
import { r2Key } from './img'
|
||||
import { autoFlags, classify, type CandidateRow, type CandidateStatus, type QueueStore } from './queue'
|
||||
import { editCatalog, queueForIngest, type CatalogEditResult } from './repo'
|
||||
|
||||
const MAX_URLS = 200
|
||||
const STATUSES: readonly CandidateStatus[] = ['pending', 'rejected', 'queued', 'ingested', 'failed']
|
||||
const THUMB_WIDTHS = [160, 640]
|
||||
|
||||
// The admin app loads only its own bundle; cover images come straight from
|
||||
// itch.io (candidates are not in the catalog yet, so /img won't serve them).
|
||||
export const ADMIN_CSP = [
|
||||
"default-src 'self'",
|
||||
"script-src 'self'",
|
||||
"style-src 'self' 'unsafe-inline'",
|
||||
"img-src 'self' data: https://img.itch.zone",
|
||||
"connect-src 'self'",
|
||||
"font-src 'self'",
|
||||
"object-src 'none'",
|
||||
"base-uri 'self'",
|
||||
"form-action 'self'",
|
||||
"frame-ancestors 'none'",
|
||||
].join('; ')
|
||||
|
||||
/** Writes to the repo; injectable so tests don't talk to GitHub. */
|
||||
export interface RepoOps {
|
||||
queueForIngest(urls: string[], unblock: string[]): Promise<{ sha: string | null; added: string[] }>
|
||||
editCatalog(edits: Map<string, FieldEdit>, removals: { url: string; reason: string }[]): Promise<CatalogEditResult>
|
||||
}
|
||||
|
||||
export interface AdminContext {
|
||||
actor: string
|
||||
store: QueueStore
|
||||
data: DataSource
|
||||
/** null when the GitHub App is not configured: read-only admin. */
|
||||
repo: RepoOps | null
|
||||
thumbs: Pick<R2Bucket, 'delete'>
|
||||
now: Date
|
||||
}
|
||||
|
||||
function githubRepo(env: WorkerEnv): RepoOps | null {
|
||||
if (missingConfig(env, GITHUB_CONFIG).length > 0) return null
|
||||
const gh = env as WorkerEnv & GitHubEnv
|
||||
return {
|
||||
queueForIngest: (urls, unblock) => queueForIngest(gh, urls, unblock),
|
||||
editCatalog: (edits, removals) => editCatalog(gh, edits, removals),
|
||||
}
|
||||
}
|
||||
|
||||
async function maintainer(request: Request, env: WorkerEnv): Promise<string> {
|
||||
const missing = missingConfig(env, [...ACCESS_CONFIG, 'ACCESS_AUD_ADMIN', 'ADMIN_EMAILS'])
|
||||
if (missing.length > 0) {
|
||||
console.error(`admin: not configured (${missing.join(', ')})`)
|
||||
throw new HttpError(503, 'not_configured')
|
||||
}
|
||||
try {
|
||||
return (await requireMaintainer(request, env as WorkerEnv & AccessEnv)).id
|
||||
} catch (e) {
|
||||
if (e instanceof AccessError) throw new HttpError(403, 'forbidden')
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
function urlList(value: unknown): string[] {
|
||||
if (!Array.isArray(value) || value.length === 0) throw new HttpError(400, 'invalid_body', '`urls` must be a non-empty array')
|
||||
if (value.length > MAX_URLS) throw new HttpError(413, 'too_many_urls', `At most ${MAX_URLS} URLs per request`)
|
||||
if (!value.every((u) => typeof u === 'string' && u.length <= 500)) throw new HttpError(400, 'invalid_body')
|
||||
return value as string[]
|
||||
}
|
||||
|
||||
function gameUrl(value: unknown): string {
|
||||
const url = typeof value === 'string' ? canonicalize(value) : null
|
||||
if (!url) throw new HttpError(400, 'invalid_url')
|
||||
return url
|
||||
}
|
||||
|
||||
function needRepo(ctx: AdminContext): RepoOps {
|
||||
if (!ctx.repo) throw new HttpError(503, 'github_not_configured')
|
||||
return ctx.repo
|
||||
}
|
||||
|
||||
function parseFlags(row: CandidateRow): string[] {
|
||||
try {
|
||||
const flags: unknown = JSON.parse(row.flags)
|
||||
return Array.isArray(flags) ? flags.filter((f): f is string => typeof f === 'string') : []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
interface Skipped {
|
||||
url: string
|
||||
reason: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue candidate rows for ingest. A game that was removed from the catalog
|
||||
* before (it is in the deleted log) needs `override`: approving it also
|
||||
* unblocks it, or the ingest workflow would skip it again.
|
||||
*/
|
||||
async function approve(ctx: AdminContext, urls: string[], override: boolean) {
|
||||
const rows = new Map((await ctx.store.get(urls)).map((r) => [r.url, r]))
|
||||
const deleted = await deletedGames(ctx.data)
|
||||
const approved: string[] = []
|
||||
const skipped: Skipped[] = []
|
||||
for (const url of urls) {
|
||||
const row = rows.get(url)
|
||||
if (!row) skipped.push({ url, reason: 'not_in_queue' })
|
||||
else if (row.status === 'queued' || row.status === 'ingested') skipped.push({ url, reason: `already_${row.status}` })
|
||||
else if (!override && (deleted.has(url) || parseFlags(row).includes('previously_deleted'))) {
|
||||
skipped.push({ url, reason: 'needs_override' })
|
||||
} else approved.push(url)
|
||||
}
|
||||
if (approved.length === 0) return { sha: null, approved, skipped }
|
||||
const { sha } = await needRepo(ctx).queueForIngest(approved, override ? approved : [])
|
||||
await ctx.store.setStatus(approved, 'queued', ctx.actor, ctx.now)
|
||||
await ctx.store.audit(ctx.actor, override ? 'approve_override' : 'approve', approved.join(' '), sha)
|
||||
return { sha, approved, skipped }
|
||||
}
|
||||
|
||||
async function listQueue(url: URL, ctx: AdminContext): Promise<Response> {
|
||||
const status = (url.searchParams.get('status') ?? 'pending') as CandidateStatus
|
||||
if (!STATUSES.includes(status)) throw new HttpError(400, 'invalid_status')
|
||||
const limit = Math.min(Math.max(Number(url.searchParams.get('limit')) || 200, 1), 500)
|
||||
const [counts, rows] = await Promise.all([ctx.store.counts(), ctx.store.list(status, limit)])
|
||||
return json({ counts, items: rows.map((r) => ({ ...r, flags: parseFlags(r) })) })
|
||||
}
|
||||
|
||||
async function decide(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
|
||||
const urls = urlList(body.urls).map(gameUrl)
|
||||
if (body.action === 'reject') {
|
||||
await ctx.store.setStatus(urls, 'rejected', ctx.actor, ctx.now)
|
||||
await ctx.store.audit(ctx.actor, 'reject', urls.join(' '), null)
|
||||
return json({ rejected: urls })
|
||||
}
|
||||
if (body.action !== 'approve') throw new HttpError(400, 'invalid_action')
|
||||
return json(await approve(ctx, urls, body.override === true))
|
||||
}
|
||||
|
||||
async function add(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
|
||||
const inputs = urlList(body.urls)
|
||||
const override = body.override === true
|
||||
const canonical = inputs.map((u) => canonicalize(u)).filter((u): u is string => u !== null)
|
||||
const [catalog, deleted, known] = await Promise.all([
|
||||
catalogUrls(ctx.data),
|
||||
deletedGames(ctx.data),
|
||||
ctx.store.statuses(canonical),
|
||||
])
|
||||
const results = classify(inputs, { catalog: new Set(catalog.keys()), deleted, known })
|
||||
await ctx.store.insert(
|
||||
results
|
||||
.filter((r) => r.store && r.canonical)
|
||||
.map((r) => ({
|
||||
url: r.canonical as string,
|
||||
source: 'admin' as const,
|
||||
submitter: 'admin',
|
||||
flags: autoFlags(r.outcome),
|
||||
})),
|
||||
ctx.now,
|
||||
)
|
||||
// Rows that exist now and are waiting (new, pending, failed, or rejected
|
||||
// and explicitly re-added) are approved right away.
|
||||
const approvable = results
|
||||
.filter(
|
||||
(r) =>
|
||||
r.canonical &&
|
||||
(r.store ||
|
||||
(r.outcome === 'duplicate_pending' && ['pending', 'failed'].includes(known.get(r.canonical) ?? '')) ||
|
||||
(r.outcome === 'rejected' && override)),
|
||||
)
|
||||
.map((r) => r.canonical as string)
|
||||
const decision = approvable.length > 0 ? await approve(ctx, approvable, override) : null
|
||||
const approved = new Set(decision?.approved ?? [])
|
||||
const skipped = new Map((decision?.skipped ?? []).map((s) => [s.url, s.reason]))
|
||||
return json({
|
||||
sha: decision?.sha ?? null,
|
||||
results: results.map((r) => ({
|
||||
input: r.input,
|
||||
url: r.canonical,
|
||||
outcome: r.outcome,
|
||||
approved: r.canonical !== null && approved.has(r.canonical),
|
||||
...(r.canonical && skipped.has(r.canonical) ? { reason: skipped.get(r.canonical) } : {}),
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
async function patchGame(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
|
||||
const url = gameUrl(body.url)
|
||||
if (!isObject(body.edit) || Object.keys(body.edit).length === 0) throw new HttpError(400, 'invalid_edit')
|
||||
const problem = validateEdit(body.edit)
|
||||
if (problem) throw new HttpError(400, 'invalid_edit', problem)
|
||||
const edit = body.edit as FieldEdit
|
||||
const result = await needRepo(ctx).editCatalog(new Map([[url, edit]]), [])
|
||||
if (result.sha) await ctx.store.audit(ctx.actor, 'edit', url, result.sha, JSON.stringify(edit))
|
||||
return json({ sha: result.sha, edited: result.summary.edited })
|
||||
}
|
||||
|
||||
async function deleteGame(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
|
||||
const url = gameUrl(body.url)
|
||||
const reason = typeof body.reason === 'string' ? body.reason.trim() : ''
|
||||
if (!reason || reason.length > 200) throw new HttpError(400, 'invalid_reason', 'A reason (≤ 200 characters) is required')
|
||||
const result = await needRepo(ctx).editCatalog(new Map(), [{ url, reason }])
|
||||
if (result.summary.removed === 0) throw new HttpError(404, 'not_in_catalog')
|
||||
// Resized covers of a removed game (content-removal requests included).
|
||||
const keys = await Promise.all(
|
||||
result.thumbnails.flatMap((thumb) => THUMB_WIDTHS.map((w) => r2Key(w, thumb))),
|
||||
)
|
||||
if (keys.length > 0) await ctx.thumbs.delete(keys)
|
||||
await ctx.store.audit(ctx.actor, 'delete', url, result.sha, reason)
|
||||
return json({ sha: result.sha, removed: result.summary.removed })
|
||||
}
|
||||
|
||||
async function restoreGame(body: Record<string, unknown>, ctx: AdminContext): Promise<Response> {
|
||||
const url = gameUrl(body.url)
|
||||
const { sha, added } = await needRepo(ctx).queueForIngest([url], [url])
|
||||
await ctx.store.insert([{ url, source: 'admin', submitter: 'admin' }], ctx.now)
|
||||
await ctx.store.setStatus([url], 'queued', ctx.actor, ctx.now)
|
||||
await ctx.store.audit(ctx.actor, 'restore', url, sha)
|
||||
return json({ sha, queued: added.length > 0 })
|
||||
}
|
||||
|
||||
/** Route an authenticated admin API request. */
|
||||
export async function adminApi(request: Request, url: URL, ctx: AdminContext): Promise<Response> {
|
||||
try {
|
||||
return await route(request, url, ctx)
|
||||
} catch (e) {
|
||||
return errorResponse(e)
|
||||
}
|
||||
}
|
||||
|
||||
async function route(request: Request, url: URL, ctx: AdminContext): Promise<Response> {
|
||||
const route = `${request.method} ${url.pathname.replace(/\/+$/, '')}`
|
||||
if (route === 'GET /api/admin/me') {
|
||||
return json({ email: ctx.actor, writes: ctx.repo !== null })
|
||||
}
|
||||
if (route === 'GET /api/admin/queue') return listQueue(url, ctx)
|
||||
const body = await readJson(request, 64 * 1024)
|
||||
if (!isObject(body)) throw new HttpError(400, 'invalid_body')
|
||||
switch (route) {
|
||||
case 'POST /api/admin/queue/decide':
|
||||
return decide(body, ctx)
|
||||
case 'POST /api/admin/add':
|
||||
return add(body, ctx)
|
||||
case 'PATCH /api/admin/games':
|
||||
return patchGame(body, ctx)
|
||||
case 'DELETE /api/admin/games':
|
||||
return deleteGame(body, ctx)
|
||||
case 'POST /api/admin/games/restore':
|
||||
return restoreGame(body, ctx)
|
||||
default:
|
||||
throw new HttpError(404, 'not_found')
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleAdminApi(
|
||||
request: Request,
|
||||
url: URL,
|
||||
env: WorkerEnv,
|
||||
store: QueueStore | null,
|
||||
): Promise<Response> {
|
||||
try {
|
||||
const actor = await maintainer(request, env)
|
||||
if (!store) throw new HttpError(503, 'not_configured')
|
||||
return await adminApi(request, url, {
|
||||
actor,
|
||||
store,
|
||||
data: { assets: env.ASSETS, origin: url.origin },
|
||||
repo: githubRepo(env),
|
||||
thumbs: env.THUMBS,
|
||||
now: new Date(),
|
||||
})
|
||||
} catch (e) {
|
||||
return errorResponse(e)
|
||||
}
|
||||
}
|
||||
|
||||
/** /admin and /admin/* → the admin app shell (HashRouter), maintainer only. */
|
||||
export async function serveAdminApp(request: Request, url: URL, env: WorkerEnv): Promise<Response> {
|
||||
const text = (status: number, body: string) =>
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { 'Content-Type': 'text/plain; charset=utf-8', 'Cache-Control': 'no-store', ...SECURITY_HEADERS },
|
||||
})
|
||||
if (request.method !== 'GET' && request.method !== 'HEAD') return text(405, 'Method not allowed')
|
||||
if (/\.[a-z0-9]+$/i.test(url.pathname) && url.pathname !== '/admin/index.html') return text(404, 'Not found')
|
||||
try {
|
||||
await maintainer(request, env)
|
||||
} catch (e) {
|
||||
return e instanceof HttpError && e.status === 503 ? text(503, 'Admin is not configured.') : text(403, 'Forbidden')
|
||||
}
|
||||
const shell = await env.ASSETS.fetch(new Request(new URL('/admin/', url), { headers: request.headers }))
|
||||
const headers = new Headers(shell.headers)
|
||||
headers.set('Content-Security-Policy', ADMIN_CSP)
|
||||
headers.set('Cache-Control', 'no-store')
|
||||
headers.set('X-Robots-Tag', 'noindex, nofollow')
|
||||
headers.set('X-Frame-Options', 'DENY')
|
||||
for (const [k, v] of Object.entries(SECURITY_HEADERS)) headers.set(k, v)
|
||||
return new Response(request.method === 'HEAD' ? null : shell.body, { status: shell.status, headers })
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { canonicalize } from './canonical'
|
||||
// Shared with the Python pipeline (tests/test_canonical.py).
|
||||
import vectors from '../../tests/fixtures/url_vectors.json'
|
||||
|
||||
describe('canonicalize (shared vectors)', () => {
|
||||
it.each(vectors.map((v) => [v.input, v.expected] as const))('%j → %j', (input, expected) => {
|
||||
expect(canonicalize(input)).toBe(expected)
|
||||
})
|
||||
|
||||
it('rejects non-strings', () => {
|
||||
expect(canonicalize(undefined)).toBeNull()
|
||||
expect(canonicalize(42)).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* One canonical form for itch.io game URLs — port of scripts/canonical.py.
|
||||
* Both implementations are tested against tests/fixtures/url_vectors.json.
|
||||
*
|
||||
* Canonical form: https://{creator}.itch.io/{slug}
|
||||
* - https only, host lower-cased, no port / userinfo / query / fragment
|
||||
* - exactly one lower-cased path segment, no trailing slash
|
||||
* - creator must be a *.itch.io subdomain (not itch.io / www.itch.io)
|
||||
*/
|
||||
const HOST = /^[a-z0-9][a-z0-9-]*\.itch\.io$/
|
||||
const SLUG = /^[a-z0-9][a-z0-9_-]*$/
|
||||
const RESERVED_HOSTS = new Set(['itch.io', 'www.itch.io'])
|
||||
|
||||
export function canonicalize(raw: unknown): string | null {
|
||||
if (typeof raw !== 'string') return null
|
||||
let text = raw.trim()
|
||||
if (!text) return null
|
||||
if (!text.includes('://')) text = `https://${text}`
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(text)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
|
||||
if (url.username || url.password) return null
|
||||
const host = url.hostname.toLowerCase()
|
||||
if (RESERVED_HOSTS.has(host) || !HOST.test(host)) return null
|
||||
const segments = url.pathname.split('/').filter(Boolean)
|
||||
if (segments.length !== 1) return null
|
||||
const slug = segments[0].toLowerCase()
|
||||
if (!SLUG.test(slug)) return null
|
||||
return `https://${host}/${slug}`
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
// Shared with the Python pipeline (tests/test_data_store.py).
|
||||
import planCases from '../../tests/fixtures/golden/plan_chunks.json'
|
||||
import serializeInput from '../../tests/fixtures/golden/serialize_input.json'
|
||||
import serializeExpected from '../../tests/fixtures/golden/serialize_expected.txt?raw'
|
||||
import {
|
||||
applyAdminChange,
|
||||
buildIndex,
|
||||
dedupDeleted,
|
||||
dumps,
|
||||
planChunks,
|
||||
upsertCountHistory,
|
||||
validateEdit,
|
||||
type CatalogFiles,
|
||||
type Game,
|
||||
} from './catalog'
|
||||
|
||||
const NOW = new Date('2026-09-20T10:00:00.123Z')
|
||||
const SMALL = { chunkSize: 2, minFill: 1 }
|
||||
|
||||
function game(slug: string, extra: Record<string, unknown> = {}): Game {
|
||||
return { url: `https://dev.itch.io/${slug}`, name: slug, safe_virus: '?', notes: '', nsfw: 'No', ...extra }
|
||||
}
|
||||
|
||||
describe('golden fixtures shared with Python', () => {
|
||||
it.each(planCases.map((c) => [c.name, c] as const))('planChunks: %s', (_name, c) => {
|
||||
const old = c.old.map((chunk) => chunk.map((url) => ({ url })))
|
||||
const games = c.games.map((url) => ({ url }))
|
||||
const got = planChunks(old, games, c.chunk_size, c.min_fill)
|
||||
expect(got.map((chunk) => chunk.map((g) => g.url))).toEqual(c.expected)
|
||||
})
|
||||
|
||||
it('dumps matches json.dumps(indent=4, ensure_ascii=False) byte for byte', () => {
|
||||
expect(dumps(serializeInput)).toBe(serializeExpected)
|
||||
})
|
||||
})
|
||||
|
||||
describe('index and history', () => {
|
||||
it('keeps last_updated when nothing changed', () => {
|
||||
const chunks = [[game('a')]]
|
||||
const prev = buildIndex(chunks, null, true, new Date('2026-01-01T00:00:00Z'))
|
||||
expect(prev.last_updated).toBe('2026-01-01T00:00:00Z')
|
||||
expect(buildIndex(chunks, prev, false, NOW).last_updated).toBe('2026-01-01T00:00:00Z')
|
||||
expect(buildIndex(chunks, prev, true, NOW).last_updated).toBe('2026-09-20T10:00:00Z')
|
||||
})
|
||||
|
||||
it('adds a history row only when the total changes', () => {
|
||||
const h = [{ date: '2026-09-01', total: 10 }]
|
||||
expect(upsertCountHistory(h, 10, NOW)).toEqual(h)
|
||||
expect(upsertCountHistory(h, 9, NOW)).toEqual([...h, { date: '2026-09-20', total: 9 }])
|
||||
expect(upsertCountHistory([{ date: '2026-09-20', total: 5 }], 9, NOW)).toEqual([{ date: '2026-09-20', total: 9 }])
|
||||
})
|
||||
|
||||
it('dedupDeleted keeps the earliest entry per url', () => {
|
||||
const e = (url: string, at: string) => ({ url, name: url, reason: 'r', deleted_at: at })
|
||||
expect(dedupDeleted([e('u1', '2026-02'), e('u1', '2026-01'), e('', '2025'), e('u2', '2025-12')])).toEqual([
|
||||
e('u2', '2025-12'),
|
||||
e('u1', '2026-01'),
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateEdit', () => {
|
||||
it('allows only the three editable fields with valid values', () => {
|
||||
expect(validateEdit({ safe_virus: 'Yes', notes: 'ok', nsfw: 'No' })).toBeNull()
|
||||
expect(validateEdit({ name: 'x' })).toMatch(/not editable/)
|
||||
expect(validateEdit({ safe_virus: 'y' })).toMatch(/safe_virus/)
|
||||
expect(validateEdit({ nsfw: 'maybe' })).toMatch(/nsfw/)
|
||||
expect(validateEdit({ notes: 1 })).toMatch(/string/)
|
||||
expect(validateEdit({ notes: 'x'.repeat(2001) })).toMatch(/2000/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('applyAdminChange', () => {
|
||||
function files(): CatalogFiles {
|
||||
const chunks = [[game('a'), game('b')], [game('c')]]
|
||||
return {
|
||||
chunks,
|
||||
index: buildIndex(chunks, null, true, new Date('2026-09-01T00:00:00Z')),
|
||||
history: [{ date: '2026-09-01', total: 3 }],
|
||||
deleted: [],
|
||||
}
|
||||
}
|
||||
|
||||
it('edits in place and only rewrites the touched chunk + index', () => {
|
||||
const change = applyAdminChange(files(), new Map([['https://dev.itch.io/c', { safe_virus: 'Yes' }]]), [], NOW, SMALL)
|
||||
expect([...change.writes.keys()].sort()).toEqual(['data_game/game_info_002.json', 'data_game/index.json'])
|
||||
expect(JSON.parse(change.writes.get('data_game/game_info_002.json')!)[0].safe_virus).toBe('Yes')
|
||||
expect(change.summary).toEqual({ edited: 1, removed: 0 })
|
||||
})
|
||||
|
||||
it('removes games, logs them and updates index + history in one change', () => {
|
||||
const change = applyAdminChange(
|
||||
files(),
|
||||
new Map(),
|
||||
[{ url: 'https://dev.itch.io/c', reason: 'Removed at the creator’s request' }],
|
||||
NOW,
|
||||
SMALL,
|
||||
)
|
||||
expect(change.deletes).toEqual(['data_game/game_info_002.json'])
|
||||
const index = JSON.parse(change.writes.get('data_game/index.json')!)
|
||||
expect(index.total_games).toBe(2)
|
||||
expect(index.files).toEqual([{ name: 'game_info_001.json', count: 2 }])
|
||||
expect(JSON.parse(change.writes.get('data_game/count_history.json')!).at(-1)).toEqual({ date: '2026-09-20', total: 2 })
|
||||
const deleted = JSON.parse(change.writes.get('scripts/deleted_games.json')!)
|
||||
expect(deleted).toEqual([
|
||||
{ url: 'https://dev.itch.io/c', name: 'c', reason: 'Removed at the creator’s request', deleted_at: '2026-09-20T10:00:00Z' },
|
||||
])
|
||||
})
|
||||
|
||||
it('writes nothing for a no-op edit', () => {
|
||||
const change = applyAdminChange(files(), new Map([['https://dev.itch.io/a', { safe_virus: '?' }]]), [], NOW, SMALL)
|
||||
expect(change.writes.size).toBe(0)
|
||||
expect(change.deletes).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,234 @@
|
||||
/**
|
||||
* Catalog file logic for admin writes — a TypeScript port of
|
||||
* scripts/data_store.py + scripts/json_io.py. Pure functions only: the
|
||||
* GitHub layer reads the files at a commit, calls these, and commits the
|
||||
* result, so the repo stays the single source of truth.
|
||||
*
|
||||
* Kept byte-for-byte compatible with the Python pipeline (same chunk
|
||||
* placement policy, same `json.dumps(indent=4, ensure_ascii=False)` text);
|
||||
* both sides are tested against tests/fixtures/golden/.
|
||||
*/
|
||||
|
||||
const CHUNK_SIZE = 500
|
||||
const MIN_FILL = 250
|
||||
|
||||
/** The only fields a maintainer edits; everything else is scraper-owned. */
|
||||
const EDITABLE_FIELDS = ['safe_virus', 'notes', 'nsfw'] as const
|
||||
type EditableField = (typeof EDITABLE_FIELDS)[number]
|
||||
|
||||
const SAFE_VIRUS_VALUES = ['?', 'Yes', 'No', 'Caution'] as const
|
||||
const NSFW_VALUES = ['Yes', 'No'] as const
|
||||
|
||||
export interface Game {
|
||||
url: string
|
||||
name?: string
|
||||
[field: string]: unknown
|
||||
}
|
||||
|
||||
export interface CatalogIndex {
|
||||
total_games: number
|
||||
max_per_file: number
|
||||
last_updated: string
|
||||
files: { name: string; count: number }[]
|
||||
}
|
||||
|
||||
export interface CountPoint {
|
||||
date: string
|
||||
total: number
|
||||
}
|
||||
|
||||
export interface DeletedEntry {
|
||||
url: string
|
||||
name: string
|
||||
reason: string
|
||||
deleted_at: string
|
||||
}
|
||||
|
||||
/** Same text as Python's json.dumps(data, ensure_ascii=False, indent=4). */
|
||||
export function dumps(data: unknown): string {
|
||||
return JSON.stringify(data, null, 4)
|
||||
}
|
||||
|
||||
function chunkName(index: number): string {
|
||||
return `game_info_${String(index).padStart(3, '0')}.json`
|
||||
}
|
||||
|
||||
/**
|
||||
* Place `games` into chunks, keeping each game in its current chunk (port of
|
||||
* data_store.plan_chunks). New games go to the last chunk; the whole list is
|
||||
* re-sliced only when a chunk overflows or a non-last chunk under-fills.
|
||||
*/
|
||||
export function planChunks(
|
||||
oldChunks: Game[][],
|
||||
games: Game[],
|
||||
chunkSize = CHUNK_SIZE,
|
||||
minFill = MIN_FILL,
|
||||
): Game[][] {
|
||||
const byUrl = new Map<string, Game>()
|
||||
for (const g of games) if (!byUrl.has(g.url)) byUrl.set(g.url, g)
|
||||
|
||||
const placed = new Set<string>()
|
||||
let chunks: Game[][] = oldChunks.map((old) => {
|
||||
const chunk: Game[] = []
|
||||
for (const g of old) {
|
||||
const current = byUrl.get(g.url)
|
||||
if (current && !placed.has(g.url)) {
|
||||
chunk.push(current)
|
||||
placed.add(g.url)
|
||||
}
|
||||
}
|
||||
return chunk
|
||||
})
|
||||
|
||||
const fresh = [...byUrl.values()].filter((g) => !placed.has(g.url))
|
||||
if (fresh.length > 0) {
|
||||
if (chunks.length === 0) chunks.push([])
|
||||
chunks[chunks.length - 1].push(...fresh)
|
||||
}
|
||||
|
||||
chunks = chunks.filter((c) => c.length > 0)
|
||||
const needsReslice =
|
||||
chunks.some((c) => c.length > chunkSize) || chunks.slice(0, -1).some((c) => c.length < minFill)
|
||||
if (needsReslice) {
|
||||
const flat = chunks.flat()
|
||||
chunks = []
|
||||
for (let i = 0; i < flat.length; i += chunkSize) chunks.push(flat.slice(i, i + chunkSize))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
/** index.json content; `last_updated` moves only when data or layout changed. */
|
||||
export function buildIndex(
|
||||
chunks: Game[][],
|
||||
previous: Partial<CatalogIndex> | null,
|
||||
dataChanged: boolean,
|
||||
now: Date,
|
||||
): CatalogIndex {
|
||||
const files = chunks.map((c, i) => ({ name: chunkName(i + 1), count: c.length }))
|
||||
const total = files.reduce((sum, f) => sum + f.count, 0)
|
||||
const layoutChanged =
|
||||
JSON.stringify(previous?.files) !== JSON.stringify(files) || previous?.total_games !== total
|
||||
const lastUpdated =
|
||||
dataChanged || layoutChanged || !previous?.last_updated
|
||||
? now.toISOString().replace(/\.\d{3}Z$/, 'Z')
|
||||
: previous.last_updated
|
||||
return { total_games: total, max_per_file: CHUNK_SIZE, last_updated: lastUpdated, files }
|
||||
}
|
||||
|
||||
/** Date-keyed upsert; a day whose total equals the last recorded one adds no row. */
|
||||
export function upsertCountHistory(history: CountPoint[], total: number, now: Date): CountPoint[] {
|
||||
const today = now.toISOString().slice(0, 10)
|
||||
const rows = [...history].sort((a, b) => a.date.localeCompare(b.date))
|
||||
const existing = rows.find((r) => r.date === today)
|
||||
if (existing) {
|
||||
existing.total = total
|
||||
return rows
|
||||
}
|
||||
if (rows.length > 0 && rows[rows.length - 1].total === total) return rows
|
||||
return [...rows, { date: today, total }]
|
||||
}
|
||||
|
||||
/** Keep the earliest deleted_at per URL, sorted by deleted_at (dedup_deleted). */
|
||||
export function dedupDeleted(entries: DeletedEntry[]): DeletedEntry[] {
|
||||
const byUrl = new Map<string, DeletedEntry>()
|
||||
for (const e of entries) {
|
||||
if (!e.url) continue
|
||||
const current = byUrl.get(e.url)
|
||||
if (!current || e.deleted_at < current.deleted_at) byUrl.set(e.url, e)
|
||||
}
|
||||
return [...byUrl.values()].sort((a, b) => a.deleted_at.localeCompare(b.deleted_at))
|
||||
}
|
||||
|
||||
export type FieldEdit = Partial<Record<EditableField, string>>
|
||||
|
||||
/** Validate an admin edit; returns an error message or null. */
|
||||
export function validateEdit(edit: Record<string, unknown>): string | null {
|
||||
for (const [key, value] of Object.entries(edit)) {
|
||||
if (!(EDITABLE_FIELDS as readonly string[]).includes(key)) return `field '${key}' is not editable`
|
||||
if (typeof value !== 'string') return `field '${key}' must be a string`
|
||||
if (key === 'safe_virus' && !(SAFE_VIRUS_VALUES as readonly string[]).includes(value))
|
||||
return `safe_virus must be one of ${SAFE_VIRUS_VALUES.join(', ')}`
|
||||
if (key === 'nsfw' && !(NSFW_VALUES as readonly string[]).includes(value))
|
||||
return `nsfw must be one of ${NSFW_VALUES.join(', ')}`
|
||||
if (key === 'notes' && value.length > 2000) return 'notes is limited to 2000 characters'
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export interface CatalogFiles {
|
||||
chunks: Game[][]
|
||||
index: CatalogIndex | null
|
||||
history: CountPoint[]
|
||||
deleted: DeletedEntry[]
|
||||
}
|
||||
|
||||
export interface CatalogChange {
|
||||
/** path → new file text (only files whose text changed). */
|
||||
writes: Map<string, string>
|
||||
/** paths to delete (chunks that disappeared). */
|
||||
deletes: string[]
|
||||
summary: { edited: number; removed: number }
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply edits and removals to the catalog as read at one commit. Produces
|
||||
* exactly the file texts the Python pipeline would write for the same change.
|
||||
*/
|
||||
export function applyAdminChange(
|
||||
files: CatalogFiles,
|
||||
edits: Map<string, FieldEdit>,
|
||||
removals: { url: string; reason: string }[],
|
||||
now: Date,
|
||||
sizes: { chunkSize: number; minFill: number } = { chunkSize: CHUNK_SIZE, minFill: MIN_FILL },
|
||||
): CatalogChange {
|
||||
const games = files.chunks.flat().map((g) => ({ ...g }))
|
||||
let edited = 0
|
||||
for (const g of games) {
|
||||
const edit = edits.get(g.url)
|
||||
if (!edit) continue
|
||||
let changed = false
|
||||
for (const [key, value] of Object.entries(edit)) {
|
||||
if (g[key] !== value) {
|
||||
g[key] = value
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if (changed) edited++
|
||||
}
|
||||
|
||||
const removeUrls = new Map(removals.map((r) => [r.url, r.reason]))
|
||||
const stamp = now.toISOString().replace(/\.\d{3}Z$/, 'Z')
|
||||
const deletedEntries: DeletedEntry[] = []
|
||||
const kept = games.filter((g) => {
|
||||
const reason = removeUrls.get(g.url)
|
||||
if (reason === undefined) return true
|
||||
deletedEntries.push({ url: g.url, name: String(g.name ?? 'N/A'), reason, deleted_at: stamp })
|
||||
return false
|
||||
})
|
||||
|
||||
const writes = new Map<string, string>()
|
||||
const newChunks = planChunks(files.chunks, kept, sizes.chunkSize, sizes.minFill)
|
||||
const oldTexts = files.chunks.map((c) => dumps(c))
|
||||
let dataChanged = false
|
||||
newChunks.forEach((chunk, i) => {
|
||||
const text = dumps(chunk)
|
||||
if (oldTexts[i] !== text) {
|
||||
writes.set(`data_game/${chunkName(i + 1)}`, text)
|
||||
dataChanged = true
|
||||
}
|
||||
})
|
||||
const deletes: string[] = []
|
||||
for (let i = newChunks.length; i < files.chunks.length; i++) {
|
||||
deletes.push(`data_game/${chunkName(i + 1)}`)
|
||||
dataChanged = true
|
||||
}
|
||||
|
||||
const index = buildIndex(newChunks, files.index, dataChanged, now)
|
||||
if (!files.index || dumps(files.index) !== dumps(index)) writes.set('data_game/index.json', dumps(index))
|
||||
const history = upsertCountHistory(files.history, index.total_games, now)
|
||||
if (dumps(history) !== dumps(files.history)) writes.set('data_game/count_history.json', dumps(history))
|
||||
if (deletedEntries.length > 0) {
|
||||
writes.set('scripts/deleted_games.json', dumps(dedupDeleted([...files.deleted, ...deletedEntries])))
|
||||
}
|
||||
return { writes, deletes, summary: { edited, removed: deletedEntries.length } }
|
||||
}
|
||||
@@ -10,10 +10,17 @@ export interface DataSource {
|
||||
origin: string
|
||||
}
|
||||
|
||||
interface DeletedInfo {
|
||||
reason: string
|
||||
deleted_at: string
|
||||
}
|
||||
|
||||
let urlMap: Map<string, string> | undefined
|
||||
let deletedMap: Map<string, DeletedInfo> | undefined
|
||||
|
||||
export function resetDataCache(): void {
|
||||
urlMap = undefined
|
||||
deletedMap = undefined
|
||||
}
|
||||
|
||||
async function fetchJson<T>(src: DataSource, file: string): Promise<T> {
|
||||
@@ -46,3 +53,17 @@ export function catalogUrls(src: DataSource): Promise<Map<string, string>> {
|
||||
)
|
||||
}
|
||||
|
||||
/** Removed game URL → why and when (scripts/deleted_games.json). */
|
||||
export function deletedGames(src: DataSource): Promise<Map<string, DeletedInfo>> {
|
||||
return cachedLoad(
|
||||
() => deletedMap,
|
||||
(v) => (deletedMap = v),
|
||||
async () =>
|
||||
new Map(
|
||||
(await fetchJson<(DeletedInfo & { url: string })[]>(src, 'deleted_games.json')).map((e) => [
|
||||
e.url,
|
||||
{ reason: e.reason, deleted_at: e.deleted_at },
|
||||
]),
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Scheduled discovery of new free games from itch.io RSS feeds. Each run
|
||||
* polls ONE feed (the one polled longest ago that is not backing off), so
|
||||
* itch.io sees at most one request per cron tick. Items go to the review
|
||||
* queue as `source = 'rss'`; nothing is added to the catalog without a
|
||||
* maintainer's approval.
|
||||
*/
|
||||
import { canonicalize } from './canonical'
|
||||
import { catalogUrls, deletedGames, type DataSource } from './data'
|
||||
import { autoFlags, classify, isoSeconds, type FeedState, type QueueStore } from './queue'
|
||||
|
||||
interface Feed {
|
||||
id: string
|
||||
url: string
|
||||
genre: string | null
|
||||
}
|
||||
|
||||
export const FEEDS: readonly Feed[] = [
|
||||
{ id: 'free', url: 'https://itch.io/games/new-and-popular/free.xml', genre: null },
|
||||
{
|
||||
id: 'visual-novel',
|
||||
url: 'https://itch.io/games/new-and-popular/free/genre-visual-novel.xml',
|
||||
genre: 'Visual Novel',
|
||||
},
|
||||
]
|
||||
|
||||
const BACKOFF_MS = 6 * 3_600_000
|
||||
const MAX_ITEMS = 60
|
||||
|
||||
export interface FeedItem {
|
||||
url: string
|
||||
title: string
|
||||
image: string | null
|
||||
genre: string | null
|
||||
free: boolean
|
||||
}
|
||||
|
||||
// Bracket tags itch.io appends to <title> that are not genres.
|
||||
const NOT_GENRE = new Set(['free', 'windows', 'macos', 'linux', 'android', 'ios', 'html5', 'web', 'browser'])
|
||||
|
||||
function decode(text: string): string {
|
||||
return text
|
||||
.replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, '$1')
|
||||
.replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(parseInt(n, 16)))
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/&/g, '&')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function tag(block: string, name: string): string | null {
|
||||
const m = new RegExp(`<${name}>([\\s\\S]*?)</${name}>`).exec(block)
|
||||
return m ? decode(m[1]) : null
|
||||
}
|
||||
|
||||
/** Items of an itch.io browse feed (`<item>` with plainTitle / price / imageurl). */
|
||||
export function parseFeed(xml: string): FeedItem[] {
|
||||
const items: FeedItem[] = []
|
||||
for (const [, block] of xml.matchAll(/<item>([\s\S]*?)<\/item>/g)) {
|
||||
const url = tag(block, 'link') ?? tag(block, 'guid')
|
||||
if (!url) continue
|
||||
const rawTitle = tag(block, 'title') ?? ''
|
||||
const brackets = [...rawTitle.matchAll(/\[([^\]]+)\]/g)].map((m) => m[1].trim())
|
||||
const genre = brackets.find((b) => !NOT_GENRE.has(b.toLowerCase())) ?? null
|
||||
const price = tag(block, 'price') ?? ''
|
||||
const image = tag(block, 'imageurl')
|
||||
items.push({
|
||||
url,
|
||||
title: tag(block, 'plainTitle') ?? rawTitle.replace(/\s*\[[^\]]*\]/g, '').trim(),
|
||||
image: image?.startsWith('https://img.itch.zone/') ? image : null,
|
||||
genre,
|
||||
free: /^[^0-9]*0+(?:[.,]0+)?$/.test(price) || brackets.some((b) => b.toLowerCase() === 'free'),
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
/** The feed to poll now: not backing off, least recently polled first. */
|
||||
export function pickFeed(feeds: readonly Feed[], states: Map<string, FeedState>, now: Date): Feed | null {
|
||||
const stamp = isoSeconds(now)
|
||||
const ready = feeds.filter((f) => (states.get(f.id)?.backoff_until ?? '') <= stamp)
|
||||
ready.sort((a, b) => (states.get(a.id)?.last_polled_at ?? '').localeCompare(states.get(b.id)?.last_polled_at ?? ''))
|
||||
return ready[0] ?? null
|
||||
}
|
||||
|
||||
function retryAfterMs(header: string | null, now: Date): number {
|
||||
if (!header) return 0
|
||||
const seconds = Number(header)
|
||||
if (Number.isFinite(seconds)) return seconds * 1000
|
||||
const at = Date.parse(header)
|
||||
return Number.isNaN(at) ? 0 : at - now.getTime()
|
||||
}
|
||||
|
||||
export interface DiscoverDeps {
|
||||
store: QueueStore
|
||||
data: DataSource
|
||||
fetch: typeof fetch
|
||||
userAgent: string
|
||||
now: Date
|
||||
}
|
||||
|
||||
export interface DiscoverResult {
|
||||
feed: string | null
|
||||
status: number | null
|
||||
items: number
|
||||
added: number
|
||||
}
|
||||
|
||||
export async function discover(deps: DiscoverDeps): Promise<DiscoverResult> {
|
||||
const { store, now } = deps
|
||||
const feed = pickFeed(FEEDS, await store.feeds(), now)
|
||||
if (!feed) return { feed: null, status: null, items: 0, added: 0 }
|
||||
|
||||
const save = (status: number, backoffMs = 0) =>
|
||||
store.saveFeed({
|
||||
feed: feed.id,
|
||||
last_polled_at: isoSeconds(now),
|
||||
backoff_until: backoffMs > 0 ? isoSeconds(new Date(now.getTime() + backoffMs)) : null,
|
||||
last_status: status,
|
||||
})
|
||||
|
||||
let res: Response
|
||||
try {
|
||||
res = await deps.fetch(feed.url, {
|
||||
headers: { 'User-Agent': deps.userAgent, Accept: 'application/rss+xml, application/xml;q=0.9' },
|
||||
})
|
||||
} catch {
|
||||
await save(0)
|
||||
return { feed: feed.id, status: 0, items: 0, added: 0 }
|
||||
}
|
||||
if (res.status === 429 || res.status === 503) {
|
||||
res.body?.cancel().catch(() => {})
|
||||
await save(res.status, Math.max(BACKOFF_MS, retryAfterMs(res.headers.get('retry-after'), now)))
|
||||
return { feed: feed.id, status: res.status, items: 0, added: 0 }
|
||||
}
|
||||
if (!res.ok) {
|
||||
res.body?.cancel().catch(() => {})
|
||||
await save(res.status)
|
||||
return { feed: feed.id, status: res.status, items: 0, added: 0 }
|
||||
}
|
||||
|
||||
const items = parseFeed(await res.text())
|
||||
.filter((i) => i.free)
|
||||
.slice(0, MAX_ITEMS)
|
||||
const canonical = items.map((i) => canonicalize(i.url)).filter((u): u is string => u !== null)
|
||||
const [catalog, deleted, known] = await Promise.all([
|
||||
catalogUrls(deps.data),
|
||||
deletedGames(deps.data),
|
||||
store.statuses(canonical),
|
||||
])
|
||||
const results = classify(
|
||||
items.map((i) => i.url),
|
||||
{ catalog: new Set(catalog.keys()), deleted, known },
|
||||
)
|
||||
const fresh = results.flatMap((r, i) =>
|
||||
r.store && r.canonical
|
||||
? [
|
||||
{
|
||||
url: r.canonical,
|
||||
source: 'rss' as const,
|
||||
title: items[i].title.slice(0, 200),
|
||||
image_url: items[i].image,
|
||||
genre_hint: items[i].genre ?? feed.genre,
|
||||
flags: autoFlags(r.outcome, items[i].title),
|
||||
submitter: `rss:${feed.id}`,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
)
|
||||
await store.insert(fresh, now)
|
||||
await save(200)
|
||||
return { feed: feed.id, status: 200, items: items.length, added: fresh.length }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Secrets live only in Cloudflare (`wrangler secret put`, or .dev.vars for
|
||||
* `wrangler dev`) and are typed here instead of in the generated
|
||||
* worker-configuration.d.ts (`npm run cf:types` ignores .dev.vars, so the
|
||||
* committed file never depends on who generated it). They are optional at
|
||||
* runtime: a route whose configuration is incomplete answers 503.
|
||||
*/
|
||||
interface Secrets {
|
||||
GH_APP_PRIVATE_KEY?: string // PKCS#8 PEM of the GitHub App
|
||||
ADMIN_EMAILS?: string // comma-separated maintainer emails
|
||||
TURNSTILE_SECRET?: string
|
||||
}
|
||||
|
||||
export type WorkerEnv = Env & Secrets
|
||||
|
||||
/** Names of the settings in `keys` that are empty or unset. */
|
||||
export function missingConfig<K extends keyof WorkerEnv>(env: WorkerEnv, keys: readonly K[]): K[] {
|
||||
return keys.filter((k) => !env[k])
|
||||
}
|
||||
|
||||
export const ACCESS_CONFIG = ['ACCESS_TEAM_DOMAIN', 'SITE_ORIGIN'] as const
|
||||
export const GITHUB_CONFIG = ['GITHUB_REPO', 'GH_APP_ID', 'GH_APP_INSTALLATION_ID', 'GH_APP_PRIVATE_KEY'] as const
|
||||
@@ -0,0 +1,129 @@
|
||||
import { exportPKCS8, generateKeyPair, jwtVerify } from 'jose'
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
ConflictError,
|
||||
commit,
|
||||
commitWithRetry,
|
||||
readFile,
|
||||
resetTokenCache,
|
||||
utf8ToBase64,
|
||||
type GitHubEnv,
|
||||
} from './github'
|
||||
|
||||
let env: GitHubEnv
|
||||
let publicKey: CryptoKey
|
||||
|
||||
beforeAll(async () => {
|
||||
const pair = await generateKeyPair('RS256', { extractable: true })
|
||||
publicKey = pair.publicKey
|
||||
env = {
|
||||
GITHUB_REPO: 'owner/repo',
|
||||
GH_APP_ID: '12345',
|
||||
GH_APP_INSTALLATION_ID: '678',
|
||||
GH_APP_PRIVATE_KEY: await exportPKCS8(pair.privateKey),
|
||||
}
|
||||
})
|
||||
|
||||
type Handler = (url: string, init: RequestInit) => Response | Promise<Response>
|
||||
|
||||
function mockFetch(handler: Handler) {
|
||||
const calls: { url: string; init: RequestInit }[] = []
|
||||
vi.stubGlobal('fetch', async (input: string, init: RequestInit = {}) => {
|
||||
calls.push({ url: String(input), init })
|
||||
return handler(String(input), init)
|
||||
})
|
||||
return calls
|
||||
}
|
||||
|
||||
const tokenResponse = () =>
|
||||
Response.json({ token: 'inst-token', expires_at: new Date(Date.now() + 3600_000).toISOString() })
|
||||
|
||||
beforeEach(() => resetTokenCache())
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
describe('utf8ToBase64', () => {
|
||||
it('encodes non-ASCII text as UTF-8', () => {
|
||||
expect(atob(utf8ToBase64('Trò chơi'))).toBe(String.fromCharCode(...new TextEncoder().encode('Trò chơi')))
|
||||
expect(utf8ToBase64('x'.repeat(100_000))).toHaveLength(Math.ceil(100_000 / 3) * 4)
|
||||
})
|
||||
})
|
||||
|
||||
describe('GitHub App auth + reads', () => {
|
||||
it('signs an app JWT, caches the installation token and reads raw files', async () => {
|
||||
const calls = mockFetch(async (url) => {
|
||||
if (url.endsWith('/access_tokens')) return tokenResponse()
|
||||
if (url.includes('/contents/data_game/index.json')) return new Response('{"total_games": 1}')
|
||||
return new Response(null, { status: 404 })
|
||||
})
|
||||
expect(await readFile(env, 'data_game/index.json', 'abc')).toBe('{"total_games": 1}')
|
||||
expect(await readFile(env, 'missing.json', 'abc')).toBeNull()
|
||||
|
||||
const tokenCalls = calls.filter((c) => c.url.endsWith('/access_tokens'))
|
||||
expect(tokenCalls).toHaveLength(1) // cached for the second read
|
||||
const jwt = String((tokenCalls[0].init.headers as Record<string, string>).Authorization).slice(7)
|
||||
const { payload } = await jwtVerify(jwt, publicKey)
|
||||
expect(payload.iss).toBe('12345')
|
||||
|
||||
const read = calls.find((c) => c.url.includes('index.json'))!
|
||||
const headers = read.init.headers as Record<string, string>
|
||||
expect(headers.Accept).toBe('application/vnd.github.raw+json')
|
||||
expect(headers.Authorization).toBe('Bearer inst-token')
|
||||
expect(headers['User-Agent']).toBeTruthy()
|
||||
expect(read.url).toContain('ref=abc')
|
||||
})
|
||||
})
|
||||
|
||||
describe('commit', () => {
|
||||
it('sends createCommitOnBranch with the expected head and base64 contents', async () => {
|
||||
const calls = mockFetch(async (url) => {
|
||||
if (url.endsWith('/access_tokens')) return tokenResponse()
|
||||
return Response.json({ data: { createCommitOnBranch: { commit: { oid: 'new-oid' } } } })
|
||||
})
|
||||
const oid = await commit(env, {
|
||||
expectedHeadOid: 'head1',
|
||||
headline: 'admin: test',
|
||||
writes: new Map([['scripts/temp_link.json', '[]']]),
|
||||
deletes: ['data_game/game_info_009.json'],
|
||||
})
|
||||
expect(oid).toBe('new-oid')
|
||||
const body = JSON.parse(String(calls.at(-1)!.init.body))
|
||||
const input = body.variables.input
|
||||
expect(input.expectedHeadOid).toBe('head1')
|
||||
expect(input.branch).toEqual({ repositoryNameWithOwner: 'owner/repo', branchName: 'main' })
|
||||
expect(input.fileChanges.additions).toEqual([{ path: 'scripts/temp_link.json', contents: btoa('[]') }])
|
||||
expect(input.fileChanges.deletions).toEqual([{ path: 'data_game/game_info_009.json' }])
|
||||
})
|
||||
|
||||
it('retries on a moved head, then succeeds', async () => {
|
||||
let heads = 0
|
||||
let commits = 0
|
||||
mockFetch(async (url) => {
|
||||
if (url.endsWith('/access_tokens')) return tokenResponse()
|
||||
if (url.includes('/git/ref/heads/main')) return Response.json({ object: { sha: `head${++heads}` } })
|
||||
commits++
|
||||
if (commits === 1) {
|
||||
return Response.json({ errors: [{ message: 'Expected branch to point to "head1" but it did not.' }] })
|
||||
}
|
||||
return Response.json({ data: { createCommitOnBranch: { commit: { oid: 'ok' } } } })
|
||||
})
|
||||
const seen: string[] = []
|
||||
const oid = await commitWithRetry(env, async (head) => {
|
||||
seen.push(head)
|
||||
return { headline: 'x', writes: new Map([['a.json', '{}']]) }
|
||||
})
|
||||
expect(oid).toBe('ok')
|
||||
expect(seen).toEqual(['head1', 'head2'])
|
||||
})
|
||||
|
||||
it('gives up after the attempt limit and skips no-op changes', async () => {
|
||||
mockFetch(async (url) => {
|
||||
if (url.endsWith('/access_tokens')) return tokenResponse()
|
||||
if (url.includes('/git/ref/heads/main')) return Response.json({ object: { sha: 'h' } })
|
||||
return Response.json({ errors: [{ message: 'expectedHeadOid mismatch' }] })
|
||||
})
|
||||
await expect(
|
||||
commitWithRetry(env, async () => ({ headline: 'x', writes: new Map([['a', 'b']]) }), 2),
|
||||
).rejects.toBeInstanceOf(ConflictError)
|
||||
expect(await commitWithRetry(env, async () => null)).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* GitHub access for admin writes, as a GitHub App (no personal token).
|
||||
*
|
||||
* - Installation token: RS256 app JWT → POST /app/installations/:id/access_tokens,
|
||||
* cached per isolate until 5 minutes before expiry.
|
||||
* - Reads: the Contents API with the raw media type at an exact commit
|
||||
* (no base64 / UTF-8 mojibake — CLAUDE.md gotcha #8).
|
||||
* - Writes: GraphQL `createCommitOnBranch` with `expectedHeadOid`. Commits made
|
||||
* by an app through the API are signed by GitHub ("Verified"), and the
|
||||
* expected head makes a write fail instead of silently overwriting a commit
|
||||
* another writer (pipeline, extension) pushed in between; callers retry.
|
||||
*/
|
||||
import { SignJWT, importPKCS8 } from 'jose'
|
||||
|
||||
export interface GitHubEnv {
|
||||
GITHUB_REPO: string // "owner/name"
|
||||
GH_APP_ID: string
|
||||
GH_APP_INSTALLATION_ID: string
|
||||
GH_APP_PRIVATE_KEY: string // PKCS#8 PEM
|
||||
}
|
||||
|
||||
const API = 'https://api.github.com'
|
||||
const BRANCH = 'main'
|
||||
const BASE_HEADERS = {
|
||||
Accept: 'application/vnd.github+json',
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
// GitHub rejects API requests without a User-Agent.
|
||||
'User-Agent': 'freeitchgames-admin',
|
||||
}
|
||||
|
||||
export class ConflictError extends Error {}
|
||||
|
||||
let cached: { token: string; expiresAt: number } | undefined
|
||||
|
||||
export function resetTokenCache(): void {
|
||||
cached = undefined
|
||||
}
|
||||
|
||||
async function appJwt(env: GitHubEnv): Promise<string> {
|
||||
const key = await importPKCS8(env.GH_APP_PRIVATE_KEY, 'RS256')
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
return new SignJWT({})
|
||||
.setProtectedHeader({ alg: 'RS256' })
|
||||
.setIssuer(env.GH_APP_ID)
|
||||
.setIssuedAt(now - 60) // clock skew allowance
|
||||
.setExpirationTime(now + 9 * 60)
|
||||
.sign(key)
|
||||
}
|
||||
|
||||
async function installationToken(env: GitHubEnv): Promise<string> {
|
||||
if (cached && cached.expiresAt - Date.now() > 5 * 60_000) return cached.token
|
||||
const res = await fetch(`${API}/app/installations/${env.GH_APP_INSTALLATION_ID}/access_tokens`, {
|
||||
method: 'POST',
|
||||
headers: { ...BASE_HEADERS, Authorization: `Bearer ${await appJwt(env)}` },
|
||||
})
|
||||
if (!res.ok) throw new Error(`GitHub App token: HTTP ${res.status}`)
|
||||
const body = (await res.json()) as { token: string; expires_at: string }
|
||||
cached = { token: body.token, expiresAt: Date.parse(body.expires_at) }
|
||||
return body.token
|
||||
}
|
||||
|
||||
async function gh(env: GitHubEnv, path: string, init: RequestInit = {}): Promise<Response> {
|
||||
const token = await installationToken(env)
|
||||
return fetch(`${API}${path}`, {
|
||||
...init,
|
||||
headers: { ...BASE_HEADERS, Authorization: `Bearer ${token}`, ...(init.headers ?? {}) },
|
||||
})
|
||||
}
|
||||
|
||||
async function headOid(env: GitHubEnv): Promise<string> {
|
||||
const res = await gh(env, `/repos/${env.GITHUB_REPO}/git/ref/heads/${BRANCH}`)
|
||||
if (!res.ok) throw new Error(`read ref: HTTP ${res.status}`)
|
||||
return ((await res.json()) as { object: { sha: string } }).object.sha
|
||||
}
|
||||
|
||||
/** File text at a commit, or null when it does not exist. */
|
||||
export async function readFile(env: GitHubEnv, path: string, ref: string): Promise<string | null> {
|
||||
const res = await gh(env, `/repos/${env.GITHUB_REPO}/contents/${path}?ref=${ref}`, {
|
||||
headers: { Accept: 'application/vnd.github.raw+json' },
|
||||
})
|
||||
if (res.status === 404) return null
|
||||
if (!res.ok) throw new Error(`read ${path}: HTTP ${res.status}`)
|
||||
return res.text()
|
||||
}
|
||||
|
||||
/** Names of the files directly in a directory at a commit. */
|
||||
export async function listDir(env: GitHubEnv, path: string, ref: string): Promise<string[]> {
|
||||
const res = await gh(env, `/repos/${env.GITHUB_REPO}/contents/${path}?ref=${ref}`)
|
||||
if (!res.ok) throw new Error(`list ${path}: HTTP ${res.status}`)
|
||||
const items = (await res.json()) as { name: string; type: string }[]
|
||||
return items.filter((i) => i.type === 'file').map((i) => i.name)
|
||||
}
|
||||
|
||||
/** Base64 of the UTF-8 bytes of `text` (chunked: large files overflow apply()). */
|
||||
export function utf8ToBase64(text: string): string {
|
||||
const bytes = new TextEncoder().encode(text)
|
||||
let binary = ''
|
||||
for (let i = 0; i < bytes.length; i += 0x8000) {
|
||||
binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000))
|
||||
}
|
||||
return btoa(binary)
|
||||
}
|
||||
|
||||
export interface CommitInput {
|
||||
expectedHeadOid: string
|
||||
headline: string
|
||||
body?: string
|
||||
writes: Map<string, string>
|
||||
deletes?: string[]
|
||||
}
|
||||
|
||||
/** One atomic, GitHub-signed commit on main. Throws ConflictError if main moved. */
|
||||
export async function commit(env: GitHubEnv, input: CommitInput): Promise<string> {
|
||||
const query = `mutation($input: CreateCommitOnBranchInput!) {
|
||||
createCommitOnBranch(input: $input) { commit { oid } }
|
||||
}`
|
||||
const variables = {
|
||||
input: {
|
||||
branch: { repositoryNameWithOwner: env.GITHUB_REPO, branchName: BRANCH },
|
||||
expectedHeadOid: input.expectedHeadOid,
|
||||
message: { headline: input.headline, ...(input.body ? { body: input.body } : {}) },
|
||||
fileChanges: {
|
||||
additions: [...input.writes].map(([path, text]) => ({ path, contents: utf8ToBase64(text) })),
|
||||
deletions: (input.deletes ?? []).map((path) => ({ path })),
|
||||
},
|
||||
},
|
||||
}
|
||||
const res = await gh(env, '/graphql', { method: 'POST', body: JSON.stringify({ query, variables }) })
|
||||
const body = (await res.json()) as {
|
||||
data?: { createCommitOnBranch?: { commit: { oid: string } } }
|
||||
errors?: { message: string }[]
|
||||
}
|
||||
const errors = body.errors ?? []
|
||||
if (errors.some((e) => /expected|head|oid/i.test(e.message))) {
|
||||
throw new ConflictError(errors.map((e) => e.message).join('; '))
|
||||
}
|
||||
const oid = body.data?.createCommitOnBranch?.commit.oid
|
||||
if (!res.ok || errors.length > 0 || !oid) {
|
||||
throw new Error(`commit: HTTP ${res.status} ${errors.map((e) => e.message).join('; ')}`)
|
||||
}
|
||||
return oid
|
||||
}
|
||||
|
||||
/**
|
||||
* Read → change → commit, retried when another writer moved main meanwhile.
|
||||
* `build` receives the head it must read from and returns the commit to make
|
||||
* (or null when there is nothing to change).
|
||||
*/
|
||||
export async function commitWithRetry(
|
||||
env: GitHubEnv,
|
||||
build: (head: string) => Promise<Omit<CommitInput, 'expectedHeadOid'> | null>,
|
||||
attempts = 3,
|
||||
): Promise<string | null> {
|
||||
for (let i = 1; ; i++) {
|
||||
const head = await headOid(env)
|
||||
const change = await build(head)
|
||||
if (!change) return null
|
||||
try {
|
||||
return await commit(env, { ...change, expectedHeadOid: head })
|
||||
} catch (e) {
|
||||
if (!(e instanceof ConflictError) || i >= attempts) throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,3 +17,48 @@ export function json(body: unknown, status = 200, headers: Record<string, string
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/** A request the handler rejects; `status` + `error` become the JSON response. */
|
||||
export class HttpError extends Error {
|
||||
readonly status: number
|
||||
readonly error: string
|
||||
|
||||
constructor(status: number, error: string, message?: string) {
|
||||
super(message ?? error)
|
||||
this.status = status
|
||||
this.error = error
|
||||
}
|
||||
}
|
||||
|
||||
export function errorResponse(e: unknown): Response {
|
||||
if (e instanceof HttpError) {
|
||||
return json({ error: e.error, ...(e.message !== e.error ? { message: e.message } : {}) }, e.status)
|
||||
}
|
||||
console.error(e)
|
||||
return json({ error: 'internal_error' }, 500)
|
||||
}
|
||||
|
||||
/** Parse a JSON request body of at most `maxBytes`. */
|
||||
export async function readJson(request: Request, maxBytes: number): Promise<unknown> {
|
||||
if (!(request.headers.get('content-type') ?? '').includes('application/json')) {
|
||||
throw new HttpError(415, 'unsupported_media_type', 'Send application/json')
|
||||
}
|
||||
const declared = Number(request.headers.get('content-length') ?? '0')
|
||||
if (declared > maxBytes) throw new HttpError(413, 'payload_too_large')
|
||||
const bytes = new Uint8Array(await request.arrayBuffer())
|
||||
if (bytes.byteLength > maxBytes) throw new HttpError(413, 'payload_too_large')
|
||||
try {
|
||||
return JSON.parse(new TextDecoder().decode(bytes))
|
||||
} catch {
|
||||
throw new HttpError(400, 'invalid_json')
|
||||
}
|
||||
}
|
||||
|
||||
export async function sha256Hex(text: string): Promise<string> {
|
||||
const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text))
|
||||
return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
|
||||
export function isObject(v: unknown): v is Record<string, unknown> {
|
||||
return typeof v === 'object' && v !== null && !Array.isArray(v)
|
||||
}
|
||||
+67
-7
@@ -3,16 +3,37 @@
|
||||
* served by the assets layer without invoking this script. It runs for the
|
||||
* paths in wrangler.jsonc `assets.run_worker_first`, and for every request
|
||||
* that matches no asset (`not_found_handling: "none"`, see spa.ts).
|
||||
*
|
||||
* /img/* resized covers (img.ts)
|
||||
* /api/suggest public suggestion form (suggest.ts)
|
||||
* /api/ingest browser extension, Access service token (ingest.ts)
|
||||
* /api/admin/* maintainer API, Access + JWT check (admin.ts)
|
||||
* /admin, /admin/* admin app shell, maintainer only (admin.ts)
|
||||
*
|
||||
* Cron: one RSS feed per run into the review queue, then queue bookkeeping.
|
||||
*/
|
||||
import { handleImg } from './img'
|
||||
import { handleAdminApi, serveAdminApp } from './admin'
|
||||
import { catalogUrls } from './data'
|
||||
import { discover } from './discover'
|
||||
import type { WorkerEnv } from './env'
|
||||
import { json } from './http'
|
||||
import { handleImg } from './img'
|
||||
import { accessService, handleIngest } from './ingest'
|
||||
import { QueueStore } from './queue'
|
||||
import { handleMiss } from './spa'
|
||||
import { handleSuggest } from './suggest'
|
||||
|
||||
function queueStore(env: WorkerEnv): QueueStore | null {
|
||||
return env.DB ? new QueueStore(env.DB) : null
|
||||
}
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx): Promise<Response> {
|
||||
async fetch(request, env: WorkerEnv, ctx): Promise<Response> {
|
||||
const url = new URL(request.url)
|
||||
const path = url.pathname
|
||||
const data = { assets: env.ASSETS, origin: url.origin }
|
||||
|
||||
if (url.pathname.startsWith('/img/')) {
|
||||
if (path.startsWith('/img/')) {
|
||||
return handleImg(request, url, {
|
||||
assets: env.ASSETS,
|
||||
bucket: env.THUMBS,
|
||||
@@ -21,10 +42,49 @@ export default {
|
||||
waitUntil: (p) => ctx.waitUntil(p),
|
||||
})
|
||||
}
|
||||
|
||||
if (url.pathname === '/api/health') return json({ ok: true })
|
||||
if (url.pathname.startsWith('/api/')) return json({ error: 'not_found' }, 404)
|
||||
if (path === '/api/health') return json({ ok: true })
|
||||
if (path === '/api/suggest') {
|
||||
return handleSuggest(request, env, {
|
||||
store: queueStore(env),
|
||||
data,
|
||||
fetch: (input, init) => fetch(input, init),
|
||||
limiter: env.RL_SUGGEST,
|
||||
})
|
||||
}
|
||||
if (path === '/api/ingest') {
|
||||
return handleIngest(request, {
|
||||
store: queueStore(env),
|
||||
data,
|
||||
limiter: env.RL_INGEST,
|
||||
authenticate: accessService(env),
|
||||
})
|
||||
}
|
||||
if (path.startsWith('/api/admin/')) return handleAdminApi(request, url, env, queueStore(env))
|
||||
if (path.startsWith('/api/')) return json({ error: 'not_found' }, 404)
|
||||
if (path === '/admin' || path.startsWith('/admin/')) return serveAdminApp(request, url, env)
|
||||
|
||||
return handleMiss(request, url, env.ASSETS)
|
||||
},
|
||||
} satisfies ExportedHandler<Env>
|
||||
|
||||
async scheduled(_controller, env: WorkerEnv): Promise<void> {
|
||||
const store = queueStore(env)
|
||||
if (!store) return
|
||||
const data = { assets: env.ASSETS, origin: env.SITE_ORIGIN }
|
||||
const now = new Date()
|
||||
try {
|
||||
const found = await discover({
|
||||
store,
|
||||
data,
|
||||
fetch: (input, init) => fetch(input, init),
|
||||
userAgent: env.BOT_UA,
|
||||
now,
|
||||
})
|
||||
console.log('discover', found)
|
||||
} catch (e) {
|
||||
console.error('discover failed', e)
|
||||
}
|
||||
const catalog = new Set((await catalogUrls(data)).keys())
|
||||
console.log('reconcile', await store.reconcile(catalog, now))
|
||||
await store.purge(now)
|
||||
},
|
||||
} satisfies ExportedHandler<WorkerEnv>
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* POST /api/ingest — machine submissions (the browser extension) into the
|
||||
* review queue. Replaces the extension's direct commits with a GitHub PAT.
|
||||
*
|
||||
* Auth: a Cloudflare Access service token of the "Ingest" application
|
||||
* (the client sends CF-Access-Client-Id/Secret; Access adds the JWT that
|
||||
* requireService verifies). The submitter recorded is the token's client
|
||||
* id, never something from the body.
|
||||
*
|
||||
* Body (≤ 64 KB, ≤ 150 items), with a required Idempotency-Key header:
|
||||
* {"items": [{"url": "...", "title"?: "...", "note"?: "..."}]} or {"urls": ["..."]}
|
||||
* A retry with the same key and body gets the stored answer; the same key
|
||||
* with a different body is refused. Kept for 7 days.
|
||||
*/
|
||||
import { AccessError, requireService, type AccessEnv } from './access'
|
||||
import { canonicalize } from './canonical'
|
||||
import { catalogUrls, deletedGames, type DataSource } from './data'
|
||||
import { ACCESS_CONFIG, missingConfig, type WorkerEnv } from './env'
|
||||
import { HttpError, errorResponse, isObject, json, sha256Hex } from './http'
|
||||
import { autoFlags, classify, type QueueStore } from './queue'
|
||||
|
||||
const MAX_BYTES = 64 * 1024
|
||||
const MAX_ITEMS = 150
|
||||
const KEY = /^[A-Za-z0-9_-]{8,128}$/
|
||||
|
||||
interface Item {
|
||||
url: string
|
||||
title: string | null
|
||||
note: string | null
|
||||
}
|
||||
|
||||
function text(value: unknown, max: number): string | null {
|
||||
return typeof value === 'string' && value.trim() ? value.trim().slice(0, max) : null
|
||||
}
|
||||
|
||||
function parseItems(body: unknown): Item[] {
|
||||
if (!isObject(body)) throw new HttpError(400, 'invalid_body')
|
||||
const raw = Array.isArray(body.items) ? body.items : Array.isArray(body.urls) ? body.urls : null
|
||||
if (!raw || raw.length === 0) throw new HttpError(400, 'invalid_body', 'Send `items` or `urls`')
|
||||
if (raw.length > MAX_ITEMS) throw new HttpError(413, 'too_many_items', `At most ${MAX_ITEMS} items`)
|
||||
return raw.map((entry: unknown) => {
|
||||
if (typeof entry === 'string') return { url: entry, title: null, note: null }
|
||||
if (isObject(entry) && typeof entry.url === 'string') {
|
||||
return { url: entry.url, title: text(entry.title, 200), note: text(entry.note, 500) }
|
||||
}
|
||||
throw new HttpError(400, 'invalid_item')
|
||||
})
|
||||
}
|
||||
|
||||
export interface IngestDeps {
|
||||
store: QueueStore | null
|
||||
data: DataSource
|
||||
limiter: Pick<RateLimit, 'limit'> | undefined
|
||||
/** Verifies the Access JWT; returns the service-token client id. */
|
||||
authenticate: (request: Request) => Promise<string>
|
||||
}
|
||||
|
||||
export function accessService(env: WorkerEnv) {
|
||||
return async (request: Request): Promise<string> => {
|
||||
if (missingConfig(env, [...ACCESS_CONFIG, 'ACCESS_AUD_INGEST']).length > 0) {
|
||||
throw new HttpError(503, 'not_configured')
|
||||
}
|
||||
try {
|
||||
return (await requireService(request, env as WorkerEnv & AccessEnv)).id
|
||||
} catch (e) {
|
||||
if (e instanceof AccessError) throw new HttpError(403, 'forbidden')
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleIngest(request: Request, deps: IngestDeps): Promise<Response> {
|
||||
try {
|
||||
if (request.method !== 'POST') throw new HttpError(405, 'method_not_allowed')
|
||||
const client = await deps.authenticate(request)
|
||||
if (!deps.store) throw new HttpError(503, 'not_configured')
|
||||
if (deps.limiter && !(await deps.limiter.limit({ key: client })).success) {
|
||||
throw new HttpError(429, 'rate_limited')
|
||||
}
|
||||
const key = request.headers.get('idempotency-key') ?? ''
|
||||
if (!KEY.test(key)) throw new HttpError(400, 'idempotency_key_required')
|
||||
|
||||
if (Number(request.headers.get('content-length') ?? '0') > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
|
||||
const raw = await request.text()
|
||||
if (new TextEncoder().encode(raw).byteLength > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
|
||||
const bodyHash = await sha256Hex(raw)
|
||||
|
||||
const previous = await deps.store.idempotent(client, key)
|
||||
if (previous) {
|
||||
if (previous.body_hash !== bodyHash) throw new HttpError(422, 'idempotency_key_reused')
|
||||
return json(JSON.parse(previous.response), 200, { 'Idempotent-Replayed': 'true' })
|
||||
}
|
||||
|
||||
let body: unknown
|
||||
try {
|
||||
body = JSON.parse(raw)
|
||||
} catch {
|
||||
throw new HttpError(400, 'invalid_json')
|
||||
}
|
||||
const items = parseItems(body)
|
||||
const canonical = items.map((i) => canonicalize(i.url)).filter((u): u is string => u !== null)
|
||||
const [catalog, deleted, known] = await Promise.all([
|
||||
catalogUrls(deps.data),
|
||||
deletedGames(deps.data),
|
||||
deps.store.statuses(canonical),
|
||||
])
|
||||
const results = classify(
|
||||
items.map((i) => i.url),
|
||||
{ catalog: new Set(catalog.keys()), deleted, known },
|
||||
)
|
||||
const fresh = results.flatMap((r, i) =>
|
||||
r.store && r.canonical
|
||||
? [
|
||||
{
|
||||
url: r.canonical,
|
||||
source: 'ext' as const,
|
||||
title: items[i].title,
|
||||
note: items[i].note,
|
||||
submitter: client,
|
||||
flags: autoFlags(r.outcome, items[i].title),
|
||||
},
|
||||
]
|
||||
: [],
|
||||
)
|
||||
await deps.store.insert(fresh)
|
||||
const response = JSON.stringify({
|
||||
received: fresh.length,
|
||||
results: results.map((r) => ({ url: r.input, canonical: r.canonical, outcome: r.outcome })),
|
||||
})
|
||||
await deps.store.remember(client, key, bodyHash, response)
|
||||
return json(JSON.parse(response))
|
||||
} catch (e) {
|
||||
return errorResponse(e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
-- Review queue + bookkeeping for the admin Worker. The catalog itself stays
|
||||
-- JSON in the GitHub repo; D1 only holds what is waiting for a decision.
|
||||
-- Migrations are additive only (never edit or drop in a later file).
|
||||
|
||||
-- Games proposed for the catalog (RSS discovery, the public Suggest form, the
|
||||
-- browser extension via /api/ingest, or the maintainer). `rejected` rows are
|
||||
-- kept as a permanent blocklist (URL only is needed for that).
|
||||
CREATE TABLE candidates (
|
||||
url TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL CHECK (source IN ('rss', 'form', 'ext', 'admin')),
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'rejected', 'queued', 'ingested', 'failed')),
|
||||
title TEXT,
|
||||
image_url TEXT,
|
||||
genre_hint TEXT,
|
||||
flags TEXT NOT NULL DEFAULT '[]', -- JSON array: previously_deleted, demo, nsfw_keyword
|
||||
note TEXT, -- Suggest form / extension note (<= 500 chars)
|
||||
submitter TEXT, -- service-token client id or 'form' / 'rss:<feed>'
|
||||
discovered_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
decided_at TEXT,
|
||||
decided_by TEXT
|
||||
);
|
||||
CREATE INDEX idx_candidates_status ON candidates (status, discovered_at);
|
||||
|
||||
-- Round-robin state for itch.io RSS discovery (one feed per cron run).
|
||||
CREATE TABLE feed_state (
|
||||
feed TEXT PRIMARY KEY,
|
||||
last_polled_at TEXT,
|
||||
backoff_until TEXT,
|
||||
last_status INTEGER
|
||||
);
|
||||
|
||||
-- Who changed what (admin actions, with the resulting commit).
|
||||
CREATE TABLE audit_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
actor TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
target TEXT,
|
||||
commit_sha TEXT,
|
||||
detail TEXT
|
||||
);
|
||||
|
||||
-- Replay protection for /api/ingest (Idempotency-Key header), kept 7 days.
|
||||
CREATE TABLE idempotency (
|
||||
client TEXT NOT NULL, -- service-token client id
|
||||
key TEXT NOT NULL, -- Idempotency-Key header
|
||||
body_hash TEXT NOT NULL,
|
||||
response TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
PRIMARY KEY (client, key)
|
||||
);
|
||||
CREATE INDEX idx_idempotency_created ON idempotency (created_at);
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { classify, type ClassifyContext } from './queue'
|
||||
|
||||
const ctx = (over: Partial<ClassifyContext> = {}): ClassifyContext => ({
|
||||
catalog: new Set(['https://dev.itch.io/in-catalog']),
|
||||
deleted: new Map([['https://dev.itch.io/gone', { reason: 'paid', deleted_at: '2026-06-19T03:00:00Z' }]]),
|
||||
known: new Map([
|
||||
['https://dev.itch.io/waiting', 'pending'],
|
||||
['https://dev.itch.io/approved', 'queued'],
|
||||
['https://dev.itch.io/blocked', 'rejected'],
|
||||
['https://dev.itch.io/broke', 'failed'],
|
||||
]),
|
||||
...over,
|
||||
})
|
||||
|
||||
describe('classify', () => {
|
||||
it('assigns each outcome in precedence order', () => {
|
||||
const out = classify(
|
||||
[
|
||||
'not a url',
|
||||
'https://dev.itch.io/new',
|
||||
'HTTPS://DEV.itch.io/new/', // same game after canonicalization
|
||||
'https://dev.itch.io/in-catalog',
|
||||
'https://dev.itch.io/blocked',
|
||||
'https://dev.itch.io/waiting',
|
||||
'https://dev.itch.io/approved',
|
||||
'https://dev.itch.io/gone',
|
||||
'https://dev.itch.io/broke',
|
||||
],
|
||||
ctx(),
|
||||
)
|
||||
expect(out.map((c) => [c.outcome, c.store])).toEqual([
|
||||
['invalid_url', false],
|
||||
['queued', true],
|
||||
['duplicate_in_request', false],
|
||||
['duplicate_catalog', false],
|
||||
['rejected', false],
|
||||
['duplicate_pending', false],
|
||||
['duplicate_pending', false],
|
||||
['previously_deleted', true],
|
||||
['queued', true], // failed rows are reopened
|
||||
])
|
||||
expect(out[1].canonical).toBe('https://dev.itch.io/new')
|
||||
expect(out[7].reason).toBe('paid (2026-06-19)')
|
||||
})
|
||||
|
||||
it('checks the catalog before the blocklist and the deleted log', () => {
|
||||
const url = 'https://dev.itch.io/in-catalog'
|
||||
const out = classify([url], ctx({ known: new Map([[url, 'rejected']]), deleted: new Map() }))
|
||||
expect(out[0].outcome).toBe('duplicate_catalog')
|
||||
})
|
||||
|
||||
it('keeps the caller input verbatim', () => {
|
||||
const [c] = classify([' https://dev.itch.io/new?ref=x '], ctx())
|
||||
expect(c.input).toBe(' https://dev.itch.io/new?ref=x ')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,275 @@
|
||||
/**
|
||||
* Review queue: classification of proposed URLs (pure) and the D1 store.
|
||||
*
|
||||
* Order of checks for each proposed URL:
|
||||
* invalid_url → duplicate_in_request → duplicate_catalog → rejected (blocklist)
|
||||
* → duplicate_pending (already waiting / queued) → previously_deleted
|
||||
* (kept for review, flagged; approving needs an explicit override) → queued.
|
||||
*/
|
||||
import { canonicalize } from './canonical'
|
||||
|
||||
type CandidateSource = 'rss' | 'form' | 'ext' | 'admin'
|
||||
export type CandidateStatus = 'pending' | 'rejected' | 'queued' | 'ingested' | 'failed'
|
||||
|
||||
export type Outcome =
|
||||
| 'queued'
|
||||
| 'invalid_url'
|
||||
| 'duplicate_in_request'
|
||||
| 'duplicate_catalog'
|
||||
| 'duplicate_pending'
|
||||
| 'previously_deleted'
|
||||
| 'rejected'
|
||||
|
||||
export interface ClassifyContext {
|
||||
catalog: Set<string>
|
||||
deleted: Map<string, { reason: string; deleted_at: string }>
|
||||
/** Status of URLs already in D1 (only the ones asked about). */
|
||||
known: Map<string, CandidateStatus>
|
||||
}
|
||||
|
||||
export interface Classified {
|
||||
input: string
|
||||
canonical: string | null
|
||||
outcome: Outcome
|
||||
/** Whether a new candidate row should be stored. */
|
||||
store: boolean
|
||||
reason?: string
|
||||
}
|
||||
|
||||
export function classify(inputs: string[], ctx: ClassifyContext): Classified[] {
|
||||
const seen = new Set<string>()
|
||||
return inputs.map((input) => {
|
||||
const canonical = canonicalize(input)
|
||||
if (!canonical) return { input, canonical, outcome: 'invalid_url', store: false }
|
||||
if (seen.has(canonical)) return { input, canonical, outcome: 'duplicate_in_request', store: false }
|
||||
seen.add(canonical)
|
||||
if (ctx.catalog.has(canonical)) return { input, canonical, outcome: 'duplicate_catalog', store: false }
|
||||
const status = ctx.known.get(canonical)
|
||||
if (status === 'rejected') return { input, canonical, outcome: 'rejected', store: false }
|
||||
if (status && status !== 'failed') return { input, canonical, outcome: 'duplicate_pending', store: false }
|
||||
const deleted = ctx.deleted.get(canonical)
|
||||
if (deleted) {
|
||||
return {
|
||||
input,
|
||||
canonical,
|
||||
outcome: 'previously_deleted',
|
||||
store: true,
|
||||
reason: `${deleted.reason} (${deleted.deleted_at.slice(0, 10)})`,
|
||||
}
|
||||
}
|
||||
return { input, canonical, outcome: 'queued', store: true }
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
const NSFW_WORDS = /(?:^|[^a-z0-9])(nsfw|18\+|hentai|porn|sex|adult|lewd|erotic)(?![a-z0-9])/i
|
||||
|
||||
/** Review hints stored with a new candidate (never decisions by themselves). */
|
||||
export function autoFlags(outcome: Outcome, title?: string | null): string[] {
|
||||
const flags: string[] = []
|
||||
if (outcome === 'previously_deleted') flags.push('previously_deleted')
|
||||
if (title && NSFW_WORDS.test(title)) flags.push('nsfw_keyword')
|
||||
if (title && /(?:^|[^a-z0-9])demo(?![a-z0-9])/i.test(title)) flags.push('demo')
|
||||
return flags
|
||||
}
|
||||
|
||||
export interface CandidateRow {
|
||||
url: string
|
||||
source: CandidateSource
|
||||
status: CandidateStatus
|
||||
title: string | null
|
||||
image_url: string | null
|
||||
genre_hint: string | null
|
||||
flags: string
|
||||
note: string | null
|
||||
submitter: string | null
|
||||
discovered_at: string
|
||||
decided_at: string | null
|
||||
decided_by: string | null
|
||||
}
|
||||
|
||||
export interface NewCandidate {
|
||||
url: string
|
||||
source: CandidateSource
|
||||
title?: string | null
|
||||
image_url?: string | null
|
||||
genre_hint?: string | null
|
||||
flags?: string[]
|
||||
note?: string | null
|
||||
submitter?: string | null
|
||||
}
|
||||
|
||||
export interface FeedState {
|
||||
feed: string
|
||||
last_polled_at: string | null
|
||||
backoff_until: string | null
|
||||
last_status: number | null
|
||||
}
|
||||
|
||||
/** D1 binds at most 100 parameters per statement. */
|
||||
const PARAM_SLICE = 90
|
||||
const QUEUED_TIMEOUT_MS = 3 * 86_400_000
|
||||
const IDEMPOTENCY_TTL_MS = 7 * 86_400_000
|
||||
const FORM_NOTE_TTL_MS = 180 * 86_400_000
|
||||
|
||||
export function isoSeconds(date: Date): string {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, 'Z')
|
||||
}
|
||||
|
||||
function slices<T>(items: T[]): T[][] {
|
||||
const out: T[][] = []
|
||||
for (let i = 0; i < items.length; i += PARAM_SLICE) out.push(items.slice(i, i + PARAM_SLICE))
|
||||
return out
|
||||
}
|
||||
|
||||
export class QueueStore {
|
||||
private readonly db: D1Database
|
||||
|
||||
constructor(db: D1Database) {
|
||||
this.db = db
|
||||
}
|
||||
|
||||
async statuses(urls: string[]): Promise<Map<string, CandidateStatus>> {
|
||||
const out = new Map<string, CandidateStatus>()
|
||||
for (const slice of slices(urls)) {
|
||||
const { results } = await this.db
|
||||
.prepare(`SELECT url, status FROM candidates WHERE url IN (${slice.map(() => '?').join(',')})`)
|
||||
.bind(...slice)
|
||||
.all<{ url: string; status: CandidateStatus }>()
|
||||
for (const r of results) out.set(r.url, r.status)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Insert new candidates; an existing `failed` row is reopened as pending. */
|
||||
async insert(items: NewCandidate[], now = new Date()): Promise<void> {
|
||||
if (items.length === 0) return
|
||||
const stmt = this.db.prepare(
|
||||
`INSERT INTO candidates (url, source, title, image_url, genre_hint, flags, note, submitter, discovered_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (url) DO UPDATE SET status = 'pending', source = excluded.source,
|
||||
title = coalesce(excluded.title, candidates.title),
|
||||
image_url = coalesce(excluded.image_url, candidates.image_url),
|
||||
flags = excluded.flags, note = excluded.note, submitter = excluded.submitter,
|
||||
discovered_at = excluded.discovered_at, decided_at = NULL, decided_by = NULL
|
||||
WHERE candidates.status = 'failed'`,
|
||||
)
|
||||
await this.db.batch(
|
||||
items.map((c) =>
|
||||
stmt.bind(
|
||||
c.url,
|
||||
c.source,
|
||||
c.title ?? null,
|
||||
c.image_url ?? null,
|
||||
c.genre_hint ?? null,
|
||||
JSON.stringify(c.flags ?? []),
|
||||
c.note ?? null,
|
||||
c.submitter ?? null,
|
||||
isoSeconds(now),
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
async list(status: CandidateStatus, limit = 200): Promise<CandidateRow[]> {
|
||||
const { results } = await this.db
|
||||
.prepare('SELECT * FROM candidates WHERE status = ? ORDER BY discovered_at DESC, url LIMIT ?')
|
||||
.bind(status, limit)
|
||||
.all<CandidateRow>()
|
||||
return results
|
||||
}
|
||||
|
||||
async counts(): Promise<Record<CandidateStatus, number>> {
|
||||
const out: Record<CandidateStatus, number> = { pending: 0, rejected: 0, queued: 0, ingested: 0, failed: 0 }
|
||||
const { results } = await this.db
|
||||
.prepare('SELECT status, count(*) AS n FROM candidates GROUP BY status')
|
||||
.all<{ status: CandidateStatus; n: number }>()
|
||||
for (const r of results) out[r.status] = r.n
|
||||
return out
|
||||
}
|
||||
|
||||
async get(urls: string[]): Promise<CandidateRow[]> {
|
||||
const out: CandidateRow[] = []
|
||||
for (const slice of slices(urls)) {
|
||||
const { results } = await this.db
|
||||
.prepare(`SELECT * FROM candidates WHERE url IN (${slice.map(() => '?').join(',')})`)
|
||||
.bind(...slice)
|
||||
.all<CandidateRow>()
|
||||
out.push(...results)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
async setStatus(urls: string[], status: CandidateStatus, actor: string, now = new Date()): Promise<void> {
|
||||
if (urls.length === 0) return
|
||||
const stmt = this.db.prepare('UPDATE candidates SET status = ?, decided_by = ?, decided_at = ? WHERE url = ?')
|
||||
await this.db.batch(urls.map((u) => stmt.bind(status, actor, isoSeconds(now), u)))
|
||||
}
|
||||
|
||||
/**
|
||||
* queued → ingested once the game is in the published catalog; queued for
|
||||
* more than 3 days without showing up (the ingest workflow gave up on it)
|
||||
* → failed, so it comes back for review.
|
||||
*/
|
||||
async reconcile(catalog: Set<string>, now = new Date()): Promise<{ ingested: number; failed: number }> {
|
||||
const { results } = await this.db
|
||||
.prepare("SELECT url, decided_at FROM candidates WHERE status = 'queued'")
|
||||
.all<{ url: string; decided_at: string | null }>()
|
||||
const cutoff = isoSeconds(new Date(now.getTime() - QUEUED_TIMEOUT_MS))
|
||||
const ingested = results.filter((c) => catalog.has(c.url)).map((c) => c.url)
|
||||
const failed = results.filter((c) => !catalog.has(c.url) && (c.decided_at ?? '') < cutoff).map((c) => c.url)
|
||||
const set = this.db.prepare('UPDATE candidates SET status = ? WHERE url = ?')
|
||||
const updates = [...ingested.map((u) => set.bind('ingested', u)), ...failed.map((u) => set.bind('failed', u))]
|
||||
if (updates.length > 0) await this.db.batch(updates)
|
||||
return { ingested: ingested.length, failed: failed.length }
|
||||
}
|
||||
|
||||
/** Retention: idempotency records after 7 days, Suggest-form notes after 180. */
|
||||
async purge(now = new Date()): Promise<void> {
|
||||
await this.db.batch([
|
||||
this.db
|
||||
.prepare('DELETE FROM idempotency WHERE created_at < ?')
|
||||
.bind(isoSeconds(new Date(now.getTime() - IDEMPOTENCY_TTL_MS))),
|
||||
this.db
|
||||
.prepare("UPDATE candidates SET note = NULL WHERE source = 'form' AND note IS NOT NULL AND discovered_at < ?")
|
||||
.bind(isoSeconds(new Date(now.getTime() - FORM_NOTE_TTL_MS))),
|
||||
])
|
||||
}
|
||||
|
||||
async idempotent(client: string, key: string): Promise<{ body_hash: string; response: string } | null> {
|
||||
return this.db
|
||||
.prepare('SELECT body_hash, response FROM idempotency WHERE client = ? AND key = ?')
|
||||
.bind(client, key)
|
||||
.first<{ body_hash: string; response: string }>()
|
||||
}
|
||||
|
||||
async remember(client: string, key: string, bodyHash: string, response: string, now = new Date()): Promise<void> {
|
||||
await this.db
|
||||
.prepare('INSERT OR IGNORE INTO idempotency (client, key, body_hash, response, created_at) VALUES (?, ?, ?, ?, ?)')
|
||||
.bind(client, key, bodyHash, response, isoSeconds(now))
|
||||
.run()
|
||||
}
|
||||
|
||||
async feeds(): Promise<Map<string, FeedState>> {
|
||||
const { results } = await this.db.prepare('SELECT * FROM feed_state').all<FeedState>()
|
||||
return new Map(results.map((f) => [f.feed, f]))
|
||||
}
|
||||
|
||||
async saveFeed(state: FeedState): Promise<void> {
|
||||
await this.db
|
||||
.prepare(
|
||||
`INSERT INTO feed_state (feed, last_polled_at, backoff_until, last_status) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (feed) DO UPDATE SET last_polled_at = excluded.last_polled_at,
|
||||
backoff_until = excluded.backoff_until, last_status = excluded.last_status`,
|
||||
)
|
||||
.bind(state.feed, state.last_polled_at, state.backoff_until, state.last_status)
|
||||
.run()
|
||||
}
|
||||
|
||||
async audit(actor: string, action: string, target: string, commitSha: string | null, detail?: string) {
|
||||
await this.db
|
||||
.prepare('INSERT INTO audit_log (actor, action, target, commit_sha, detail) VALUES (?, ?, ?, ?, ?)')
|
||||
.bind(actor, action, target, commitSha, detail ?? null)
|
||||
.run()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Admin writes to the repo, each as one commit built from the files at the
|
||||
* current head of main (commitWithRetry re-reads and rebuilds on a race).
|
||||
* Pure builders are exported for tests; the GitHub reads stay thin.
|
||||
* Commit messages never name the maintainer (their email stays in the
|
||||
* private D1 audit log): the repo and its history are public.
|
||||
*/
|
||||
import { canonicalize } from './canonical'
|
||||
import {
|
||||
applyAdminChange,
|
||||
dumps,
|
||||
type CatalogFiles,
|
||||
type CatalogIndex,
|
||||
type CountPoint,
|
||||
type DeletedEntry,
|
||||
type FieldEdit,
|
||||
type Game,
|
||||
} from './catalog'
|
||||
import { commitWithRetry, listDir, readFile, type CommitInput, type GitHubEnv } from './github'
|
||||
|
||||
const TEMP_LINK = 'scripts/temp_link.json'
|
||||
const DELETED_LOG = 'scripts/deleted_games.json'
|
||||
const CHUNK_FILE = /^game_info_\d+\.json$/
|
||||
|
||||
type Change = Omit<CommitInput, 'expectedHeadOid'>
|
||||
|
||||
async function readJsonFile<T>(env: GitHubEnv, path: string, head: string, fallback: T): Promise<T> {
|
||||
const text = await readFile(env, path, head)
|
||||
return text === null ? fallback : (JSON.parse(text) as T)
|
||||
}
|
||||
|
||||
/** The catalog files at `head`, chunks in the same order as data_store.py. */
|
||||
async function readCatalog(env: GitHubEnv, head: string): Promise<CatalogFiles> {
|
||||
const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort()
|
||||
const [chunks, index, history, deleted] = await Promise.all([
|
||||
Promise.all(names.map((n) => readJsonFile<Game[]>(env, `data_game/${n}`, head, []))),
|
||||
readJsonFile<CatalogIndex | null>(env, 'data_game/index.json', head, null),
|
||||
readJsonFile<CountPoint[]>(env, 'data_game/count_history.json', head, []),
|
||||
readJsonFile<DeletedEntry[]>(env, DELETED_LOG, head, []),
|
||||
])
|
||||
return { chunks, index, history, deleted }
|
||||
}
|
||||
|
||||
/**
|
||||
* temp_link.json with `urls` appended (skipping ones already queued in any
|
||||
* spelling). Unknown entries are kept as they are; apply_patch.py cleans them.
|
||||
*/
|
||||
export function withQueued(queueText: string | null, urls: string[]): { text: string; added: string[] } {
|
||||
const parsed: unknown = queueText === null ? [] : JSON.parse(queueText)
|
||||
const queue: unknown[] = Array.isArray(parsed) ? parsed : []
|
||||
const present = new Set(queue.map((u) => (typeof u === 'string' ? (canonicalize(u) ?? u) : '')))
|
||||
const added: string[] = []
|
||||
for (const url of urls) {
|
||||
if (present.has(url)) continue
|
||||
present.add(url)
|
||||
added.push(url)
|
||||
}
|
||||
return { text: dumps([...queue, ...added]), added }
|
||||
}
|
||||
|
||||
/** deleted_games.json without `urls` (a game the maintainer allows back in). */
|
||||
export function withoutDeleted(deletedText: string | null, urls: Set<string>): { text: string; removed: number } {
|
||||
const log = deletedText === null ? [] : (JSON.parse(deletedText) as DeletedEntry[])
|
||||
const kept = log.filter((e) => !urls.has(e.url))
|
||||
return { text: dumps(kept), removed: log.length - kept.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue `urls` for the ingest workflow (the push to temp_link.json triggers
|
||||
* update.yml). With `unblock`, also drop those URLs from the deleted log,
|
||||
* which update_info.py otherwise refuses to re-add.
|
||||
*/
|
||||
export async function queueForIngest(
|
||||
env: GitHubEnv,
|
||||
urls: string[],
|
||||
unblock: string[],
|
||||
): Promise<{ sha: string | null; added: string[] }> {
|
||||
let added: string[] = []
|
||||
const sha = await commitWithRetry(env, async (head): Promise<Change | null> => {
|
||||
const writes = new Map<string, string>()
|
||||
const queued = withQueued(await readFile(env, TEMP_LINK, head), urls)
|
||||
added = queued.added
|
||||
if (queued.added.length > 0) writes.set(TEMP_LINK, queued.text)
|
||||
if (unblock.length > 0) {
|
||||
const log = withoutDeleted(await readFile(env, DELETED_LOG, head), new Set(unblock))
|
||||
if (log.removed > 0) writes.set(DELETED_LOG, log.text)
|
||||
}
|
||||
if (writes.size === 0) return null
|
||||
return {
|
||||
headline: `Queue ${urls.length} game(s) for ingest [admin]`,
|
||||
body: `Via the admin page.\n\n${urls.join('\n')}`,
|
||||
writes,
|
||||
}
|
||||
})
|
||||
return { sha, added }
|
||||
}
|
||||
|
||||
export interface CatalogEditResult {
|
||||
sha: string | null
|
||||
summary: { edited: number; removed: number }
|
||||
/** Thumbnails of removed games (their resized copies are deleted from R2). */
|
||||
thumbnails: string[]
|
||||
}
|
||||
|
||||
/** Edit maintainer fields and/or remove games in one commit. */
|
||||
export async function editCatalog(
|
||||
env: GitHubEnv,
|
||||
edits: Map<string, FieldEdit>,
|
||||
removals: { url: string; reason: string }[],
|
||||
now: () => Date = () => new Date(),
|
||||
): Promise<CatalogEditResult> {
|
||||
let result: CatalogEditResult = { sha: null, summary: { edited: 0, removed: 0 }, thumbnails: [] }
|
||||
const sha = await commitWithRetry(env, async (head): Promise<Change | null> => {
|
||||
const files = await readCatalog(env, head)
|
||||
const removing = new Set(removals.map((r) => r.url))
|
||||
const thumbnails = files.chunks
|
||||
.flat()
|
||||
.filter((g) => removing.has(g.url) && typeof g.thumbnail === 'string' && g.thumbnail.startsWith('https://'))
|
||||
.map((g) => g.thumbnail as string)
|
||||
const change = applyAdminChange(files, edits, removals, now())
|
||||
result = { sha: null, summary: change.summary, thumbnails }
|
||||
if (change.writes.size === 0 && change.deletes.length === 0) return null
|
||||
const parts = [
|
||||
change.summary.edited ? `edit ${change.summary.edited}` : '',
|
||||
change.summary.removed ? `remove ${change.summary.removed}` : '',
|
||||
].filter(Boolean)
|
||||
const lines = [
|
||||
...[...edits].map(([url, edit]) => `edit ${url} ${JSON.stringify(edit)}`),
|
||||
...removals.map((r) => `remove ${r.url} (${r.reason})`),
|
||||
]
|
||||
return {
|
||||
headline: `Catalog: ${parts.join(', ')} [admin]`,
|
||||
body: `Via the admin page.\n\n${lines.join('\n')}`,
|
||||
writes: change.writes,
|
||||
deletes: change.deletes,
|
||||
}
|
||||
})
|
||||
return { ...result, sha }
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Public "suggest a game" form: GET /api/suggest (is it enabled + Turnstile
|
||||
* site key), POST /api/suggest {url, note?, token}.
|
||||
*
|
||||
* Guards: same-origin POST, per-IP rate limit (RL_SUGGEST), Turnstile
|
||||
* siteverify (the only thing a bot cannot forge), canonical itch.io URL. A
|
||||
* suggestion only enters the review queue; the answer never says more than
|
||||
* the public catalog already shows.
|
||||
*/
|
||||
import { canonicalize } from './canonical'
|
||||
import { catalogUrls, deletedGames, type DataSource } from './data'
|
||||
import { missingConfig, type WorkerEnv } from './env'
|
||||
import { HttpError, errorResponse, isObject, json, readJson } from './http'
|
||||
import { autoFlags, classify, type Outcome, type QueueStore } from './queue'
|
||||
|
||||
const SITEVERIFY = 'https://challenges.cloudflare.com/turnstile/v0/siteverify'
|
||||
const MAX_NOTE = 500
|
||||
|
||||
type SuggestStatus = 'received' | 'already_listed' | 'already_suggested' | 'not_accepted'
|
||||
|
||||
const PUBLIC_STATUS: Record<Exclude<Outcome, 'invalid_url' | 'duplicate_in_request'>, SuggestStatus> = {
|
||||
queued: 'received',
|
||||
previously_deleted: 'received', // flagged for the maintainer
|
||||
duplicate_catalog: 'already_listed',
|
||||
duplicate_pending: 'already_suggested',
|
||||
rejected: 'not_accepted',
|
||||
}
|
||||
|
||||
export interface SuggestDeps {
|
||||
store: QueueStore | null
|
||||
data: DataSource
|
||||
fetch: typeof fetch
|
||||
limiter: Pick<RateLimit, 'limit'> | undefined
|
||||
}
|
||||
|
||||
function enabled(env: WorkerEnv, deps: SuggestDeps): boolean {
|
||||
return missingConfig(env, ['TURNSTILE_SECRET', 'TURNSTILE_SITEKEY', 'SITE_ORIGIN']).length === 0 && deps.store !== null
|
||||
}
|
||||
|
||||
async function verifyTurnstile(env: WorkerEnv, deps: SuggestDeps, token: string, ip: string): Promise<boolean> {
|
||||
const form = new FormData()
|
||||
form.set('secret', env.TURNSTILE_SECRET ?? '')
|
||||
form.set('response', token)
|
||||
if (ip) form.set('remoteip', ip)
|
||||
try {
|
||||
const res = await deps.fetch(SITEVERIFY, { method: 'POST', body: form })
|
||||
const outcome = (await res.json()) as { success?: boolean; hostname?: string }
|
||||
if (!outcome.success) return false
|
||||
const expected = new URL(env.SITE_ORIGIN).hostname
|
||||
// Turnstile's test keys answer with a placeholder hostname; only local dev uses them.
|
||||
return !outcome.hostname || outcome.hostname === expected || expected === 'localhost'
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSuggest(request: Request, env: WorkerEnv, deps: SuggestDeps): Promise<Response> {
|
||||
try {
|
||||
if (request.method === 'GET') {
|
||||
const on = enabled(env, deps)
|
||||
return json({ enabled: on, sitekey: on ? env.TURNSTILE_SITEKEY : null })
|
||||
}
|
||||
if (request.method !== 'POST') throw new HttpError(405, 'method_not_allowed')
|
||||
if (!enabled(env, deps) || !deps.store) throw new HttpError(503, 'unavailable')
|
||||
if (request.headers.get('origin') !== env.SITE_ORIGIN) throw new HttpError(403, 'bad_origin')
|
||||
|
||||
const ip = request.headers.get('cf-connecting-ip') ?? ''
|
||||
if (deps.limiter && !(await deps.limiter.limit({ key: ip || 'unknown' })).success) {
|
||||
throw new HttpError(429, 'rate_limited')
|
||||
}
|
||||
const body = await readJson(request, 4096)
|
||||
if (!isObject(body) || typeof body.url !== 'string' || typeof body.token !== 'string') {
|
||||
throw new HttpError(400, 'invalid_body')
|
||||
}
|
||||
const note = typeof body.note === 'string' ? body.note.trim() : ''
|
||||
if (note.length > MAX_NOTE) throw new HttpError(400, 'note_too_long')
|
||||
const url = canonicalize(body.url)
|
||||
if (!url) throw new HttpError(400, 'invalid_url')
|
||||
if (!(await verifyTurnstile(env, deps, body.token, ip))) throw new HttpError(403, 'challenge_failed')
|
||||
|
||||
const [catalog, deleted, known] = await Promise.all([
|
||||
catalogUrls(deps.data),
|
||||
deletedGames(deps.data),
|
||||
deps.store.statuses([url]),
|
||||
])
|
||||
const [result] = classify([url], { catalog: new Set(catalog.keys()), deleted, known })
|
||||
if (result.store) {
|
||||
await deps.store.insert([
|
||||
{ url, source: 'form', note: note || null, submitter: 'form', flags: autoFlags(result.outcome) },
|
||||
])
|
||||
}
|
||||
const status = PUBLIC_STATUS[result.outcome as keyof typeof PUBLIC_STATUS]
|
||||
return json({ status })
|
||||
} catch (e) {
|
||||
return errorResponse(e)
|
||||
}
|
||||
}
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
// Vite/Vitest `?raw` imports (used by tests for golden text fixtures).
|
||||
declare module '*?raw' {
|
||||
const text: string
|
||||
export default text
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { SignJWT, exportJWK, generateKeyPair } from 'jose'
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
import { ADMIN_CSP, handleAdminApi, serveAdminApp } from '../admin'
|
||||
import type { WorkerEnv } from '../env'
|
||||
import { ORIGIN } from './fakes'
|
||||
|
||||
const TEAM = 'https://team.cloudflareaccess.com'
|
||||
let privateKey: CryptoKey
|
||||
|
||||
const ADMIN_SHELL = '<!doctype html><title>admin</title>'
|
||||
const env = {
|
||||
SITE_ORIGIN: ORIGIN,
|
||||
ACCESS_TEAM_DOMAIN: TEAM,
|
||||
ACCESS_AUD_ADMIN: 'aud-admin',
|
||||
ACCESS_AUD_INGEST: 'aud-ingest',
|
||||
ADMIN_EMAILS: 'owner@example.com',
|
||||
ASSETS: {
|
||||
fetch: async (req: Request) =>
|
||||
new URL(req.url).pathname === '/admin/'
|
||||
? new Response(ADMIN_SHELL, { headers: { 'content-type': 'text/html', 'content-security-policy': "default-src 'self'; script-src https://static.cloudflareinsights.com" } })
|
||||
: new Response(null, { status: 404 }),
|
||||
},
|
||||
} as unknown as WorkerEnv
|
||||
|
||||
beforeAll(async () => {
|
||||
const pair = await generateKeyPair('RS256')
|
||||
privateKey = pair.privateKey
|
||||
const jwk = { ...(await exportJWK(pair.publicKey)), kid: 'k1', alg: 'RS256', use: 'sig' }
|
||||
vi.stubGlobal('fetch', async (url: string) =>
|
||||
String(url) === `${TEAM}/cdn-cgi/access/certs` ? Response.json({ keys: [jwk] }) : new Response(null, { status: 404 }),
|
||||
)
|
||||
})
|
||||
afterAll(() => vi.unstubAllGlobals())
|
||||
|
||||
const jwt = (email: string, aud = 'aud-admin') =>
|
||||
new SignJWT({ email })
|
||||
.setProtectedHeader({ alg: 'RS256', kid: 'k1' })
|
||||
.setIssuer(TEAM)
|
||||
.setAudience(aud)
|
||||
.setExpirationTime('5m')
|
||||
.sign(privateKey)
|
||||
|
||||
async function get(path: string, token?: string, e: WorkerEnv = env) {
|
||||
const url = new URL(path, ORIGIN)
|
||||
const headers: Record<string, string> = token ? { 'cf-access-jwt-assertion': token } : {}
|
||||
const request = new Request(url, { headers })
|
||||
return path.startsWith('/api/') ? handleAdminApi(request, url, e, null) : serveAdminApp(request, url, e)
|
||||
}
|
||||
|
||||
describe('admin authentication', () => {
|
||||
it('serves the admin shell only to the maintainer, with its own CSP', async () => {
|
||||
const res = await get('/admin/', await jwt('owner@example.com'))
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.text()).toBe(ADMIN_SHELL)
|
||||
expect(res.headers.get('content-security-policy')).toBe(ADMIN_CSP)
|
||||
expect(res.headers.get('cache-control')).toBe('no-store')
|
||||
expect(res.headers.get('x-robots-tag')).toContain('noindex')
|
||||
})
|
||||
|
||||
it.each([
|
||||
['no token', undefined],
|
||||
['someone else', 'stranger@example.com'],
|
||||
])('refuses %s', async (_name, email) => {
|
||||
const token = email ? await jwt(email) : undefined
|
||||
expect((await get('/admin', token)).status).toBe(403)
|
||||
expect((await get('/api/admin/me', token)).status).toBe(403)
|
||||
})
|
||||
|
||||
it('refuses a token minted for the ingest application', async () => {
|
||||
expect((await get('/admin/', await jwt('owner@example.com', 'aud-ingest'))).status).toBe(403)
|
||||
})
|
||||
|
||||
it('answers 503 until Access is configured, never serving the shell', async () => {
|
||||
const bare = { ...env, ACCESS_AUD_ADMIN: '' } as WorkerEnv
|
||||
expect((await get('/admin/', await jwt('owner@example.com'), bare)).status).toBe(503)
|
||||
expect((await get('/api/admin/me', await jwt('owner@example.com'), bare)).status).toBe(503)
|
||||
})
|
||||
|
||||
it('does not serve files from under /admin', async () => {
|
||||
expect((await get('/admin/secrets.json', await jwt('owner@example.com'))).status).toBe(404)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,195 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { adminApi, type AdminContext, type RepoOps } from '../admin'
|
||||
import { resetDataCache } from '../data'
|
||||
import { r2Key } from '../img'
|
||||
import { QueueStore } from '../queue'
|
||||
import { withoutDeleted, withQueued } from '../repo'
|
||||
import { testDb } from './d1'
|
||||
import { fakeData, ORIGIN } from './fakes'
|
||||
|
||||
let db: D1Database
|
||||
let dispose: () => Promise<void>
|
||||
let store: QueueStore
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ db, dispose } = await testDb())
|
||||
}, 60_000)
|
||||
afterAll(() => dispose())
|
||||
beforeEach(async () => {
|
||||
resetDataCache()
|
||||
await db.batch(['candidates', 'audit_log'].map((t) => db.prepare(`DELETE FROM ${t}`)))
|
||||
store = new QueueStore(db)
|
||||
})
|
||||
|
||||
const NOW = new Date('2026-09-18T12:00:00Z')
|
||||
const THUMB = 'https://img.itch.zone/aW1n/original/cover.png'
|
||||
|
||||
function fakeRepo() {
|
||||
const repo = {
|
||||
queueForIngest: vi.fn(async (urls: string[]) => ({ sha: 'sha-queue', added: urls })),
|
||||
editCatalog: vi.fn(async (edits: Map<string, unknown>, removals: { url: string }[]) => ({
|
||||
sha: 'sha-edit',
|
||||
summary: { edited: edits.size, removed: removals.length },
|
||||
thumbnails: removals.length ? [THUMB] : [],
|
||||
})),
|
||||
}
|
||||
return repo satisfies RepoOps
|
||||
}
|
||||
|
||||
function context(over: Partial<AdminContext> = {}) {
|
||||
const thumbs = { delete: vi.fn(async () => undefined) }
|
||||
const ctx: AdminContext = {
|
||||
actor: 'owner@example.com',
|
||||
store,
|
||||
data: fakeData(['https://dev.itch.io/listed'], [{ url: 'https://dev.itch.io/gone' }]),
|
||||
repo: fakeRepo(),
|
||||
thumbs,
|
||||
now: NOW,
|
||||
...over,
|
||||
}
|
||||
return { ctx, thumbs, repo: ctx.repo as ReturnType<typeof fakeRepo> }
|
||||
}
|
||||
|
||||
async function call(ctx: AdminContext, method: string, path: string, body?: unknown) {
|
||||
const url = new URL(path, ORIGIN)
|
||||
const init: RequestInit = { method }
|
||||
if (body !== undefined) {
|
||||
init.body = JSON.stringify(body)
|
||||
init.headers = { 'content-type': 'application/json' }
|
||||
}
|
||||
const res = await adminApi(new Request(url, init), url, ctx)
|
||||
return { status: res.status, body: (await res.json()) as Record<string, unknown> }
|
||||
}
|
||||
|
||||
describe('admin API', () => {
|
||||
it('reports the maintainer and whether writes are possible', async () => {
|
||||
expect((await call(context().ctx, 'GET', '/api/admin/me')).body).toEqual({ email: 'owner@example.com', writes: true })
|
||||
expect((await call(context({ repo: null }).ctx, 'GET', '/api/admin/me')).body.writes).toBe(false)
|
||||
})
|
||||
|
||||
it('lists the queue with parsed flags and counts', async () => {
|
||||
await store.insert([{ url: 'https://dev.itch.io/a', source: 'rss', flags: ['demo'] }], NOW)
|
||||
const { body } = await call(context().ctx, 'GET', '/api/admin/queue?status=pending')
|
||||
expect(body.counts).toMatchObject({ pending: 1 })
|
||||
expect(body.items).toEqual([expect.objectContaining({ url: 'https://dev.itch.io/a', flags: ['demo'] })])
|
||||
expect((await call(context().ctx, 'GET', '/api/admin/queue?status=bogus')).status).toBe(400)
|
||||
})
|
||||
|
||||
it('approves pending rows into one commit and records the decision', async () => {
|
||||
await store.insert(['a', 'b'].map((s) => ({ url: `https://dev.itch.io/${s}`, source: 'rss' as const })), NOW)
|
||||
const { ctx, repo } = context()
|
||||
const { body } = await call(ctx, 'POST', '/api/admin/queue/decide', {
|
||||
action: 'approve',
|
||||
urls: ['https://dev.itch.io/a', 'https://dev.itch.io/b', 'https://dev.itch.io/unknown'],
|
||||
})
|
||||
expect(body).toEqual({
|
||||
sha: 'sha-queue',
|
||||
approved: ['https://dev.itch.io/a', 'https://dev.itch.io/b'],
|
||||
skipped: [{ url: 'https://dev.itch.io/unknown', reason: 'not_in_queue' }],
|
||||
})
|
||||
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/a', 'https://dev.itch.io/b'], [])
|
||||
expect((await store.statuses(['https://dev.itch.io/a'])).get('https://dev.itch.io/a')).toBe('queued')
|
||||
const audit = await db.prepare('SELECT actor, action, commit_sha FROM audit_log').first()
|
||||
expect(audit).toEqual({ actor: 'owner@example.com', action: 'approve', commit_sha: 'sha-queue' })
|
||||
})
|
||||
|
||||
it('needs an override to bring back a game that was removed', async () => {
|
||||
await store.insert([{ url: 'https://dev.itch.io/gone', source: 'form', flags: ['previously_deleted'] }], NOW)
|
||||
const { ctx, repo } = context()
|
||||
const refused = await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'approve', urls: ['https://dev.itch.io/gone'] })
|
||||
expect(refused.body.skipped).toEqual([{ url: 'https://dev.itch.io/gone', reason: 'needs_override' }])
|
||||
expect(repo.queueForIngest).not.toHaveBeenCalled()
|
||||
await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'approve', urls: ['https://dev.itch.io/gone'], override: true })
|
||||
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/gone'], ['https://dev.itch.io/gone'])
|
||||
})
|
||||
|
||||
it('rejects rows (a permanent blocklist) without touching the repo', async () => {
|
||||
await store.insert([{ url: 'https://dev.itch.io/spam', source: 'form' }], NOW)
|
||||
const { ctx, repo } = context()
|
||||
await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'reject', urls: ['https://dev.itch.io/spam'] })
|
||||
expect((await store.statuses(['https://dev.itch.io/spam'])).get('https://dev.itch.io/spam')).toBe('rejected')
|
||||
expect(repo.queueForIngest).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('adds and approves pasted URLs in one step', async () => {
|
||||
await store.insert([{ url: 'https://dev.itch.io/waiting', source: 'rss' }], NOW)
|
||||
await store.insert([{ url: 'https://dev.itch.io/blocked', source: 'form' }], NOW)
|
||||
await store.setStatus(['https://dev.itch.io/blocked'], 'rejected', 'x', NOW)
|
||||
const { ctx, repo } = context()
|
||||
const { body } = await call(ctx, 'POST', '/api/admin/add', {
|
||||
urls: ['https://dev.itch.io/new', 'https://dev.itch.io/listed', 'https://dev.itch.io/waiting', 'https://dev.itch.io/blocked', 'x'],
|
||||
})
|
||||
const results = body.results as { url: string | null; outcome: string; approved: boolean }[]
|
||||
expect(results.map((r) => [r.outcome, r.approved])).toEqual([
|
||||
['queued', true],
|
||||
['duplicate_catalog', false],
|
||||
['duplicate_pending', true],
|
||||
['rejected', false],
|
||||
['invalid_url', false],
|
||||
])
|
||||
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/new', 'https://dev.itch.io/waiting'], [])
|
||||
expect((await store.get(['https://dev.itch.io/new']))[0]).toMatchObject({ source: 'admin', status: 'queued' })
|
||||
})
|
||||
|
||||
it('edits only maintainer fields', async () => {
|
||||
const { ctx, repo } = context()
|
||||
const ok = await call(ctx, 'PATCH', '/api/admin/games', { url: 'https://dev.itch.io/listed', edit: { nsfw: 'Yes', notes: 'x' } })
|
||||
expect(ok.body).toEqual({ sha: 'sha-edit', edited: 1 })
|
||||
expect(repo.editCatalog).toHaveBeenCalledWith(new Map([['https://dev.itch.io/listed', { nsfw: 'Yes', notes: 'x' }]]), [])
|
||||
expect((await call(ctx, 'PATCH', '/api/admin/games', { url: 'https://dev.itch.io/listed', edit: { rating: '5' } })).status).toBe(400)
|
||||
expect((await call(ctx, 'PATCH', '/api/admin/games', { url: 'https://dev.itch.io/listed', edit: { nsfw: 'maybe' } })).status).toBe(400)
|
||||
})
|
||||
|
||||
it('removes a game with a reason and deletes its resized covers', async () => {
|
||||
const { ctx, repo, thumbs } = context()
|
||||
expect((await call(ctx, 'DELETE', '/api/admin/games', { url: 'https://dev.itch.io/listed' })).status).toBe(400)
|
||||
const { body } = await call(ctx, 'DELETE', '/api/admin/games', { url: 'https://dev.itch.io/listed', reason: 'Takedown request' })
|
||||
expect(body).toEqual({ sha: 'sha-edit', removed: 1 })
|
||||
expect(repo.editCatalog).toHaveBeenCalledWith(new Map(), [{ url: 'https://dev.itch.io/listed', reason: 'Takedown request' }])
|
||||
expect(thumbs.delete).toHaveBeenCalledWith([await r2Key(160, THUMB), await r2Key(640, THUMB)])
|
||||
})
|
||||
|
||||
it('restores a removed game: unblock + queue again', async () => {
|
||||
const { ctx, repo } = context()
|
||||
const { body } = await call(ctx, 'POST', '/api/admin/games/restore', { url: 'https://dev.itch.io/gone' })
|
||||
expect(body).toEqual({ sha: 'sha-queue', queued: true })
|
||||
expect(repo.queueForIngest).toHaveBeenCalledWith(['https://dev.itch.io/gone'], ['https://dev.itch.io/gone'])
|
||||
expect((await store.statuses(['https://dev.itch.io/gone'])).get('https://dev.itch.io/gone')).toBe('queued')
|
||||
})
|
||||
|
||||
it('stays read-only without the GitHub App', async () => {
|
||||
await store.insert([{ url: 'https://dev.itch.io/a', source: 'rss' }], NOW)
|
||||
const { ctx } = context({ repo: null })
|
||||
const res = await call(ctx, 'POST', '/api/admin/queue/decide', { action: 'approve', urls: ['https://dev.itch.io/a'] })
|
||||
expect(res.status).toBe(503)
|
||||
expect((await store.statuses(['https://dev.itch.io/a'])).get('https://dev.itch.io/a')).toBe('pending')
|
||||
})
|
||||
|
||||
it('validates bodies and routes', async () => {
|
||||
const { ctx } = context()
|
||||
expect((await call(ctx, 'POST', '/api/admin/add', { urls: [] })).status).toBe(400)
|
||||
expect((await call(ctx, 'POST', '/api/admin/add', { urls: Array(201).fill('https://dev.itch.io/a') })).status).toBe(413)
|
||||
expect((await call(ctx, 'POST', '/api/admin/nope', {})).status).toBe(404)
|
||||
})
|
||||
})
|
||||
|
||||
describe('repo builders', () => {
|
||||
it('appends queued URLs once, in any spelling, keeping odd entries', () => {
|
||||
const current = JSON.stringify(['https://DEV.itch.io/a/', { url: 'https://dev.itch.io/obj' }], null, 4)
|
||||
const { text, added } = withQueued(current, ['https://dev.itch.io/a', 'https://dev.itch.io/b', 'https://dev.itch.io/b'])
|
||||
expect(added).toEqual(['https://dev.itch.io/b'])
|
||||
expect(JSON.parse(text)).toEqual(['https://DEV.itch.io/a/', { url: 'https://dev.itch.io/obj' }, 'https://dev.itch.io/b'])
|
||||
expect(text).toBe(JSON.stringify(JSON.parse(text), null, 4)) // json_io.dumps format
|
||||
expect(withQueued(null, ['https://dev.itch.io/x']).text).toBe('[\n "https://dev.itch.io/x"\n]')
|
||||
})
|
||||
|
||||
it('drops unblocked URLs from the deleted log', () => {
|
||||
const log = JSON.stringify([
|
||||
{ url: 'https://dev.itch.io/a', name: 'A', reason: 'r', deleted_at: 't' },
|
||||
{ url: 'https://dev.itch.io/b', name: 'B', reason: 'r', deleted_at: 't' },
|
||||
])
|
||||
const { text, removed } = withoutDeleted(log, new Set(['https://dev.itch.io/a']))
|
||||
expect(removed).toBe(1)
|
||||
expect(JSON.parse(text).map((e: { url: string }) => e.url)).toEqual(['https://dev.itch.io/b'])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* A real (local, in-memory) D1 database with the migrations applied, for
|
||||
* tests of the SQL in queue.ts. Runs workerd through wrangler's platform proxy.
|
||||
*/
|
||||
import { getPlatformProxy } from 'wrangler'
|
||||
import init from '../migrations/0001_init.sql?raw'
|
||||
|
||||
export async function testDb(): Promise<{ db: D1Database; dispose: () => Promise<void> }> {
|
||||
const proxy = await getPlatformProxy<{ DB: D1Database }>({
|
||||
configPath: new URL('./wrangler.test.jsonc', import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'),
|
||||
persist: false,
|
||||
})
|
||||
const statements = init
|
||||
.replace(/--.*$/gm, '')
|
||||
.split(';')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
for (const sql of statements) await proxy.env.DB.prepare(sql).run()
|
||||
return { db: proxy.env.DB, dispose: proxy.dispose }
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { resetDataCache } from '../data'
|
||||
import { discover, FEEDS, parseFeed, pickFeed } from '../discover'
|
||||
import { QueueStore, type FeedState } from '../queue'
|
||||
import { testDb } from './d1'
|
||||
import { fakeData } from './fakes'
|
||||
|
||||
// Same shape as itch.io's browse feeds; made-up games.
|
||||
const item = (slug: string, title: string, tags: string, price = '$0.00', image = true) => `<item>
|
||||
<guid>https://studio.itch.io/${slug}</guid><title>${title} ${tags}</title><plainTitle>${title}</plainTitle>
|
||||
${image ? `<imageurl>https://img.itch.zone/aW1n/315x250%23c/${slug}.png</imageurl>` : '<imageurl>https://evil.example/x.png</imageurl>'}
|
||||
<price>${price}</price><currency>USD</currency><link>https://studio.itch.io/${slug}</link>
|
||||
<description><![CDATA[Short blurb <img src="x"/>]]></description><pubDate>Mon, 14 Sep 2026 10:00:00 GMT</pubDate>
|
||||
</item>`
|
||||
const FEED = `<?xml version="1.0" encoding="UTF-8" ?><rss version="2.0"><channel><title>t</title>
|
||||
${item('moon-garden', 'Moon Garden', '[Free] [Puzzle] [Windows] [macOS]')}
|
||||
${item('ink-and-rain', 'Ink & Rain', '[Free] [Visual Novel] [HTML5]')}
|
||||
${item('paid-thing', 'Paid Thing', '[$4.99] [Action]', '$4.99')}
|
||||
${item('lewd-demo', 'Lewd Demo', '[Free] [Simulation]', '$0.00', false)}
|
||||
${item('already-listed', 'Already Listed', '[Free] [Platformer]')}
|
||||
</channel></rss>`
|
||||
|
||||
describe('parseFeed', () => {
|
||||
it('reads link, plain title, genre, price and a safe cover', () => {
|
||||
const items = parseFeed(FEED)
|
||||
expect(items).toHaveLength(5)
|
||||
expect(items[0]).toEqual({
|
||||
url: 'https://studio.itch.io/moon-garden',
|
||||
title: 'Moon Garden',
|
||||
image: 'https://img.itch.zone/aW1n/315x250%23c/moon-garden.png',
|
||||
genre: 'Puzzle',
|
||||
free: true,
|
||||
})
|
||||
expect(items[1]).toMatchObject({ title: 'Ink & Rain', genre: 'Visual Novel' })
|
||||
expect(items[2].free).toBe(false)
|
||||
expect(items[3].image).toBeNull() // only img.itch.zone covers are kept
|
||||
})
|
||||
})
|
||||
|
||||
describe('pickFeed', () => {
|
||||
const now = new Date('2026-09-18T12:00:00Z')
|
||||
const state = (feed: string, polled: string | null, backoff: string | null = null): [string, FeedState] => [
|
||||
feed,
|
||||
{ feed, last_polled_at: polled, backoff_until: backoff, last_status: 200 },
|
||||
]
|
||||
it('takes the feed polled longest ago that is not backing off', () => {
|
||||
expect(pickFeed(FEEDS, new Map(), now)?.id).toBe('free')
|
||||
expect(pickFeed(FEEDS, new Map([state('free', '2026-09-18T08:00:00Z')]), now)?.id).toBe('visual-novel')
|
||||
const both = new Map([state('free', '2026-09-18T04:00:00Z'), state('visual-novel', '2026-09-18T08:00:00Z')])
|
||||
expect(pickFeed(FEEDS, both, now)?.id).toBe('free')
|
||||
both.set(...state('free', '2026-09-18T04:00:00Z', '2026-09-18T14:00:00Z'))
|
||||
expect(pickFeed(FEEDS, both, now)?.id).toBe('visual-novel')
|
||||
both.set(...state('visual-novel', '2026-09-18T08:00:00Z', '2026-09-18T13:00:00Z'))
|
||||
expect(pickFeed(FEEDS, both, now)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('discover', () => {
|
||||
let db: D1Database
|
||||
let dispose: () => Promise<void>
|
||||
let store: QueueStore
|
||||
const now = new Date('2026-09-18T12:00:00Z')
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ db, dispose } = await testDb())
|
||||
}, 60_000)
|
||||
afterAll(() => dispose())
|
||||
beforeEach(async () => {
|
||||
resetDataCache()
|
||||
await db.batch(['candidates', 'feed_state'].map((t) => db.prepare(`DELETE FROM ${t}`)))
|
||||
store = new QueueStore(db)
|
||||
})
|
||||
|
||||
it('queues new free games from one feed, with hints and flags', async () => {
|
||||
const fetch = vi.fn(async () => new Response(FEED, { headers: { 'content-type': 'application/xml' } }))
|
||||
const result = await discover({
|
||||
store,
|
||||
data: fakeData(['https://studio.itch.io/already-listed']),
|
||||
fetch: fetch as unknown as typeof globalThis.fetch,
|
||||
userAgent: 'FreeItchGamesBot/test',
|
||||
now,
|
||||
})
|
||||
expect(result).toEqual({ feed: 'free', status: 200, items: 4, added: 3 })
|
||||
const [, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]
|
||||
expect((init.headers as Record<string, string>)['User-Agent']).toBe('FreeItchGamesBot/test')
|
||||
const rows = new Map((await store.list('pending')).map((r) => [r.url, r]))
|
||||
expect([...rows.keys()].sort()).toEqual([
|
||||
'https://studio.itch.io/ink-and-rain',
|
||||
'https://studio.itch.io/lewd-demo',
|
||||
'https://studio.itch.io/moon-garden',
|
||||
])
|
||||
expect(rows.get('https://studio.itch.io/moon-garden')).toMatchObject({
|
||||
source: 'rss',
|
||||
submitter: 'rss:free',
|
||||
genre_hint: 'Puzzle',
|
||||
title: 'Moon Garden',
|
||||
})
|
||||
expect(JSON.parse(rows.get('https://studio.itch.io/lewd-demo')!.flags)).toEqual(['nsfw_keyword', 'demo'])
|
||||
expect((await store.feeds()).get('free')).toMatchObject({ last_status: 200, backoff_until: null })
|
||||
})
|
||||
|
||||
it('backs off for at least 6 hours on 429, longer if asked', async () => {
|
||||
const fetch = vi.fn(async () => new Response(null, { status: 429, headers: { 'retry-after': '43200' } }))
|
||||
const result = await discover({ store, data: fakeData(), fetch: fetch as unknown as typeof globalThis.fetch, userAgent: 'x', now })
|
||||
expect(result.status).toBe(429)
|
||||
expect((await store.feeds()).get('free')?.backoff_until).toBe('2026-09-19T00:00:00Z')
|
||||
// Next run moves on to the other feed.
|
||||
const ok = vi.fn(async () => new Response('<rss></rss>'))
|
||||
expect((await discover({ store, data: fakeData(), fetch: ok as unknown as typeof globalThis.fetch, userAgent: 'x', now })).feed).toBe(
|
||||
'visual-novel',
|
||||
)
|
||||
})
|
||||
|
||||
it('records network errors without throwing', async () => {
|
||||
const fetch = vi.fn(async () => {
|
||||
throw new TypeError('Network connection lost.')
|
||||
})
|
||||
const result = await discover({ store, data: fakeData(), fetch: fetch as unknown as typeof globalThis.fetch, userAgent: 'x', now })
|
||||
expect(result).toMatchObject({ feed: 'free', status: 0, added: 0 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
/** Shared fakes for the Worker handler tests. */
|
||||
import type { DataSource } from '../data'
|
||||
|
||||
export const ORIGIN = 'https://freeitchgames.win'
|
||||
|
||||
/** Static assets holding a catalog (/data/urls.json) and a deleted log. */
|
||||
export function fakeData(
|
||||
catalog: string[] = [],
|
||||
deleted: { url: string; reason?: string; deleted_at?: string }[] = [],
|
||||
): DataSource {
|
||||
const urls = Object.fromEntries(catalog.map((u) => [u, 'https://img.itch.zone/x/original/y.png']))
|
||||
const log = deleted.map((d) => ({
|
||||
url: d.url,
|
||||
name: 'N',
|
||||
reason: d.reason ?? 'Game became paid',
|
||||
deleted_at: d.deleted_at ?? '2026-06-19T03:00:00Z',
|
||||
}))
|
||||
return {
|
||||
origin: ORIGIN,
|
||||
assets: {
|
||||
fetch: async (input: RequestInfo | URL) => {
|
||||
const path = new URL(input instanceof Request ? input.url : String(input)).pathname
|
||||
if (path === '/data/urls.json') return Response.json(urls)
|
||||
if (path === '/data/deleted_games.json') return Response.json(log)
|
||||
return new Response(null, { status: 404 })
|
||||
},
|
||||
} as unknown as Fetcher,
|
||||
}
|
||||
}
|
||||
|
||||
/** A rate limiter that allows `allowed` calls, then refuses. */
|
||||
export function fakeLimiter(allowed = Infinity) {
|
||||
let calls = 0
|
||||
const keys: string[] = []
|
||||
return {
|
||||
keys,
|
||||
limit: async ({ key }: { key: string }) => {
|
||||
keys.push(key)
|
||||
calls += 1
|
||||
return { success: calls <= allowed }
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { QueueStore } from '../queue'
|
||||
import { testDb } from './d1'
|
||||
|
||||
let db: D1Database
|
||||
let dispose: () => Promise<void>
|
||||
let store: QueueStore
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ db, dispose } = await testDb())
|
||||
}, 60_000)
|
||||
afterAll(() => dispose())
|
||||
beforeEach(async () => {
|
||||
await db.batch(['candidates', 'feed_state', 'audit_log', 'idempotency'].map((t) => db.prepare(`DELETE FROM ${t}`)))
|
||||
store = new QueueStore(db)
|
||||
})
|
||||
|
||||
const T0 = new Date('2026-09-18T00:00:00Z')
|
||||
const hours = (h: number) => new Date(T0.getTime() + h * 3_600_000)
|
||||
const url = (slug: string) => `https://dev.itch.io/${slug}`
|
||||
|
||||
describe('QueueStore', () => {
|
||||
it('stores candidates, reports statuses and lists newest first', async () => {
|
||||
await store.insert([{ url: url('a'), source: 'rss', title: 'A', flags: ['demo'] }], T0)
|
||||
await store.insert([{ url: url('b'), source: 'form', note: 'hi' }], hours(1))
|
||||
expect(await store.statuses([url('a'), url('b'), url('zzz')])).toEqual(
|
||||
new Map([
|
||||
[url('a'), 'pending'],
|
||||
[url('b'), 'pending'],
|
||||
]),
|
||||
)
|
||||
const rows = await store.list('pending')
|
||||
expect(rows.map((r) => r.url)).toEqual([url('b'), url('a')])
|
||||
expect(JSON.parse(rows[1].flags)).toEqual(['demo'])
|
||||
expect((await store.counts()).pending).toBe(2)
|
||||
})
|
||||
|
||||
it('never overwrites a live row, but reopens a failed one', async () => {
|
||||
await store.insert([{ url: url('a'), source: 'rss', title: 'Old' }], T0)
|
||||
await store.setStatus([url('a')], 'rejected', 'me@example.com', T0)
|
||||
await store.insert([{ url: url('a'), source: 'form', title: 'New' }], hours(1))
|
||||
expect((await store.get([url('a')]))[0]).toMatchObject({ status: 'rejected', source: 'rss', title: 'Old' })
|
||||
|
||||
await store.setStatus([url('a')], 'failed', 'reconcile', hours(2))
|
||||
await store.insert([{ url: url('a'), source: 'ext', note: 'again' }], hours(3))
|
||||
expect((await store.get([url('a')]))[0]).toMatchObject({
|
||||
status: 'pending',
|
||||
source: 'ext',
|
||||
title: 'Old', // kept when the new submission has none
|
||||
note: 'again',
|
||||
decided_at: null,
|
||||
discovered_at: '2026-09-18T03:00:00Z',
|
||||
})
|
||||
})
|
||||
|
||||
it('queries more URLs than one statement can bind', async () => {
|
||||
const urls = Array.from({ length: 250 }, (_, i) => url(`g${i}`))
|
||||
await store.insert(urls.map((u) => ({ url: u, source: 'rss' as const })), T0)
|
||||
expect((await store.statuses(urls)).size).toBe(250)
|
||||
expect((await store.get(urls)).length).toBe(250)
|
||||
})
|
||||
|
||||
it('reconciles queued rows: ingested when in the catalog, failed after 3 days', async () => {
|
||||
await store.insert([url('in'), url('slow'), url('fresh')].map((u) => ({ url: u, source: 'admin' as const })), T0)
|
||||
await store.setStatus([url('in'), url('slow')], 'queued', 'me', T0)
|
||||
await store.setStatus([url('fresh')], 'queued', 'me', hours(70))
|
||||
expect(await store.reconcile(new Set([url('in')]), hours(73))).toEqual({ ingested: 1, failed: 1 })
|
||||
expect(await store.statuses([url('in'), url('slow'), url('fresh')])).toEqual(
|
||||
new Map([
|
||||
[url('in'), 'ingested'],
|
||||
[url('slow'), 'failed'],
|
||||
[url('fresh'), 'queued'],
|
||||
]),
|
||||
)
|
||||
})
|
||||
|
||||
it('purges old idempotency records and old Suggest-form notes', async () => {
|
||||
await store.remember('ext', 'key-old-0001', 'h', '{}', T0)
|
||||
await store.remember('ext', 'key-new-0001', 'h', '{}', hours(24 * 6))
|
||||
await store.insert([{ url: url('form'), source: 'form', note: 'contact me' }], T0)
|
||||
await store.insert([{ url: url('ext'), source: 'ext', note: 'kept' }], T0)
|
||||
await store.purge(hours(24 * 181))
|
||||
expect(await store.idempotent('ext', 'key-old-0001')).toBeNull()
|
||||
expect(await store.idempotent('ext', 'key-new-0001')).toBeNull() // 175 days old by then
|
||||
const notes = new Map((await store.get([url('form'), url('ext')])).map((r) => [r.url, r.note]))
|
||||
expect(notes.get(url('form'))).toBeNull()
|
||||
expect(notes.get(url('ext'))).toBe('kept')
|
||||
})
|
||||
|
||||
it('keeps idempotency records per client and ignores a duplicate insert', async () => {
|
||||
await store.remember('ext-a', 'same-key-01', 'h1', '{"n":1}', T0)
|
||||
await store.remember('ext-a', 'same-key-01', 'h2', '{"n":2}', T0)
|
||||
await store.remember('ext-b', 'same-key-01', 'h3', '{"n":3}', T0)
|
||||
expect(await store.idempotent('ext-a', 'same-key-01')).toEqual({ body_hash: 'h1', response: '{"n":1}' })
|
||||
expect(await store.idempotent('ext-b', 'same-key-01')).toEqual({ body_hash: 'h3', response: '{"n":3}' })
|
||||
})
|
||||
|
||||
it('upserts feed state and writes the audit log', async () => {
|
||||
await store.saveFeed({ feed: 'free', last_polled_at: 'a', backoff_until: null, last_status: 200 })
|
||||
await store.saveFeed({ feed: 'free', last_polled_at: 'b', backoff_until: 'c', last_status: 429 })
|
||||
expect((await store.feeds()).get('free')).toEqual({ feed: 'free', last_polled_at: 'b', backoff_until: 'c', last_status: 429 })
|
||||
await store.audit('me@example.com', 'approve', url('a'), 'abc123')
|
||||
const row = await db.prepare('SELECT actor, action, target, commit_sha FROM audit_log').first()
|
||||
expect(row).toEqual({ actor: 'me@example.com', action: 'approve', target: url('a'), commit_sha: 'abc123' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,134 @@
|
||||
import { exportPKCS8, generateKeyPair } from 'jose'
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { dumps } from '../catalog'
|
||||
import { resetTokenCache, type GitHubEnv } from '../github'
|
||||
import { editCatalog, queueForIngest } from '../repo'
|
||||
|
||||
let env: GitHubEnv
|
||||
|
||||
beforeAll(async () => {
|
||||
const pair = await generateKeyPair('RS256', { extractable: true })
|
||||
env = {
|
||||
GITHUB_REPO: 'owner/repo',
|
||||
GH_APP_ID: '1',
|
||||
GH_APP_INSTALLATION_ID: '2',
|
||||
GH_APP_PRIVATE_KEY: await exportPKCS8(pair.privateKey),
|
||||
}
|
||||
})
|
||||
|
||||
const game = (slug: string, extra: Record<string, unknown> = {}) => ({
|
||||
url: `https://dev.itch.io/${slug}`,
|
||||
name: slug,
|
||||
thumbnail: `https://img.itch.zone/aW1n/original/${slug}.png`,
|
||||
nsfw: 'No',
|
||||
notes: '',
|
||||
safe_virus: '?',
|
||||
...extra,
|
||||
})
|
||||
|
||||
/** A tiny in-memory GitHub: files at one head, captures the commit mutation. */
|
||||
function fakeGitHub(files: Record<string, string>) {
|
||||
const commits: { headline: string; body?: string; additions: Record<string, string>; deletions: string[] }[] = []
|
||||
vi.stubGlobal('fetch', async (input: string, init: RequestInit = {}) => {
|
||||
const url = new URL(String(input))
|
||||
const path = url.pathname
|
||||
if (path.endsWith('/access_tokens')) {
|
||||
return Response.json({ token: 't', expires_at: new Date(Date.now() + 3_600_000).toISOString() })
|
||||
}
|
||||
if (path === '/repos/owner/repo/git/ref/heads/main') return Response.json({ object: { sha: 'head1' } })
|
||||
if (path === '/graphql') {
|
||||
const input = JSON.parse(String(init.body)).variables.input
|
||||
const decode = (b64: string) => new TextDecoder().decode(Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)))
|
||||
commits.push({
|
||||
headline: input.message.headline,
|
||||
body: input.message.body,
|
||||
additions: Object.fromEntries(input.fileChanges.additions.map((a: { path: string; contents: string }) => [a.path, decode(a.contents)])),
|
||||
deletions: input.fileChanges.deletions.map((d: { path: string }) => d.path),
|
||||
})
|
||||
return Response.json({ data: { createCommitOnBranch: { commit: { oid: 'new-sha' } } } })
|
||||
}
|
||||
const file = path.replace('/repos/owner/repo/contents/', '')
|
||||
if (url.searchParams.get('ref') !== 'head1') return new Response(null, { status: 404 })
|
||||
if (file === 'data_game') {
|
||||
return Response.json(
|
||||
Object.keys(files)
|
||||
.filter((f) => f.startsWith('data_game/'))
|
||||
.map((f) => ({ name: f.slice('data_game/'.length), type: 'file' })),
|
||||
)
|
||||
}
|
||||
return file in files ? new Response(files[file]) : new Response(null, { status: 404 })
|
||||
})
|
||||
return commits
|
||||
}
|
||||
|
||||
beforeEach(() => resetTokenCache())
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
describe('editCatalog', () => {
|
||||
const chunk = [game('a'), game('b'), game('c')]
|
||||
const files = {
|
||||
'data_game/game_info_001.json': dumps(chunk),
|
||||
'data_game/index.json': dumps({
|
||||
total_games: 3,
|
||||
max_per_file: 500,
|
||||
last_updated: '2026-09-01T00:00:00Z',
|
||||
files: [{ name: 'game_info_001.json', count: 3 }],
|
||||
}),
|
||||
'data_game/count_history.json': dumps([{ date: '2026-09-01', total: 3 }]),
|
||||
'scripts/deleted_games.json': dumps([]),
|
||||
}
|
||||
|
||||
it('removes a game in one commit: chunk, index, history, deleted log', async () => {
|
||||
const commits = fakeGitHub(files)
|
||||
const result = await editCatalog(env, new Map(), [{ url: 'https://dev.itch.io/b', reason: 'Takedown request' }], () => new Date('2026-09-18T12:00:00Z'))
|
||||
expect(result).toEqual({
|
||||
sha: 'new-sha',
|
||||
summary: { edited: 0, removed: 1 },
|
||||
thumbnails: ['https://img.itch.zone/aW1n/original/b.png'],
|
||||
})
|
||||
const [c] = commits
|
||||
expect(Object.keys(c.additions).sort()).toEqual([
|
||||
'data_game/count_history.json',
|
||||
'data_game/game_info_001.json',
|
||||
'data_game/index.json',
|
||||
'scripts/deleted_games.json',
|
||||
])
|
||||
expect(JSON.parse(c.additions['data_game/game_info_001.json']).map((g: { url: string }) => g.url)).toEqual([
|
||||
'https://dev.itch.io/a',
|
||||
'https://dev.itch.io/c',
|
||||
])
|
||||
expect(JSON.parse(c.additions['scripts/deleted_games.json'])).toEqual([
|
||||
{ url: 'https://dev.itch.io/b', name: 'b', reason: 'Takedown request', deleted_at: '2026-09-18T12:00:00Z' },
|
||||
])
|
||||
expect(`${c.headline}\n${c.body}`).not.toMatch(/@/) // never the maintainer's email
|
||||
})
|
||||
|
||||
it('edits one field and makes no commit when nothing changes', async () => {
|
||||
const commits = fakeGitHub(files)
|
||||
await editCatalog(env, new Map([['https://dev.itch.io/a', { nsfw: 'Yes' }]]), [])
|
||||
expect(Object.keys(commits[0].additions)).toEqual(['data_game/game_info_001.json', 'data_game/index.json'])
|
||||
const unchanged = await editCatalog(env, new Map([['https://dev.itch.io/a', { nsfw: 'No' }]]), [])
|
||||
expect(unchanged.sha).toBeNull()
|
||||
expect(commits).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('queueForIngest', () => {
|
||||
it('appends to temp_link.json and, when unblocking, prunes the deleted log', async () => {
|
||||
const commits = fakeGitHub({
|
||||
'scripts/temp_link.json': dumps(['https://dev.itch.io/queued']),
|
||||
'scripts/deleted_games.json': dumps([{ url: 'https://dev.itch.io/b', name: 'b', reason: 'r', deleted_at: 't' }]),
|
||||
})
|
||||
const { sha, added } = await queueForIngest(env, ['https://dev.itch.io/queued', 'https://dev.itch.io/b'], ['https://dev.itch.io/b'])
|
||||
expect(sha).toBe('new-sha')
|
||||
expect(added).toEqual(['https://dev.itch.io/b'])
|
||||
expect(JSON.parse(commits[0].additions['scripts/temp_link.json'])).toEqual(['https://dev.itch.io/queued', 'https://dev.itch.io/b'])
|
||||
expect(JSON.parse(commits[0].additions['scripts/deleted_games.json'])).toEqual([])
|
||||
})
|
||||
|
||||
it('makes no commit when everything is already queued', async () => {
|
||||
const commits = fakeGitHub({ 'scripts/temp_link.json': dumps(['https://dev.itch.io/queued']) })
|
||||
expect((await queueForIngest(env, ['https://dev.itch.io/queued'], [])).sha).toBeNull()
|
||||
expect(commits).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,176 @@
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { resetDataCache } from '../data'
|
||||
import type { WorkerEnv } from '../env'
|
||||
import { HttpError } from '../http'
|
||||
import { handleIngest } from '../ingest'
|
||||
import { QueueStore } from '../queue'
|
||||
import { handleSuggest } from '../suggest'
|
||||
import { testDb } from './d1'
|
||||
import { fakeData, fakeLimiter, ORIGIN } from './fakes'
|
||||
|
||||
let db: D1Database
|
||||
let dispose: () => Promise<void>
|
||||
let store: QueueStore
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ db, dispose } = await testDb())
|
||||
}, 60_000)
|
||||
afterAll(() => dispose())
|
||||
beforeEach(async () => {
|
||||
resetDataCache()
|
||||
await db.batch(['candidates', 'idempotency'].map((t) => db.prepare(`DELETE FROM ${t}`)))
|
||||
store = new QueueStore(db)
|
||||
})
|
||||
|
||||
const env = {
|
||||
SITE_ORIGIN: ORIGIN,
|
||||
TURNSTILE_SITEKEY: '0x4AAA-site',
|
||||
TURNSTILE_SECRET: 'secret',
|
||||
} as unknown as WorkerEnv
|
||||
|
||||
function siteverify(success: boolean, hostname = 'freeitchgames.win') {
|
||||
return vi.fn(async () => Response.json({ success, hostname }))
|
||||
}
|
||||
|
||||
function suggestRequest(body: unknown, headers: Record<string, string> = {}) {
|
||||
return new Request(`${ORIGIN}/api/suggest`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', origin: ORIGIN, 'cf-connecting-ip': '203.0.113.9', ...headers },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
}
|
||||
|
||||
describe('POST /api/suggest', () => {
|
||||
const deps = (over: Partial<Parameters<typeof handleSuggest>[2]> = {}) => ({
|
||||
store,
|
||||
data: fakeData(['https://dev.itch.io/listed'], [{ url: 'https://dev.itch.io/gone' }]),
|
||||
fetch: siteverify(true) as unknown as typeof fetch,
|
||||
limiter: fakeLimiter(),
|
||||
...over,
|
||||
})
|
||||
|
||||
it('tells the page whether the form is enabled', async () => {
|
||||
const on = await handleSuggest(new Request(`${ORIGIN}/api/suggest`), env, deps())
|
||||
expect(await on.json()).toEqual({ enabled: true, sitekey: '0x4AAA-site' })
|
||||
const off = await handleSuggest(new Request(`${ORIGIN}/api/suggest`), { ...env, TURNSTILE_SECRET: undefined }, deps())
|
||||
expect(await off.json()).toEqual({ enabled: false, sitekey: null })
|
||||
})
|
||||
|
||||
it('queues a new game with its note', async () => {
|
||||
const fetch = siteverify(true)
|
||||
const res = await handleSuggest(
|
||||
suggestRequest({ url: 'https://Dev.itch.io/New-Game/', note: ' cozy puzzle ', token: 't' }),
|
||||
env,
|
||||
deps({ fetch: fetch as unknown as typeof globalThis.fetch }),
|
||||
)
|
||||
expect(await res.json()).toEqual({ status: 'received' })
|
||||
const [row] = await store.get(['https://dev.itch.io/new-game'])
|
||||
expect(row).toMatchObject({ source: 'form', note: 'cozy puzzle', submitter: 'form', status: 'pending' })
|
||||
const form = (fetch.mock.calls[0] as unknown as [string, RequestInit])[1].body as FormData
|
||||
expect(form.get('remoteip')).toBe('203.0.113.9')
|
||||
})
|
||||
|
||||
it('answers what is already known without storing it again', async () => {
|
||||
const listed = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/listed', token: 't' }), env, deps())
|
||||
expect(await listed.json()).toEqual({ status: 'already_listed' })
|
||||
await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/twice', token: 't' }), env, deps())
|
||||
const again = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/twice', token: 't' }), env, deps())
|
||||
expect(await again.json()).toEqual({ status: 'already_suggested' })
|
||||
const gone = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/gone', token: 't' }), env, deps())
|
||||
expect(await gone.json()).toEqual({ status: 'received' })
|
||||
expect(JSON.parse((await store.get(['https://dev.itch.io/gone']))[0].flags)).toEqual(['previously_deleted'])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['a failed challenge', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), { fetch: siteverify(false) }, 403],
|
||||
['a token for another site', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), { fetch: siteverify(true, 'evil.example') }, 403],
|
||||
['a cross-site post', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }, { origin: 'https://evil.example' }), {}, 403],
|
||||
['a non-itch URL', suggestRequest({ url: 'https://example.com/game', token: 't' }), {}, 400],
|
||||
['an over-long note', suggestRequest({ url: 'https://dev.itch.io/x', note: 'x'.repeat(501), token: 't' }), {}, 400],
|
||||
['the rate limit', suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), { limiter: fakeLimiter(0) }, 429],
|
||||
])('refuses %s', async (_name, request, over, status) => {
|
||||
const res = await handleSuggest(request, env, deps(over as Parameters<typeof deps>[0]))
|
||||
expect(res.status).toBe(status)
|
||||
expect(await store.counts()).toMatchObject({ pending: 0 })
|
||||
})
|
||||
|
||||
it('is unavailable until configured', async () => {
|
||||
const res = await handleSuggest(suggestRequest({ url: 'https://dev.itch.io/x', token: 't' }), env, deps({ store: null }))
|
||||
expect(res.status).toBe(503)
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /api/ingest', () => {
|
||||
const deps = (over: Partial<Parameters<typeof handleIngest>[1]> = {}) => ({
|
||||
store,
|
||||
data: fakeData(['https://dev.itch.io/listed']),
|
||||
limiter: fakeLimiter(),
|
||||
authenticate: async () => 'ext-client.access',
|
||||
...over,
|
||||
})
|
||||
const ingestRequest = (body: unknown, key = 'key-0000-0001') =>
|
||||
new Request(`${ORIGIN}/api/ingest`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'idempotency-key': key },
|
||||
body: typeof body === 'string' ? body : JSON.stringify(body),
|
||||
})
|
||||
|
||||
it('queues items and reports each outcome, attributed to the token', async () => {
|
||||
const res = await handleIngest(
|
||||
ingestRequest({
|
||||
items: [
|
||||
{ url: 'https://dev.itch.io/fresh', title: 'Fresh Demo', note: 'from a jam' },
|
||||
{ url: 'https://dev.itch.io/listed' },
|
||||
'https://DEV.itch.io/fresh/',
|
||||
'nope',
|
||||
],
|
||||
}),
|
||||
deps(),
|
||||
)
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.json()).toEqual({
|
||||
received: 1,
|
||||
results: [
|
||||
{ url: 'https://dev.itch.io/fresh', canonical: 'https://dev.itch.io/fresh', outcome: 'queued' },
|
||||
{ url: 'https://dev.itch.io/listed', canonical: 'https://dev.itch.io/listed', outcome: 'duplicate_catalog' },
|
||||
{ url: 'https://DEV.itch.io/fresh/', canonical: 'https://dev.itch.io/fresh', outcome: 'duplicate_in_request' },
|
||||
{ url: 'nope', canonical: null, outcome: 'invalid_url' },
|
||||
],
|
||||
})
|
||||
const [row] = await store.get(['https://dev.itch.io/fresh'])
|
||||
expect(row).toMatchObject({ source: 'ext', submitter: 'ext-client.access', title: 'Fresh Demo', note: 'from a jam' })
|
||||
expect(JSON.parse(row.flags)).toEqual(['demo'])
|
||||
})
|
||||
|
||||
it('replays a retry with the same key and refuses the key with another body', async () => {
|
||||
const first = await handleIngest(ingestRequest({ urls: ['https://dev.itch.io/a'] }), deps())
|
||||
const again = await handleIngest(ingestRequest({ urls: ['https://dev.itch.io/a'] }), deps())
|
||||
expect(again.headers.get('idempotent-replayed')).toBe('true')
|
||||
expect(await again.json()).toEqual(await first.json())
|
||||
const other = await handleIngest(ingestRequest({ urls: ['https://dev.itch.io/b'] }), deps())
|
||||
expect(other.status).toBe(422)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['no idempotency key', ingestRequest({ urls: ['https://dev.itch.io/a'] }, ''), {}, 400],
|
||||
['too many items', ingestRequest({ urls: Array.from({ length: 151 }, (_, i) => `https://dev.itch.io/g${i}`) }), {}, 413],
|
||||
['a body over 64 KB', ingestRequest({ urls: ['https://dev.itch.io/a'], pad: 'x'.repeat(70_000) }), {}, 413],
|
||||
['invalid JSON', ingestRequest('{nope'), {}, 400],
|
||||
['an empty list', ingestRequest({ items: [] }), {}, 400],
|
||||
['the rate limit', ingestRequest({ urls: ['https://dev.itch.io/a'] }), { limiter: fakeLimiter(0) }, 429],
|
||||
[
|
||||
'a failed Access check',
|
||||
ingestRequest({ urls: ['https://dev.itch.io/a'] }),
|
||||
{
|
||||
authenticate: async () => {
|
||||
throw new HttpError(403, 'forbidden')
|
||||
},
|
||||
},
|
||||
403,
|
||||
],
|
||||
])('refuses %s', async (_name, request, over, status) => {
|
||||
const res = await handleIngest(request, deps(over as Parameters<typeof deps>[0]))
|
||||
expect(res.status).toBe(status)
|
||||
expect(await store.counts()).toMatchObject({ pending: 0 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,6 @@
|
||||
// Bindings for the D1 tests only (worker/test/d1.ts, in-memory, never deployed).
|
||||
{
|
||||
"name": "freeitchgames-test",
|
||||
"compatibility_date": "2026-09-15",
|
||||
"d1_databases": [{ "binding": "DB", "database_name": "test", "database_id": "00000000-0000-0000-0000-000000000000" }]
|
||||
}
|
||||
+59
-5
@@ -3,17 +3,23 @@
|
||||
// Root directory: webapp · Build command: npm run build · Deploy command: npx wrangler deploy
|
||||
// Custom Domains (freeitchgames.win, www) are attached in the dashboard and are
|
||||
// intentionally not declared here yet, so a deploy never touches them.
|
||||
//
|
||||
// Secrets are NOT in this file: `npx wrangler secret put <NAME>` (and
|
||||
// webapp/.dev.vars for `wrangler dev`) — GH_APP_PRIVATE_KEY, ADMIN_EMAILS,
|
||||
// TURNSTILE_SECRET. Until a feature's settings are filled in, its routes
|
||||
// answer 503 (see worker/env.ts); the public site works without any of them.
|
||||
// After editing this file run `npm run cf:types`.
|
||||
{
|
||||
"$schema": "./node_modules/wrangler/config-schema.json",
|
||||
// MUST match the Worker name in the dashboard or Workers Builds fails.
|
||||
"name": "free-games-itchio-list",
|
||||
"main": "./worker/index.ts",
|
||||
"compatibility_date": "2026-09-15",
|
||||
// workers.dev / preview URLs would bypass Cloudflare Access (admin, Phase 4).
|
||||
// workers.dev / preview URLs would bypass Cloudflare Access (admin, ingest).
|
||||
"workers_dev": false,
|
||||
"preview_urls": false,
|
||||
"assets": {
|
||||
// Vite build output (app + bundled catalog under /data). Never a source dir.
|
||||
// Vite build output (app + admin + bundled catalog under /data). Never a source dir.
|
||||
"directory": "./dist",
|
||||
"binding": "ASSETS",
|
||||
// Misses go to the Worker, not to an asset-layer SPA fallback: that would
|
||||
@@ -23,22 +29,70 @@
|
||||
// (worker/spa.ts).
|
||||
"not_found_handling": "none",
|
||||
"html_handling": "auto-trailing-slash",
|
||||
// Existing assets never invoke the Worker (free, cached); these paths always do.
|
||||
"run_worker_first": ["/img/*", "/api/*"]
|
||||
// Existing assets never invoke the Worker (free, cached); these paths always
|
||||
// do — /admin so its shell is only ever served after the Access JWT check.
|
||||
"run_worker_first": ["/img/*", "/api/*", "/admin", "/admin/*"]
|
||||
},
|
||||
// Resized cover images, written once per image + width by worker/img.ts.
|
||||
"r2_buckets": [{ "binding": "THUMBS", "bucket_name": "freeitchgames-thumbs" }],
|
||||
// Review queue (worker/migrations). No database_id: `wrangler deploy` binds the
|
||||
// database by name (and creates it on the first deploy if it does not exist).
|
||||
// Migrations are applied by hand, never by the build:
|
||||
// npx wrangler d1 migrations apply freeitchgames --remote
|
||||
"d1_databases": [
|
||||
{ "binding": "DB", "database_name": "freeitchgames", "migrations_dir": "./worker/migrations" }
|
||||
],
|
||||
// Per-key limits (IP for the Suggest form, service-token id for ingest).
|
||||
"ratelimits": [
|
||||
{ "name": "RL_SUGGEST", "namespace_id": "1001", "simple": { "limit": 5, "period": 60 } },
|
||||
{ "name": "RL_INGEST", "namespace_id": "1002", "simple": { "limit": 20, "period": 60 } }
|
||||
],
|
||||
// RSS discovery (one feed per run) + queue bookkeeping, worker/index.ts scheduled().
|
||||
"triggers": { "crons": ["23 */4 * * *"] },
|
||||
"vars": {
|
||||
"SITE_ORIGIN": "https://freeitchgames.win",
|
||||
"GITHUB_REPO": "poli0981/free-games-itchio-list",
|
||||
"BOT_UA": "FreeItchGamesBot/4.0 (+https://freeitchgames.win/about)",
|
||||
// Filled in during setup (none of these are secret):
|
||||
"GH_APP_ID": "",
|
||||
"GH_APP_INSTALLATION_ID": "",
|
||||
"ACCESS_TEAM_DOMAIN": "", // https://<team>.cloudflareaccess.com
|
||||
"ACCESS_AUD_ADMIN": "", // AUD tag of the "Admin" Access application
|
||||
"ACCESS_AUD_INGEST": "", // AUD tag of the "Ingest" Access application
|
||||
"TURNSTILE_SITEKEY": ""
|
||||
},
|
||||
"observability": { "enabled": true, "head_sampling_rate": 0.1 },
|
||||
"upload_source_maps": false,
|
||||
|
||||
"env": {
|
||||
// Pre-release checks of the v4 branch: Worker "free-games-itchio-list-staging",
|
||||
// deployed with `npx wrangler deploy --env staging`, whole host behind Access.
|
||||
// Bindings and vars are not inherited, so every one is repeated here.
|
||||
"staging": {
|
||||
"workers_dev": false,
|
||||
"preview_urls": false,
|
||||
"routes": [{ "pattern": "staging.freeitchgames.win", "custom_domain": true }],
|
||||
"r2_buckets": [{ "binding": "THUMBS", "bucket_name": "freeitchgames-thumbs-staging" }]
|
||||
"r2_buckets": [{ "binding": "THUMBS", "bucket_name": "freeitchgames-thumbs-staging" }],
|
||||
"d1_databases": [
|
||||
{ "binding": "DB", "database_name": "freeitchgames-staging", "migrations_dir": "./worker/migrations" }
|
||||
],
|
||||
"ratelimits": [
|
||||
{ "name": "RL_SUGGEST", "namespace_id": "2001", "simple": { "limit": 5, "period": 60 } },
|
||||
{ "name": "RL_INGEST", "namespace_id": "2002", "simple": { "limit": 20, "period": 60 } }
|
||||
],
|
||||
// No RSS polling from staging: production is the only itch.io client.
|
||||
"triggers": { "crons": [] },
|
||||
"vars": {
|
||||
"SITE_ORIGIN": "https://staging.freeitchgames.win",
|
||||
"GITHUB_REPO": "poli0981/free-games-itchio-list",
|
||||
"BOT_UA": "FreeItchGamesBot/4.0 (+https://freeitchgames.win/about)",
|
||||
"GH_APP_ID": "",
|
||||
"GH_APP_INSTALLATION_ID": "",
|
||||
"ACCESS_TEAM_DOMAIN": "",
|
||||
"ACCESS_AUD_ADMIN": "",
|
||||
"ACCESS_AUD_INGEST": "",
|
||||
"TURNSTILE_SITEKEY": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user