fix: Phase 4 review findings (admin, ingest, suggest)

Worker
- The local DEV_ADMIN_EMAIL bypass keeps the same-origin check for
  writes, and JSON bodies must be exactly application/json (a cross-site
  `text/plain; application/json` POST no longer gets through without a
  preflight).
- Request bodies are capped while they stream (a chunked upload without
  Content-Length is never buffered past 4 KB / 64 KB).
- count_history.json is rewritten when today's row already exists
  (upsertCountHistory no longer mutates the rows it was given).
- A game that entered the catalog through the queue, was removed and turns
  up again is reviewed again (flagged previously removed) instead of being
  blocked as "already pending" for good.
- Restoring / approving a removed game no longer deletes its public
  removal record up front: the Worker allow-lists it in
  scripts/state/unblocked.json, update_info.py scrapes it despite the
  deleted log, and apply_patch.py drops the log entry only when the game is
  really re-added (a still-paid or dead game keeps its record).

Admin app
- The editor compares against what the repository holds (published data
  or what was saved in this session), so saved changes can be reverted and
  are never dropped silently.
- An Access session that expires mid-session shows a sign-in banner from
  any request, without losing what is on the page.
- Restore refreshes the queue; knip now checks the admin entry.

Suggest page
- The form doesn't let the browser's URL check block links without
  https:// (the page and the server accept them).
- A failed config request shows an error with Retry instead of a
  permanently disabled form or a false "closed" message.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5 committed 2026-09-18 17:55:02 +07:00
1 parent 369de32c2b
commit 41a8f8461f
21 files changed
+399 -66

No files matched your search

+17
View File
@@ -12,6 +12,9 @@ always works on the latest data. Idempotent: applying the same patch twice
- *_state: merged into scripts/state/*.json (null drops an entry); for
refresh_state the newer `checked` wins and the newest `full`
stamp from either side is kept
- unlog / unblock_remove: an unblocked (restored) game that was re-added
leaves the deleted log and scripts/state/unblocked.json; one whose
ingest ended otherwise leaves only the allow-list
- queue_add / queue_remove: appended to / dropped from scripts/temp_link.json,
which is re-read here so URLs queued meanwhile survive. Entries
that are not URL strings are dropped on every apply.
@@ -43,6 +46,7 @@ DELETED_LOG = "scripts/deleted_games.json"
TEMP_LINK = "scripts/temp_link.json"
REFRESH_STATE = "scripts/state/refresh_state.json"
INGEST_STATE = "scripts/state/ingest_state.json"
UNBLOCKED = "scripts/state/unblocked.json"
MASS_REMOVAL_FLOOR = 25
MASS_REMOVAL_SHARE = 0.05
@@ -157,6 +161,19 @@ def apply(patch: dict) -> dict:
state = {u: e for u, e in state.items() if u in live_urls}
save_state_map(path, state)
unlog = {u for u in patch.get("unlog", []) if isinstance(u, str)}
finished = unlog | {u for u in patch.get("unblock_remove", []) if isinstance(u, str)}
if unlog:
log = load_json(DELETED_LOG)
kept = [e for e in log if e.get("url") not in unlog]
if len(kept) != len(log):
save_json(DELETED_LOG, kept)
if finished and os.path.exists(UNBLOCKED):
allowed = load_json(UNBLOCKED)
remaining = sorted(u for u in allowed if u not in finished)
if remaining != allowed:
save_json(UNBLOCKED, remaining)
raw_queue = load_json(TEMP_LINK)
queue = clean_queue(raw_queue)
present = {_queue_key(u) for u in queue}
+8
View File
@@ -18,6 +18,10 @@ Patch v1:
"ingest_state": {"<url>": {...} | null},
"queue_add": ["<url>", ...], # append to temp_link.json if absent
"queue_remove": ["<url>", ...], # drop from temp_link.json
"unlog": ["<url>", ...], # unblocked + re-added: drop from the
# deleted log and state/unblocked.json
"unblock_remove": ["<url>", ...], # unblocked, ended otherwise: drop from
# state/unblocked.json only
"stats": {...} # informational
}
"""
@@ -40,6 +44,8 @@ def new_patch(kind: str) -> dict:
"ingest_state": {},
"queue_add": [],
"queue_remove": [],
"unlog": [],
"unblock_remove": [],
"stats": {},
}
@@ -67,5 +73,7 @@ def is_empty(patch: dict) -> bool:
"ingest_state",
"queue_add",
"queue_remove",
"unlog",
"unblock_remove",
)
)
+12 -1
View File
@@ -42,6 +42,10 @@ from scraper import NA, Pacer, create_session, now_iso, parse_game
TEMP_LINK = "scripts/temp_link.json"
DELETED_LOG = "scripts/deleted_games.json"
INGEST_STATE = "scripts/state/ingest_state.json"
# Removed games the maintainer asked to bring back (admin Restore / approve with
# override). They are scraped despite the deleted log; the log entry goes only
# once the game is really re-added, so a still-paid game keeps its record.
UNBLOCKED = "scripts/state/unblocked.json"
MAX_ATTEMPTS = 3
RETRY_GAP = timedelta(hours=6)
DEAD_CODES = {404, 410}
@@ -95,6 +99,7 @@ def main(argv: list[str] | None = None) -> int:
catalog_urls = get_all_urls(load_all_games())
deleted_urls = {e.get("url") for e in load_json(DELETED_LOG)}
unblocked = set(load_json(UNBLOCKED, default=[]))
ingest_state: dict = load_json(INGEST_STATE, default={})
todo: list[str] = []
@@ -113,7 +118,9 @@ def main(argv: list[str] | None = None) -> int:
if url in catalog_urls:
print(f"Skip duplicate: {url}")
stats["duplicate"] += 1
elif url in deleted_urls:
if url in unblocked:
patch["unlog"].append(url)
elif url in deleted_urls and url not in unblocked:
print(f"Skip previously deleted: {url} (restore it from the admin page)")
stats["deleted"] += 1
else:
@@ -161,6 +168,8 @@ def main(argv: list[str] | None = None) -> int:
patch["queue_remove"].append(url)
patch["ingest_state"][url] = {**entry, "gave_up": entry["last_attempt"]}
stats["gave_up"] += 1
if url in unblocked:
patch["unblock_remove"].append(url)
else:
print(f" → {outcome}; stays queued ({entry['attempts']}/{MAX_ATTEMPTS} attempts).")
patch["ingest_state"][url] = entry
@@ -171,6 +180,8 @@ def main(argv: list[str] | None = None) -> int:
stats[outcome] += 1
patch["queue_remove"].append(url)
patch["ingest_state"][url] = None
if url in unblocked:
patch["unlog" if outcome == "added" else "unblock_remove"].append(url)
write_patch(args.out, patch)
print(
+44
View File
@@ -201,3 +201,47 @@ def test_throttled_page_is_not_an_attempt(repo, monkeypatch):
patch = _run(monkeypatch, {url: Page(status=429)}, [], INPUT_URL=url)
assert url not in patch["ingest_state"] and url not in patch["queue_remove"]
assert patch["queue_add"] == [url]
def _deleted(slug):
return {
"url": f"https://dev.itch.io/{slug}",
"name": slug,
"reason": "Game became paid",
"deleted_at": "2026-03-13T00:00:00Z",
}
def test_unblocked_game_that_is_free_again_leaves_the_deleted_log(repo, monkeypatch):
seed_catalog([make_game("known")])
url = "https://dev.itch.io/free"
save_json("scripts/deleted_games.json", [_deleted("free"), _deleted("other")])
save_json("scripts/state/unblocked.json", [url])
pages = {url: Page(status=200, soup=load_fixture_soup("game_free_full.html"))}
patch = _run(monkeypatch, pages, [url])
assert [a["url"] for a in patch["additions"]] == [url] and patch["unlog"] == [url]
_apply(patch)
assert [e["url"] for e in read_json("scripts/deleted_games.json")] == [
"https://dev.itch.io/other"
]
assert read_json("scripts/state/unblocked.json") == []
def test_unblocked_game_that_is_still_paid_keeps_its_removal_record(repo, monkeypatch):
seed_catalog([make_game("known")])
url = "https://dev.itch.io/paid"
save_json("scripts/deleted_games.json", [_deleted("paid")])
save_json("scripts/state/unblocked.json", [url])
pages = {url: Page(status=200, soup=load_fixture_soup("game_paid.html"))}
patch = _run(monkeypatch, pages, [url])
assert patch["additions"] == [] and patch["unblock_remove"] == [url]
_apply(patch)
assert [e["url"] for e in read_json("scripts/deleted_games.json")] == [url]
assert read_json("scripts/state/unblocked.json") == []
def test_deleted_game_without_unblock_is_still_skipped(repo, monkeypatch):
seed_catalog([make_game("known")])
save_json("scripts/deleted_games.json", [_deleted("paid")])
patch = _run(monkeypatch, {}, ["https://dev.itch.io/paid"])
assert patch["stats"]["deleted"] == 1 and patch["unlog"] == []
+20 -1
View File
@@ -11,7 +11,7 @@ export class SessionExpired extends Error {
}
}
export class ApiError extends Error {
class ApiError extends Error {
readonly status: number
readonly code: string
@@ -22,6 +22,25 @@ export class ApiError extends Error {
}
}
// Set once any request finds the Access session gone; the app shows a sign-in banner.
let expired = false
const listeners = new Set<() => void>()
export function markSessionExpired(): void {
if (expired) return
expired = true
for (const listener of listeners) listener()
}
export function subscribeSession(listener: () => void): () => void {
listeners.add(listener)
return () => listeners.delete(listener)
}
export function isSessionExpired(): boolean {
return expired
}
async function api<T>(method: string, path: string, body?: unknown): Promise<T> {
const res = await fetch(path, {
method,
+16 -1
View File
@@ -1,7 +1,8 @@
import { useSyncExternalStore } from 'react'
import { useQuery } from '@tanstack/react-query'
import { NavLink, Navigate, Route, Routes } from 'react-router'
import { cn } from '@/lib/utils'
import { SessionExpired, adminApi, errorText } from './api'
import { SessionExpired, adminApi, errorText, isSessionExpired, subscribeSession } from './api'
import { AddPage } from './add'
import { CatalogPage } from './catalog'
import { QueuePage } from './queue'
@@ -14,6 +15,7 @@ const NAV = [
export function AdminApp() {
const me = useQuery({ queryKey: ['me'], queryFn: adminApi.me })
const expired = useSyncExternalStore(subscribeSession, isSessionExpired)
if (me.error) {
return (
@@ -56,6 +58,19 @@ export function AdminApp() {
</a>
</div>
</header>
{expired && (
<div role="alert" className="border-b border-destructive/40 bg-destructive/10 px-4 py-2 text-sm">
<div className="mx-auto flex max-w-6xl flex-wrap items-center gap-3">
<span>Your admin session has expired. Sign in again, then retry — nothing on this page is lost.</span>
<a className="font-medium underline" href="/admin/" target="_blank" rel="noopener">
Sign in (new tab)
</a>
<button className="underline" onClick={() => window.location.reload()}>
Reload this page
</button>
</div>
</div>
)}
<main className="mx-auto max-w-6xl p-4">
<Routes>
<Route path="/queue" element={<QueuePage />} />
+55 -19
View File
@@ -1,5 +1,5 @@
import { useMemo, useState } from 'react'
import { useMutation } from '@tanstack/react-query'
import { useMutation, useQueryClient } from '@tanstack/react-query'
import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input'
import { useAllGames, useDeletedGames } from '@/hooks/useGames'
@@ -21,6 +21,9 @@ export function CatalogPage() {
const [query, setQuery] = useState('')
const [selected, setSelected] = useState<Game | null>(null)
const [tab, setTab] = useState<'catalog' | 'removed'>('catalog')
// Values saved in this session, per game: the published /data lags behind
// the repository until the next build, so it can't be the baseline.
const [saved] = useState(() => new Map<string, EditableValues>())
const found = useMemo(
() => (games.data?.games ?? []).filter((g) => matches(query, g.name, g.url, g.dev)).slice(0, MAX_RESULTS),
@@ -77,30 +80,56 @@ export function CatalogPage() {
<RemovedList entries={removed} />
)}
</div>
<div>{selected && tab === 'catalog' && <GameEditor key={selected.url} game={selected} />}</div>
<div>
{selected && tab === 'catalog' && (
<GameEditor
key={selected.url}
game={selected}
known={saved.get(selected.url) ?? published(selected)}
onSaved={(values) => saved.set(selected.url, values)}
/>
)}
</div>
</section>
)
}
function GameEditor({ game }: { game: Game }) {
const [edit, setEdit] = useState<Required<GameEdit>>({
safe_virus: game.safe_virus,
nsfw: game.nsfw,
notes: game.notes ?? '',
})
type EditableValues = Required<GameEdit>
const EDITABLE = ['safe_virus', 'nsfw', 'notes'] as const
function published(game: Game): EditableValues {
return { safe_virus: game.safe_virus, nsfw: game.nsfw, notes: game.notes ?? '' }
}
/**
* `known` is what the repository holds as far as this page knows: the
* published data, or what was saved in this session. Changes are computed
* against it, so a saved edit can be reverted and never silently dropped.
*/
function GameEditor({ game, known, onSaved }: { game: Game; known: EditableValues; onSaved: (v: EditableValues) => void }) {
const [baseline, setBaseline] = useState<EditableValues>(known)
const [edit, setEdit] = useState<EditableValues>(known)
const [reason, setReason] = useState('')
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
const changed = (Object.keys(edit) as (keyof GameEdit)[]).filter((k) => edit[k] !== (game[k] ?? ''))
const changed = EDITABLE.filter((k) => edit[k] !== baseline[k])
const save = useMutation({
mutationFn: () => adminApi.edit(game.url, Object.fromEntries(changed.map((k) => [k, edit[k]]))),
onSuccess: (r) =>
mutationFn: (sent: GameEdit) => adminApi.edit(game.url, sent),
onSuccess: (r, sent) => {
const next = { ...baseline, ...sent }
setBaseline(next)
onSaved(next)
setNotice({
tone: 'info',
text: r.edited ? `Saved (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Nothing changed in the repository.',
}),
})
},
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
})
const change = (next: EditableValues) => {
setEdit(next)
if (notice?.tone === 'info') setNotice(null)
}
const remove = useMutation({
mutationFn: () => adminApi.remove(game.url, reason.trim()),
onSuccess: (r) => setNotice({ tone: 'info', text: `Removed (${shortSha(r.sha)}); its resized covers were deleted. ${PUBLISH_NOTE}` }),
@@ -120,7 +149,7 @@ function GameEditor({ game }: { game: Game }) {
<span className="text-xs text-muted-foreground">Virus check</span>
<select
value={edit.safe_virus}
onChange={(e) => setEdit({ ...edit, safe_virus: e.target.value })}
onChange={(e) => change({ ...edit, safe_virus: e.target.value })}
className="h-9 rounded-md border bg-background px-2"
>
{['?', 'Yes', 'No', 'Caution'].map((v) => (
@@ -134,7 +163,7 @@ function GameEditor({ game }: { game: Game }) {
<span className="text-xs text-muted-foreground">NSFW (18+)</span>
<select
value={edit.nsfw}
onChange={(e) => setEdit({ ...edit, nsfw: e.target.value })}
onChange={(e) => change({ ...edit, nsfw: e.target.value })}
className="h-9 rounded-md border bg-background px-2"
>
<option value="No">No</option>
@@ -148,11 +177,15 @@ function GameEditor({ game }: { game: Game }) {
value={edit.notes}
maxLength={2000}
rows={3}
onChange={(e) => setEdit({ ...edit, notes: e.target.value })}
onChange={(e) => change({ ...edit, notes: e.target.value })}
className="rounded-md border bg-background p-2"
/>
</label>
<Button className="mt-3" disabled={!changed.length || save.isPending} onClick={() => save.mutate()}>
<Button
className="mt-3"
disabled={!changed.length || save.isPending}
onClick={() => save.mutate(Object.fromEntries(changed.map((k) => [k, edit[k]])))}
>
{save.isPending ? 'Saving…' : 'Save'}
</Button>
@@ -180,13 +213,16 @@ function GameEditor({ game }: { game: Game }) {
function RemovedList({ entries }: { entries: { url: string; name: string; reason: string; deleted_at: string }[] }) {
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
const client = useQueryClient()
const restore = useMutation({
mutationFn: (url: string) => adminApi.restore(url),
onSuccess: (r) =>
onSuccess: (r) => {
setNotice({
tone: 'info',
text: r.queued ? `Queued for re-ingest (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Already queued.',
}),
})
void client.invalidateQueries({ queryKey: ['queue'] })
},
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
})
return (
@@ -206,7 +242,7 @@ function RemovedList({ entries }: { entries: { url: string; name: string; reason
variant="outline"
disabled={restore.isPending}
onClick={() => {
if (window.confirm(`Bring "${d.name}" back? It is re-scraped; a paid game is dropped again.`)) {
if (window.confirm(`Bring "${d.name}" back? It is re-scraped; if it is still paid or gone, it stays on the removed list.`)) {
restore.mutate(d.url)
}
}}
+8 -2
View File
@@ -1,10 +1,10 @@
import { StrictMode } from 'react'
import { createRoot } from 'react-dom/client'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { MutationCache, QueryCache, QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { HashRouter } from 'react-router'
import '@/index.css'
import { AdminApp } from './app'
import { SessionExpired } from './api'
import { SessionExpired, markSessionExpired } from './api'
// Follow the system theme (the admin has no settings of its own).
const dark = window.matchMedia('(prefers-color-scheme: dark)')
@@ -12,7 +12,13 @@ const applyTheme = () => document.documentElement.classList.toggle('dark', dark.
applyTheme()
dark.addEventListener('change', applyTheme)
const onError = (error: unknown) => {
if (error instanceof SessionExpired) markSessionExpired()
}
const queryClient = new QueryClient({
queryCache: new QueryCache({ onError }),
mutationCache: new MutationCache({ onError }),
defaultOptions: {
queries: {
staleTime: 30_000,
+5 -1
View File
@@ -2,6 +2,7 @@
"$schema": "https://unpkg.com/knip@6/schema.json",
"project": [
"src/**/*.{ts,tsx}",
"admin/**/*.{ts,tsx}",
"worker/**/*.ts",
"vite-plugins/**/*.ts"
],
@@ -9,5 +10,8 @@
"@/*": [
"./src/*"
]
}
},
"entry": [
"admin/main.tsx"
]
}
+1
View File
@@ -288,6 +288,7 @@ export const en = {
"settings.content.title": "Content",
"settings.content.nsfw": "Show 18+ games",
"settings.content.nsfwDesc": "Off by default. Turning it on asks you to confirm you are 18 or older; the choice stays in this browser.",
"suggest.error.config": "Couldn't reach the site to load the form. Check your connection or content blocker, then try again.",
} as const
export type MessageKey = keyof typeof en
+1
View File
@@ -290,4 +290,5 @@ export const vi: Record<MessageKey, string> = {
"settings.content.title": "Nội dung",
"settings.content.nsfw": "Hiện game 18+",
"settings.content.nsfwDesc": "Mặc định tắt. Khi bật, bạn sẽ được yêu cầu xác nhận đủ 18 tuổi; lựa chọn chỉ lưu trong trình duyệt này.",
"suggest.error.config": "Không kết nối được tới trang để tải biểu mẫu. Hãy kiểm tra kết nối hoặc trình chặn nội dung rồi thử lại.",
}
+18 -4
View File
@@ -54,11 +54,14 @@ function SuggestForm() {
const t = useT()
const config = useQuery({
queryKey: ['suggest-config'],
// The Worker always answers 200 when suggestions are deliberately off, so
// any other status is a failure to retry, not "closed".
queryFn: async (): Promise<SuggestConfig> => {
const res = await fetch('/api/suggest')
return res.ok ? ((await res.json()) as SuggestConfig) : { enabled: false, sitekey: null }
if (!res.ok) throw new Error(`suggest config: HTTP ${res.status}`)
return (await res.json()) as SuggestConfig
},
staleTime: 5 * 60_000,
staleTime: 60_000,
})
const [url, setUrl] = useState('')
const [note, setNote] = useState('')
@@ -130,10 +133,21 @@ function SuggestForm() {
<h1 className="text-2xl font-semibold">{t('titles.suggest')}</h1>
<p className="text-muted-foreground">{t('suggest.intro')}</p>
{config.data && !config.data.enabled ? (
{config.isPending ? (
<p className="text-sm text-muted-foreground">{t('common.loading')}</p>
) : config.isError ? (
<div role="alert" className="space-y-2 rounded-md border p-3 text-sm">
<p className="text-destructive">{t('suggest.error.config')}</p>
<Button size="sm" variant="outline" onClick={() => void config.refetch()}>
{t('common.retry')}
</Button>
</div>
) : !config.data.enabled ? (
<p className="rounded-md border p-3 text-sm">{t('suggest.closed')}</p>
) : (
<form onSubmit={submit} className="space-y-4">
// noValidate: the page validates the link itself (a link without
// https:// is fine; the server adds it), the browser's type=url check doesn't agree.
<form noValidate onSubmit={submit} className="space-y-4">
<div className="space-y-1.5">
<Label htmlFor="suggest-url">{t('suggest.url')}</Label>
<Input
+9 -1
View File
@@ -76,7 +76,15 @@ const LOCAL_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]'])
async function maintainer(request: Request, env: WorkerEnv): Promise<string> {
// `wrangler dev` only: Access does not run on localhost. Set DEV_ADMIN_EMAIL
// in .dev.vars; it is ignored for every other host.
if (env.DEV_ADMIN_EMAIL && LOCAL_HOSTS.has(new URL(request.url).hostname)) return env.DEV_ADMIN_EMAIL
const url = new URL(request.url)
if (env.DEV_ADMIN_EMAIL && LOCAL_HOSTS.has(url.hostname)) {
// Same cross-site-request guard as requireMaintainer: another site open in
// the browser must not be able to drive a local admin.
if (!['GET', 'HEAD', 'OPTIONS'].includes(request.method) && request.headers.get('origin') !== url.origin) {
throw new HttpError(403, 'forbidden')
}
return env.DEV_ADMIN_EMAIL
}
const missing = missingConfig(env, [...ACCESS_CONFIG, 'ACCESS_AUD_ADMIN', 'ADMIN_EMAILS'])
if (missing.length > 0) {
console.error(`admin: not configured (${missing.join(', ')})`)
+1 -1
View File
@@ -118,7 +118,7 @@ export function buildIndex(
/** Date-keyed upsert; a day whose total equals the last recorded one adds no row. */
export function upsertCountHistory(history: CountPoint[], total: number, now: Date): CountPoint[] {
const today = now.toISOString().slice(0, 10)
const rows = [...history].sort((a, b) => a.date.localeCompare(b.date))
const rows = history.map((r) => ({ ...r })).sort((a, b) => a.date.localeCompare(b.date))
const existing = rows.find((r) => r.date === today)
if (existing) {
existing.total = total
+38 -8
View File
@@ -38,15 +38,45 @@ export function errorResponse(e: unknown): Response {
return json({ error: 'internal_error' }, 500)
}
/** Parse a JSON request body of at most `maxBytes`. */
export async function readJson(request: Request, maxBytes: number): Promise<unknown> {
if (!(request.headers.get('content-type') ?? '').includes('application/json')) {
throw new HttpError(415, 'unsupported_media_type', 'Send application/json')
/**
* The request body, refusing more than `maxBytes` while it streams in: a body
* sent without Content-Length (chunked) is never buffered past the limit.
*/
export async function readBodyCapped(request: Request, maxBytes: number): Promise<Uint8Array> {
const declared = request.headers.get('content-length')
if (declared !== null && !(Number(declared) <= maxBytes)) throw new HttpError(413, 'payload_too_large')
if (!request.body) return new Uint8Array()
const reader = request.body.getReader()
const chunks: Uint8Array[] = []
let total = 0
for (;;) {
const { done, value } = await reader.read()
if (done) break
total += value.byteLength
if (total > maxBytes) {
reader.cancel().catch(() => {})
throw new HttpError(413, 'payload_too_large')
}
chunks.push(value)
}
const declared = Number(request.headers.get('content-length') ?? '0')
if (declared > maxBytes) throw new HttpError(413, 'payload_too_large')
const bytes = new Uint8Array(await request.arrayBuffer())
if (bytes.byteLength > maxBytes) throw new HttpError(413, 'payload_too_large')
const out = new Uint8Array(total)
let offset = 0
for (const chunk of chunks) {
out.set(chunk, offset)
offset += chunk.byteLength
}
return out
}
/**
* Parse a JSON request body of at most `maxBytes`. The media type must be
* exactly application/json: anything else (e.g. `text/plain; application/json`,
* which browsers send cross-site without a preflight) is refused.
*/
export async function readJson(request: Request, maxBytes: number): Promise<unknown> {
const type = (request.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase()
if (type !== 'application/json') throw new HttpError(415, 'unsupported_media_type', 'Send application/json')
const bytes = await readBodyCapped(request, maxBytes)
try {
return JSON.parse(new TextDecoder().decode(bytes))
} catch {
+2 -4
View File
@@ -16,7 +16,7 @@ import { AccessError, requireService, type AccessEnv } from './access'
import { canonicalize } from './canonical'
import { catalogUrls, deletedGames, type DataSource } from './data'
import { ACCESS_CONFIG, missingConfig, type WorkerEnv } from './env'
import { HttpError, errorResponse, isObject, json, sha256Hex } from './http'
import { HttpError, errorResponse, isObject, json, readBodyCapped, sha256Hex } from './http'
import { autoFlags, classify, type QueueStore } from './queue'
const MAX_BYTES = 64 * 1024
@@ -80,9 +80,7 @@ export async function handleIngest(request: Request, deps: IngestDeps): Promise<
const key = request.headers.get('idempotency-key') ?? ''
if (!KEY.test(key)) throw new HttpError(400, 'idempotency_key_required')
if (Number(request.headers.get('content-length') ?? '0') > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
const raw = await request.text()
if (new TextEncoder().encode(raw).byteLength > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
const raw = new TextDecoder().decode(await readBodyCapped(request, MAX_BYTES))
const bodyHash = await sha256Hex(raw)
const previous = await deps.store.idempotent(client, key)
+8 -3
View File
@@ -46,7 +46,12 @@ export function classify(inputs: string[], ctx: ClassifyContext): Classified[] {
if (ctx.catalog.has(canonical)) return { input, canonical, outcome: 'duplicate_catalog', store: false }
const status = ctx.known.get(canonical)
if (status === 'rejected') return { input, canonical, outcome: 'rejected', store: false }
if (status && status !== 'failed') return { input, canonical, outcome: 'duplicate_pending', store: false }
// 'ingested' rows whose game has since been removed (it is in the deleted
// log) are reviewed again, flagged, like any previously deleted game.
const cameBack = status === 'ingested' && ctx.deleted.has(canonical)
if (status && status !== 'failed' && !cameBack) {
return { input, canonical, outcome: 'duplicate_pending', store: false }
}
const deleted = ctx.deleted.get(canonical)
if (deleted) {
return {
@@ -141,7 +146,7 @@ export class QueueStore {
return out
}
/** Insert new candidates; an existing `failed` row is reopened as pending. */
/** Insert new candidates; an existing `failed` or `ingested` row is reopened as pending. */
async insert(items: NewCandidate[], now = new Date()): Promise<void> {
if (items.length === 0) return
const stmt = this.db.prepare(
@@ -152,7 +157,7 @@ export class QueueStore {
image_url = coalesce(excluded.image_url, candidates.image_url),
flags = excluded.flags, note = excluded.note, submitter = excluded.submitter,
discovered_at = excluded.discovered_at, decided_at = NULL, decided_by = NULL
WHERE candidates.status = 'failed'`,
WHERE candidates.status IN ('failed', 'ingested')`,
)
await this.db.batch(
items.map((c) =>
+16 -9
View File
@@ -20,6 +20,7 @@ import { commitWithRetry, listDir, readFile, type CommitInput, type GitHubEnv }
const TEMP_LINK = 'scripts/temp_link.json'
const DELETED_LOG = 'scripts/deleted_games.json'
const UNBLOCKED = 'scripts/state/unblocked.json'
const CHUNK_FILE = /^game_info_\d+\.json$/
type Change = Omit<CommitInput, 'expectedHeadOid'>
@@ -58,17 +59,23 @@ export function withQueued(queueText: string | null, urls: string[]): { text: st
return { text: dumps([...queue, ...added]), added }
}
/** deleted_games.json without `urls` (a game the maintainer allows back in). */
export function withoutDeleted(deletedText: string | null, urls: Set<string>): { text: string; removed: number } {
const log = deletedText === null ? [] : (JSON.parse(deletedText) as DeletedEntry[])
const kept = log.filter((e) => !urls.has(e.url))
return { text: dumps(kept), removed: log.length - kept.length }
/**
* scripts/state/unblocked.json with `urls` added: removed games the maintainer
* allows back in. update_info.py scrapes them despite the deleted log and drops
* the log entry only when the game is really re-added (a still-paid or dead
* game keeps its public removal record).
*/
export function withUnblocked(currentText: string | null, urls: string[]): { text: string; added: number } {
const parsed: unknown = currentText === null ? [] : JSON.parse(currentText)
const current = Array.isArray(parsed) ? parsed.filter((u): u is string => typeof u === 'string') : []
const next = [...new Set([...current, ...urls])].sort()
return { text: dumps(next), added: next.length - current.length }
}
/**
* Queue `urls` for the ingest workflow (the push to temp_link.json triggers
* update.yml). With `unblock`, also drop those URLs from the deleted log,
* which update_info.py otherwise refuses to re-add.
* update.yml). With `unblock`, also let those previously removed games
* through the ingest (see withUnblocked).
*/
export async function queueForIngest(
env: GitHubEnv,
@@ -82,8 +89,8 @@ export async function queueForIngest(
added = queued.added
if (queued.added.length > 0) writes.set(TEMP_LINK, queued.text)
if (unblock.length > 0) {
const log = withoutDeleted(await readFile(env, DELETED_LOG, head), new Set(unblock))
if (log.removed > 0) writes.set(DELETED_LOG, log.text)
const allowed = withUnblocked(await readFile(env, UNBLOCKED, head), unblock)
if (allowed.added > 0) writes.set(UNBLOCKED, allowed.text)
}
if (writes.size === 0) return null
return {
+6 -9
View File
@@ -3,7 +3,7 @@ import { adminApi, type AdminContext, type RepoOps } from '../admin'
import { resetDataCache } from '../data'
import { r2Key } from '../img'
import { QueueStore } from '../queue'
import { withoutDeleted, withQueued } from '../repo'
import { withQueued, withUnblocked } from '../repo'
import { testDb } from './d1'
import { fakeData, ORIGIN } from './fakes'
@@ -183,13 +183,10 @@ describe('repo builders', () => {
expect(withQueued(null, ['https://dev.itch.io/x']).text).toBe('[\n "https://dev.itch.io/x"\n]')
})
it('drops unblocked URLs from the deleted log', () => {
const log = JSON.stringify([
{ url: 'https://dev.itch.io/a', name: 'A', reason: 'r', deleted_at: 't' },
{ url: 'https://dev.itch.io/b', name: 'B', reason: 'r', deleted_at: 't' },
])
const { text, removed } = withoutDeleted(log, new Set(['https://dev.itch.io/a']))
expect(removed).toBe(1)
expect(JSON.parse(text).map((e: { url: string }) => e.url)).toEqual(['https://dev.itch.io/b'])
it('adds unblocked URLs to the sorted allow-list once', () => {
const { text, added } = withUnblocked('["https://dev.itch.io/b"]', ['https://dev.itch.io/a', 'https://dev.itch.io/b'])
expect(added).toBe(1)
expect(JSON.parse(text)).toEqual(['https://dev.itch.io/a', 'https://dev.itch.io/b'])
expect(withUnblocked(null, ['https://dev.itch.io/x']).text).toBe('[\n "https://dev.itch.io/x"\n]')
})
})
+3 -2
View File
@@ -114,7 +114,7 @@ describe('editCatalog', () => {
})
describe('queueForIngest', () => {
it('appends to temp_link.json and, when unblocking, prunes the deleted log', async () => {
it('appends to temp_link.json and, when unblocking, allow-lists the game (the log entry stays)', async () => {
const commits = fakeGitHub({
'scripts/temp_link.json': dumps(['https://dev.itch.io/queued']),
'scripts/deleted_games.json': dumps([{ url: 'https://dev.itch.io/b', name: 'b', reason: 'r', deleted_at: 't' }]),
@@ -123,7 +123,8 @@ describe('queueForIngest', () => {
expect(sha).toBe('new-sha')
expect(added).toEqual(['https://dev.itch.io/b'])
expect(JSON.parse(commits[0].additions['scripts/temp_link.json'])).toEqual(['https://dev.itch.io/queued', 'https://dev.itch.io/b'])
expect(JSON.parse(commits[0].additions['scripts/deleted_games.json'])).toEqual([])
expect(JSON.parse(commits[0].additions['scripts/state/unblocked.json'])).toEqual(['https://dev.itch.io/b'])
expect(commits[0].additions['scripts/deleted_games.json']).toBeUndefined()
})
it('makes no commit when everything is already queued', async () => {
@@ -0,0 +1,111 @@
/** Regressions from the Phase 4 review (see the commit that added this file). */
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { serveAdminApp, handleAdminApi } from '../admin'
import { applyAdminChange, dumps, upsertCountHistory } from '../catalog'
import type { WorkerEnv } from '../env'
import { readBodyCapped, readJson } from '../http'
import { classify, QueueStore } from '../queue'
import { testDb } from './d1'
describe('request bodies', () => {
it('stops reading a body without Content-Length once it passes the limit', async () => {
let pulled = 0
const body = new ReadableStream<Uint8Array>({
pull(controller) {
pulled += 1024
controller.enqueue(new Uint8Array(1024))
if (pulled > 1_000_000) controller.close()
},
})
const request = new Request('https://x/api', { method: 'POST', body, duplex: 'half' } as RequestInit)
await expect(readBodyCapped(request, 4096)).rejects.toMatchObject({ status: 413 })
expect(pulled).toBeLessThan(16 * 1024)
})
it('accepts only the application/json media type', async () => {
const post = (type: string) =>
new Request('https://x/api', { method: 'POST', headers: { 'content-type': type }, body: '{}' })
await expect(readJson(post('application/json; charset=utf-8'), 100)).resolves.toEqual({})
await expect(readJson(post('text/plain; application/json'), 100)).rejects.toMatchObject({ status: 415 })
})
})
describe('local admin bypass', () => {
const env = { DEV_ADMIN_EMAIL: 'dev@localhost', SITE_ORIGIN: 'https://freeitchgames.win' } as unknown as WorkerEnv
it('refuses cross-origin writes even on localhost', async () => {
const url = new URL('http://localhost:8787/api/admin/queue/decide')
const request = new Request(url, {
method: 'POST',
headers: { origin: 'https://evil.example', 'content-type': 'application/json' },
body: JSON.stringify({ action: 'reject', urls: ['https://dev.itch.io/a'] }),
})
expect((await handleAdminApi(request, url, env, null)).status).toBe(403)
})
it('still serves reads and same-origin writes locally', async () => {
const shellUrl = new URL('http://localhost:8787/admin/')
const assets = { fetch: async () => new Response('<html></html>') } as unknown as Fetcher
expect((await serveAdminApp(new Request(shellUrl), shellUrl, { ...env, ASSETS: assets } as WorkerEnv)).status).toBe(200)
})
})
describe('count history', () => {
it('does not change the caller rows and rewrites today when it already has a row', () => {
const history = [
{ date: '2026-09-17', total: 2 },
{ date: '2026-09-18', total: 3 },
]
const before = JSON.stringify(history)
const out = upsertCountHistory(history, 2, new Date('2026-09-18T10:00:00Z'))
expect(JSON.stringify(history)).toBe(before)
expect(out.at(-1)).toEqual({ date: '2026-09-18', total: 2 })
const games = ['a', 'b', 'c'].map((s) => ({ url: `https://dev.itch.io/${s}`, name: s }))
const change = applyAdminChange(
{
chunks: [games],
index: { total_games: 3, max_per_file: 500, last_updated: 'x', files: [{ name: 'game_info_001.json', count: 3 }] },
history,
deleted: [],
},
new Map(),
[{ url: 'https://dev.itch.io/a', reason: 'r' }],
new Date('2026-09-18T10:00:00Z'),
)
expect(change.writes.get('data_game/count_history.json')).toBe(
dumps([
{ date: '2026-09-17', total: 2 },
{ date: '2026-09-18', total: 2 },
]),
)
})
})
describe('games that come back after being ingested and removed', () => {
let db: D1Database
let dispose: () => Promise<void>
beforeAll(async () => {
;({ db, dispose } = await testDb())
}, 60_000)
afterAll(() => dispose())
it('are reviewed again, flagged, instead of being blocked for good', async () => {
const url = 'https://dev.itch.io/back'
const deleted = new Map([[url, { reason: 'Game became paid', deleted_at: '2026-09-01T00:00:00Z' }]])
const [result] = classify([url], { catalog: new Set(), deleted, known: new Map([[url, 'ingested']]) })
expect(result.outcome).toBe('previously_deleted')
expect(result.store).toBe(true)
// Still in the catalog: still a duplicate, not reopened.
const listed = classify([url], { catalog: new Set([url]), deleted: new Map(), known: new Map([[url, 'ingested']]) })
expect(listed[0].outcome).toBe('duplicate_catalog')
const store = new QueueStore(db)
await store.insert([{ url, source: 'rss' }])
await store.setStatus([url], 'ingested', 'reconcile')
await store.insert([{ url, source: 'rss', flags: ['previously_deleted'] }])
const [row] = await store.get([url])
expect(row.status).toBe('pending')
expect(JSON.parse(row.flags)).toEqual(['previously_deleted'])
})
})