fix: Phase 4 review findings (admin, ingest, suggest)
Worker - The local DEV_ADMIN_EMAIL bypass keeps the same-origin check for writes, and JSON bodies must be exactly application/json (a cross-site `text/plain; application/json` POST no longer gets through without a preflight). - Request bodies are capped while they stream (a chunked upload without Content-Length is never buffered past 4 KB / 64 KB). - count_history.json is rewritten when today's row already exists (upsertCountHistory no longer mutates the rows it was given). - A game that entered the catalog through the queue, was removed and turns up again is reviewed again (flagged previously removed) instead of being blocked as "already pending" for good. - Restoring / approving a removed game no longer deletes its public removal record up front: the Worker allow-lists it in scripts/state/unblocked.json, update_info.py scrapes it despite the deleted log, and apply_patch.py drops the log entry only when the game is really re-added (a still-paid or dead game keeps its record). Admin app - The editor compares against what the repository holds (published data or what was saved in this session), so saved changes can be reverted and are never dropped silently. - An Access session that expires mid-session shows a sign-in banner from any request, without losing what is on the page. - Restore refreshes the queue; knip now checks the admin entry. Suggest page - The form doesn't let the browser's URL check block links without https:// (the page and the server accept them). - A failed config request shows an error with Retry instead of a permanently disabled form or a false "closed" message. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
21 files changed
+399
-66
No files matched your search
@@ -12,6 +12,9 @@ always works on the latest data. Idempotent: applying the same patch twice
|
||||
- *_state: merged into scripts/state/*.json (null drops an entry); for
|
||||
refresh_state the newer `checked` wins and the newest `full`
|
||||
stamp from either side is kept
|
||||
- unlog / unblock_remove: an unblocked (restored) game that was re-added
|
||||
leaves the deleted log and scripts/state/unblocked.json; one whose
|
||||
ingest ended otherwise leaves only the allow-list
|
||||
- queue_add / queue_remove: appended to / dropped from scripts/temp_link.json,
|
||||
which is re-read here so URLs queued meanwhile survive. Entries
|
||||
that are not URL strings are dropped on every apply.
|
||||
@@ -43,6 +46,7 @@ DELETED_LOG = "scripts/deleted_games.json"
|
||||
TEMP_LINK = "scripts/temp_link.json"
|
||||
REFRESH_STATE = "scripts/state/refresh_state.json"
|
||||
INGEST_STATE = "scripts/state/ingest_state.json"
|
||||
UNBLOCKED = "scripts/state/unblocked.json"
|
||||
MASS_REMOVAL_FLOOR = 25
|
||||
MASS_REMOVAL_SHARE = 0.05
|
||||
|
||||
@@ -157,6 +161,19 @@ def apply(patch: dict) -> dict:
|
||||
state = {u: e for u, e in state.items() if u in live_urls}
|
||||
save_state_map(path, state)
|
||||
|
||||
unlog = {u for u in patch.get("unlog", []) if isinstance(u, str)}
|
||||
finished = unlog | {u for u in patch.get("unblock_remove", []) if isinstance(u, str)}
|
||||
if unlog:
|
||||
log = load_json(DELETED_LOG)
|
||||
kept = [e for e in log if e.get("url") not in unlog]
|
||||
if len(kept) != len(log):
|
||||
save_json(DELETED_LOG, kept)
|
||||
if finished and os.path.exists(UNBLOCKED):
|
||||
allowed = load_json(UNBLOCKED)
|
||||
remaining = sorted(u for u in allowed if u not in finished)
|
||||
if remaining != allowed:
|
||||
save_json(UNBLOCKED, remaining)
|
||||
|
||||
raw_queue = load_json(TEMP_LINK)
|
||||
queue = clean_queue(raw_queue)
|
||||
present = {_queue_key(u) for u in queue}
|
||||
|
||||
@@ -18,6 +18,10 @@ Patch v1:
|
||||
"ingest_state": {"<url>": {...} | null},
|
||||
"queue_add": ["<url>", ...], # append to temp_link.json if absent
|
||||
"queue_remove": ["<url>", ...], # drop from temp_link.json
|
||||
"unlog": ["<url>", ...], # unblocked + re-added: drop from the
|
||||
# deleted log and state/unblocked.json
|
||||
"unblock_remove": ["<url>", ...], # unblocked, ended otherwise: drop from
|
||||
# state/unblocked.json only
|
||||
"stats": {...} # informational
|
||||
}
|
||||
"""
|
||||
@@ -40,6 +44,8 @@ def new_patch(kind: str) -> dict:
|
||||
"ingest_state": {},
|
||||
"queue_add": [],
|
||||
"queue_remove": [],
|
||||
"unlog": [],
|
||||
"unblock_remove": [],
|
||||
"stats": {},
|
||||
}
|
||||
|
||||
@@ -67,5 +73,7 @@ def is_empty(patch: dict) -> bool:
|
||||
"ingest_state",
|
||||
"queue_add",
|
||||
"queue_remove",
|
||||
"unlog",
|
||||
"unblock_remove",
|
||||
)
|
||||
)
|
||||
+12
-1
@@ -42,6 +42,10 @@ from scraper import NA, Pacer, create_session, now_iso, parse_game
|
||||
TEMP_LINK = "scripts/temp_link.json"
|
||||
DELETED_LOG = "scripts/deleted_games.json"
|
||||
INGEST_STATE = "scripts/state/ingest_state.json"
|
||||
# Removed games the maintainer asked to bring back (admin Restore / approve with
|
||||
# override). They are scraped despite the deleted log; the log entry goes only
|
||||
# once the game is really re-added, so a still-paid game keeps its record.
|
||||
UNBLOCKED = "scripts/state/unblocked.json"
|
||||
MAX_ATTEMPTS = 3
|
||||
RETRY_GAP = timedelta(hours=6)
|
||||
DEAD_CODES = {404, 410}
|
||||
@@ -95,6 +99,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
catalog_urls = get_all_urls(load_all_games())
|
||||
deleted_urls = {e.get("url") for e in load_json(DELETED_LOG)}
|
||||
unblocked = set(load_json(UNBLOCKED, default=[]))
|
||||
ingest_state: dict = load_json(INGEST_STATE, default={})
|
||||
|
||||
todo: list[str] = []
|
||||
@@ -113,7 +118,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
if url in catalog_urls:
|
||||
print(f"Skip duplicate: {url}")
|
||||
stats["duplicate"] += 1
|
||||
elif url in deleted_urls:
|
||||
if url in unblocked:
|
||||
patch["unlog"].append(url)
|
||||
elif url in deleted_urls and url not in unblocked:
|
||||
print(f"Skip previously deleted: {url} (restore it from the admin page)")
|
||||
stats["deleted"] += 1
|
||||
else:
|
||||
@@ -161,6 +168,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
patch["queue_remove"].append(url)
|
||||
patch["ingest_state"][url] = {**entry, "gave_up": entry["last_attempt"]}
|
||||
stats["gave_up"] += 1
|
||||
if url in unblocked:
|
||||
patch["unblock_remove"].append(url)
|
||||
else:
|
||||
print(f" → {outcome}; stays queued ({entry['attempts']}/{MAX_ATTEMPTS} attempts).")
|
||||
patch["ingest_state"][url] = entry
|
||||
@@ -171,6 +180,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
stats[outcome] += 1
|
||||
patch["queue_remove"].append(url)
|
||||
patch["ingest_state"][url] = None
|
||||
if url in unblocked:
|
||||
patch["unlog" if outcome == "added" else "unblock_remove"].append(url)
|
||||
|
||||
write_patch(args.out, patch)
|
||||
print(
|
||||
|
||||
@@ -201,3 +201,47 @@ def test_throttled_page_is_not_an_attempt(repo, monkeypatch):
|
||||
patch = _run(monkeypatch, {url: Page(status=429)}, [], INPUT_URL=url)
|
||||
assert url not in patch["ingest_state"] and url not in patch["queue_remove"]
|
||||
assert patch["queue_add"] == [url]
|
||||
|
||||
|
||||
def _deleted(slug):
|
||||
return {
|
||||
"url": f"https://dev.itch.io/{slug}",
|
||||
"name": slug,
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-03-13T00:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
def test_unblocked_game_that_is_free_again_leaves_the_deleted_log(repo, monkeypatch):
|
||||
seed_catalog([make_game("known")])
|
||||
url = "https://dev.itch.io/free"
|
||||
save_json("scripts/deleted_games.json", [_deleted("free"), _deleted("other")])
|
||||
save_json("scripts/state/unblocked.json", [url])
|
||||
pages = {url: Page(status=200, soup=load_fixture_soup("game_free_full.html"))}
|
||||
patch = _run(monkeypatch, pages, [url])
|
||||
assert [a["url"] for a in patch["additions"]] == [url] and patch["unlog"] == [url]
|
||||
_apply(patch)
|
||||
assert [e["url"] for e in read_json("scripts/deleted_games.json")] == [
|
||||
"https://dev.itch.io/other"
|
||||
]
|
||||
assert read_json("scripts/state/unblocked.json") == []
|
||||
|
||||
|
||||
def test_unblocked_game_that_is_still_paid_keeps_its_removal_record(repo, monkeypatch):
|
||||
seed_catalog([make_game("known")])
|
||||
url = "https://dev.itch.io/paid"
|
||||
save_json("scripts/deleted_games.json", [_deleted("paid")])
|
||||
save_json("scripts/state/unblocked.json", [url])
|
||||
pages = {url: Page(status=200, soup=load_fixture_soup("game_paid.html"))}
|
||||
patch = _run(monkeypatch, pages, [url])
|
||||
assert patch["additions"] == [] and patch["unblock_remove"] == [url]
|
||||
_apply(patch)
|
||||
assert [e["url"] for e in read_json("scripts/deleted_games.json")] == [url]
|
||||
assert read_json("scripts/state/unblocked.json") == []
|
||||
|
||||
|
||||
def test_deleted_game_without_unblock_is_still_skipped(repo, monkeypatch):
|
||||
seed_catalog([make_game("known")])
|
||||
save_json("scripts/deleted_games.json", [_deleted("paid")])
|
||||
patch = _run(monkeypatch, {}, ["https://dev.itch.io/paid"])
|
||||
assert patch["stats"]["deleted"] == 1 and patch["unlog"] == []
|
||||
+20
-1
@@ -11,7 +11,7 @@ export class SessionExpired extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export class ApiError extends Error {
|
||||
class ApiError extends Error {
|
||||
readonly status: number
|
||||
readonly code: string
|
||||
|
||||
@@ -22,6 +22,25 @@ export class ApiError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
// Set once any request finds the Access session gone; the app shows a sign-in banner.
|
||||
let expired = false
|
||||
const listeners = new Set<() => void>()
|
||||
|
||||
export function markSessionExpired(): void {
|
||||
if (expired) return
|
||||
expired = true
|
||||
for (const listener of listeners) listener()
|
||||
}
|
||||
|
||||
export function subscribeSession(listener: () => void): () => void {
|
||||
listeners.add(listener)
|
||||
return () => listeners.delete(listener)
|
||||
}
|
||||
|
||||
export function isSessionExpired(): boolean {
|
||||
return expired
|
||||
}
|
||||
|
||||
async function api<T>(method: string, path: string, body?: unknown): Promise<T> {
|
||||
const res = await fetch(path, {
|
||||
method,
|
||||
|
||||
+16
-1
@@ -1,7 +1,8 @@
|
||||
import { useSyncExternalStore } from 'react'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { NavLink, Navigate, Route, Routes } from 'react-router'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { SessionExpired, adminApi, errorText } from './api'
|
||||
import { SessionExpired, adminApi, errorText, isSessionExpired, subscribeSession } from './api'
|
||||
import { AddPage } from './add'
|
||||
import { CatalogPage } from './catalog'
|
||||
import { QueuePage } from './queue'
|
||||
@@ -14,6 +15,7 @@ const NAV = [
|
||||
|
||||
export function AdminApp() {
|
||||
const me = useQuery({ queryKey: ['me'], queryFn: adminApi.me })
|
||||
const expired = useSyncExternalStore(subscribeSession, isSessionExpired)
|
||||
|
||||
if (me.error) {
|
||||
return (
|
||||
@@ -56,6 +58,19 @@ export function AdminApp() {
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
{expired && (
|
||||
<div role="alert" className="border-b border-destructive/40 bg-destructive/10 px-4 py-2 text-sm">
|
||||
<div className="mx-auto flex max-w-6xl flex-wrap items-center gap-3">
|
||||
<span>Your admin session has expired. Sign in again, then retry — nothing on this page is lost.</span>
|
||||
<a className="font-medium underline" href="/admin/" target="_blank" rel="noopener">
|
||||
Sign in (new tab)
|
||||
</a>
|
||||
<button className="underline" onClick={() => window.location.reload()}>
|
||||
Reload this page
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<main className="mx-auto max-w-6xl p-4">
|
||||
<Routes>
|
||||
<Route path="/queue" element={<QueuePage />} />
|
||||
|
||||
+55
-19
@@ -1,5 +1,5 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import { useMutation } from '@tanstack/react-query'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { useAllGames, useDeletedGames } from '@/hooks/useGames'
|
||||
@@ -21,6 +21,9 @@ export function CatalogPage() {
|
||||
const [query, setQuery] = useState('')
|
||||
const [selected, setSelected] = useState<Game | null>(null)
|
||||
const [tab, setTab] = useState<'catalog' | 'removed'>('catalog')
|
||||
// Values saved in this session, per game: the published /data lags behind
|
||||
// the repository until the next build, so it can't be the baseline.
|
||||
const [saved] = useState(() => new Map<string, EditableValues>())
|
||||
|
||||
const found = useMemo(
|
||||
() => (games.data?.games ?? []).filter((g) => matches(query, g.name, g.url, g.dev)).slice(0, MAX_RESULTS),
|
||||
@@ -77,30 +80,56 @@ export function CatalogPage() {
|
||||
<RemovedList entries={removed} />
|
||||
)}
|
||||
</div>
|
||||
<div>{selected && tab === 'catalog' && <GameEditor key={selected.url} game={selected} />}</div>
|
||||
<div>
|
||||
{selected && tab === 'catalog' && (
|
||||
<GameEditor
|
||||
key={selected.url}
|
||||
game={selected}
|
||||
known={saved.get(selected.url) ?? published(selected)}
|
||||
onSaved={(values) => saved.set(selected.url, values)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function GameEditor({ game }: { game: Game }) {
|
||||
const [edit, setEdit] = useState<Required<GameEdit>>({
|
||||
safe_virus: game.safe_virus,
|
||||
nsfw: game.nsfw,
|
||||
notes: game.notes ?? '',
|
||||
})
|
||||
type EditableValues = Required<GameEdit>
|
||||
const EDITABLE = ['safe_virus', 'nsfw', 'notes'] as const
|
||||
|
||||
function published(game: Game): EditableValues {
|
||||
return { safe_virus: game.safe_virus, nsfw: game.nsfw, notes: game.notes ?? '' }
|
||||
}
|
||||
|
||||
/**
|
||||
* `known` is what the repository holds as far as this page knows: the
|
||||
* published data, or what was saved in this session. Changes are computed
|
||||
* against it, so a saved edit can be reverted and never silently dropped.
|
||||
*/
|
||||
function GameEditor({ game, known, onSaved }: { game: Game; known: EditableValues; onSaved: (v: EditableValues) => void }) {
|
||||
const [baseline, setBaseline] = useState<EditableValues>(known)
|
||||
const [edit, setEdit] = useState<EditableValues>(known)
|
||||
const [reason, setReason] = useState('')
|
||||
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
|
||||
|
||||
const changed = (Object.keys(edit) as (keyof GameEdit)[]).filter((k) => edit[k] !== (game[k] ?? ''))
|
||||
const changed = EDITABLE.filter((k) => edit[k] !== baseline[k])
|
||||
const save = useMutation({
|
||||
mutationFn: () => adminApi.edit(game.url, Object.fromEntries(changed.map((k) => [k, edit[k]]))),
|
||||
onSuccess: (r) =>
|
||||
mutationFn: (sent: GameEdit) => adminApi.edit(game.url, sent),
|
||||
onSuccess: (r, sent) => {
|
||||
const next = { ...baseline, ...sent }
|
||||
setBaseline(next)
|
||||
onSaved(next)
|
||||
setNotice({
|
||||
tone: 'info',
|
||||
text: r.edited ? `Saved (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Nothing changed in the repository.',
|
||||
}),
|
||||
})
|
||||
},
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
const change = (next: EditableValues) => {
|
||||
setEdit(next)
|
||||
if (notice?.tone === 'info') setNotice(null)
|
||||
}
|
||||
const remove = useMutation({
|
||||
mutationFn: () => adminApi.remove(game.url, reason.trim()),
|
||||
onSuccess: (r) => setNotice({ tone: 'info', text: `Removed (${shortSha(r.sha)}); its resized covers were deleted. ${PUBLISH_NOTE}` }),
|
||||
@@ -120,7 +149,7 @@ function GameEditor({ game }: { game: Game }) {
|
||||
<span className="text-xs text-muted-foreground">Virus check</span>
|
||||
<select
|
||||
value={edit.safe_virus}
|
||||
onChange={(e) => setEdit({ ...edit, safe_virus: e.target.value })}
|
||||
onChange={(e) => change({ ...edit, safe_virus: e.target.value })}
|
||||
className="h-9 rounded-md border bg-background px-2"
|
||||
>
|
||||
{['?', 'Yes', 'No', 'Caution'].map((v) => (
|
||||
@@ -134,7 +163,7 @@ function GameEditor({ game }: { game: Game }) {
|
||||
<span className="text-xs text-muted-foreground">NSFW (18+)</span>
|
||||
<select
|
||||
value={edit.nsfw}
|
||||
onChange={(e) => setEdit({ ...edit, nsfw: e.target.value })}
|
||||
onChange={(e) => change({ ...edit, nsfw: e.target.value })}
|
||||
className="h-9 rounded-md border bg-background px-2"
|
||||
>
|
||||
<option value="No">No</option>
|
||||
@@ -148,11 +177,15 @@ function GameEditor({ game }: { game: Game }) {
|
||||
value={edit.notes}
|
||||
maxLength={2000}
|
||||
rows={3}
|
||||
onChange={(e) => setEdit({ ...edit, notes: e.target.value })}
|
||||
onChange={(e) => change({ ...edit, notes: e.target.value })}
|
||||
className="rounded-md border bg-background p-2"
|
||||
/>
|
||||
</label>
|
||||
<Button className="mt-3" disabled={!changed.length || save.isPending} onClick={() => save.mutate()}>
|
||||
<Button
|
||||
className="mt-3"
|
||||
disabled={!changed.length || save.isPending}
|
||||
onClick={() => save.mutate(Object.fromEntries(changed.map((k) => [k, edit[k]])))}
|
||||
>
|
||||
{save.isPending ? 'Saving…' : 'Save'}
|
||||
</Button>
|
||||
|
||||
@@ -180,13 +213,16 @@ function GameEditor({ game }: { game: Game }) {
|
||||
|
||||
function RemovedList({ entries }: { entries: { url: string; name: string; reason: string; deleted_at: string }[] }) {
|
||||
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
|
||||
const client = useQueryClient()
|
||||
const restore = useMutation({
|
||||
mutationFn: (url: string) => adminApi.restore(url),
|
||||
onSuccess: (r) =>
|
||||
onSuccess: (r) => {
|
||||
setNotice({
|
||||
tone: 'info',
|
||||
text: r.queued ? `Queued for re-ingest (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Already queued.',
|
||||
}),
|
||||
})
|
||||
void client.invalidateQueries({ queryKey: ['queue'] })
|
||||
},
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
return (
|
||||
@@ -206,7 +242,7 @@ function RemovedList({ entries }: { entries: { url: string; name: string; reason
|
||||
variant="outline"
|
||||
disabled={restore.isPending}
|
||||
onClick={() => {
|
||||
if (window.confirm(`Bring "${d.name}" back? It is re-scraped; a paid game is dropped again.`)) {
|
||||
if (window.confirm(`Bring "${d.name}" back? It is re-scraped; if it is still paid or gone, it stays on the removed list.`)) {
|
||||
restore.mutate(d.url)
|
||||
}
|
||||
}}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { StrictMode } from 'react'
|
||||
import { createRoot } from 'react-dom/client'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { MutationCache, QueryCache, QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { HashRouter } from 'react-router'
|
||||
import '@/index.css'
|
||||
import { AdminApp } from './app'
|
||||
import { SessionExpired } from './api'
|
||||
import { SessionExpired, markSessionExpired } from './api'
|
||||
|
||||
// Follow the system theme (the admin has no settings of its own).
|
||||
const dark = window.matchMedia('(prefers-color-scheme: dark)')
|
||||
@@ -12,7 +12,13 @@ const applyTheme = () => document.documentElement.classList.toggle('dark', dark.
|
||||
applyTheme()
|
||||
dark.addEventListener('change', applyTheme)
|
||||
|
||||
const onError = (error: unknown) => {
|
||||
if (error instanceof SessionExpired) markSessionExpired()
|
||||
}
|
||||
|
||||
const queryClient = new QueryClient({
|
||||
queryCache: new QueryCache({ onError }),
|
||||
mutationCache: new MutationCache({ onError }),
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
staleTime: 30_000,
|
||||
|
||||
+5
-1
@@ -2,6 +2,7 @@
|
||||
"$schema": "https://unpkg.com/knip@6/schema.json",
|
||||
"project": [
|
||||
"src/**/*.{ts,tsx}",
|
||||
"admin/**/*.{ts,tsx}",
|
||||
"worker/**/*.ts",
|
||||
"vite-plugins/**/*.ts"
|
||||
],
|
||||
@@ -9,5 +10,8 @@
|
||||
"@/*": [
|
||||
"./src/*"
|
||||
]
|
||||
}
|
||||
},
|
||||
"entry": [
|
||||
"admin/main.tsx"
|
||||
]
|
||||
}
|
||||
@@ -288,6 +288,7 @@ export const en = {
|
||||
"settings.content.title": "Content",
|
||||
"settings.content.nsfw": "Show 18+ games",
|
||||
"settings.content.nsfwDesc": "Off by default. Turning it on asks you to confirm you are 18 or older; the choice stays in this browser.",
|
||||
"suggest.error.config": "Couldn't reach the site to load the form. Check your connection or content blocker, then try again.",
|
||||
} as const
|
||||
|
||||
export type MessageKey = keyof typeof en
|
||||
@@ -290,4 +290,5 @@ export const vi: Record<MessageKey, string> = {
|
||||
"settings.content.title": "Nội dung",
|
||||
"settings.content.nsfw": "Hiện game 18+",
|
||||
"settings.content.nsfwDesc": "Mặc định tắt. Khi bật, bạn sẽ được yêu cầu xác nhận đủ 18 tuổi; lựa chọn chỉ lưu trong trình duyệt này.",
|
||||
"suggest.error.config": "Không kết nối được tới trang để tải biểu mẫu. Hãy kiểm tra kết nối hoặc trình chặn nội dung rồi thử lại.",
|
||||
}
|
||||
@@ -54,11 +54,14 @@ function SuggestForm() {
|
||||
const t = useT()
|
||||
const config = useQuery({
|
||||
queryKey: ['suggest-config'],
|
||||
// The Worker always answers 200 when suggestions are deliberately off, so
|
||||
// any other status is a failure to retry, not "closed".
|
||||
queryFn: async (): Promise<SuggestConfig> => {
|
||||
const res = await fetch('/api/suggest')
|
||||
return res.ok ? ((await res.json()) as SuggestConfig) : { enabled: false, sitekey: null }
|
||||
if (!res.ok) throw new Error(`suggest config: HTTP ${res.status}`)
|
||||
return (await res.json()) as SuggestConfig
|
||||
},
|
||||
staleTime: 5 * 60_000,
|
||||
staleTime: 60_000,
|
||||
})
|
||||
const [url, setUrl] = useState('')
|
||||
const [note, setNote] = useState('')
|
||||
@@ -130,10 +133,21 @@ function SuggestForm() {
|
||||
<h1 className="text-2xl font-semibold">{t('titles.suggest')}</h1>
|
||||
<p className="text-muted-foreground">{t('suggest.intro')}</p>
|
||||
|
||||
{config.data && !config.data.enabled ? (
|
||||
{config.isPending ? (
|
||||
<p className="text-sm text-muted-foreground">{t('common.loading')}</p>
|
||||
) : config.isError ? (
|
||||
<div role="alert" className="space-y-2 rounded-md border p-3 text-sm">
|
||||
<p className="text-destructive">{t('suggest.error.config')}</p>
|
||||
<Button size="sm" variant="outline" onClick={() => void config.refetch()}>
|
||||
{t('common.retry')}
|
||||
</Button>
|
||||
</div>
|
||||
) : !config.data.enabled ? (
|
||||
<p className="rounded-md border p-3 text-sm">{t('suggest.closed')}</p>
|
||||
) : (
|
||||
<form onSubmit={submit} className="space-y-4">
|
||||
// noValidate: the page validates the link itself (a link without
|
||||
// https:// is fine; the server adds it), the browser's type=url check doesn't agree.
|
||||
<form noValidate onSubmit={submit} className="space-y-4">
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="suggest-url">{t('suggest.url')}</Label>
|
||||
<Input
|
||||
|
||||
@@ -76,7 +76,15 @@ const LOCAL_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]'])
|
||||
async function maintainer(request: Request, env: WorkerEnv): Promise<string> {
|
||||
// `wrangler dev` only: Access does not run on localhost. Set DEV_ADMIN_EMAIL
|
||||
// in .dev.vars; it is ignored for every other host.
|
||||
if (env.DEV_ADMIN_EMAIL && LOCAL_HOSTS.has(new URL(request.url).hostname)) return env.DEV_ADMIN_EMAIL
|
||||
const url = new URL(request.url)
|
||||
if (env.DEV_ADMIN_EMAIL && LOCAL_HOSTS.has(url.hostname)) {
|
||||
// Same cross-site-request guard as requireMaintainer: another site open in
|
||||
// the browser must not be able to drive a local admin.
|
||||
if (!['GET', 'HEAD', 'OPTIONS'].includes(request.method) && request.headers.get('origin') !== url.origin) {
|
||||
throw new HttpError(403, 'forbidden')
|
||||
}
|
||||
return env.DEV_ADMIN_EMAIL
|
||||
}
|
||||
const missing = missingConfig(env, [...ACCESS_CONFIG, 'ACCESS_AUD_ADMIN', 'ADMIN_EMAILS'])
|
||||
if (missing.length > 0) {
|
||||
console.error(`admin: not configured (${missing.join(', ')})`)
|
||||
|
||||
@@ -118,7 +118,7 @@ export function buildIndex(
|
||||
/** Date-keyed upsert; a day whose total equals the last recorded one adds no row. */
|
||||
export function upsertCountHistory(history: CountPoint[], total: number, now: Date): CountPoint[] {
|
||||
const today = now.toISOString().slice(0, 10)
|
||||
const rows = [...history].sort((a, b) => a.date.localeCompare(b.date))
|
||||
const rows = history.map((r) => ({ ...r })).sort((a, b) => a.date.localeCompare(b.date))
|
||||
const existing = rows.find((r) => r.date === today)
|
||||
if (existing) {
|
||||
existing.total = total
|
||||
|
||||
+38
-8
@@ -38,15 +38,45 @@ export function errorResponse(e: unknown): Response {
|
||||
return json({ error: 'internal_error' }, 500)
|
||||
}
|
||||
|
||||
/** Parse a JSON request body of at most `maxBytes`. */
|
||||
export async function readJson(request: Request, maxBytes: number): Promise<unknown> {
|
||||
if (!(request.headers.get('content-type') ?? '').includes('application/json')) {
|
||||
throw new HttpError(415, 'unsupported_media_type', 'Send application/json')
|
||||
/**
|
||||
* The request body, refusing more than `maxBytes` while it streams in: a body
|
||||
* sent without Content-Length (chunked) is never buffered past the limit.
|
||||
*/
|
||||
export async function readBodyCapped(request: Request, maxBytes: number): Promise<Uint8Array> {
|
||||
const declared = request.headers.get('content-length')
|
||||
if (declared !== null && !(Number(declared) <= maxBytes)) throw new HttpError(413, 'payload_too_large')
|
||||
if (!request.body) return new Uint8Array()
|
||||
const reader = request.body.getReader()
|
||||
const chunks: Uint8Array[] = []
|
||||
let total = 0
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
total += value.byteLength
|
||||
if (total > maxBytes) {
|
||||
reader.cancel().catch(() => {})
|
||||
throw new HttpError(413, 'payload_too_large')
|
||||
}
|
||||
chunks.push(value)
|
||||
}
|
||||
const declared = Number(request.headers.get('content-length') ?? '0')
|
||||
if (declared > maxBytes) throw new HttpError(413, 'payload_too_large')
|
||||
const bytes = new Uint8Array(await request.arrayBuffer())
|
||||
if (bytes.byteLength > maxBytes) throw new HttpError(413, 'payload_too_large')
|
||||
const out = new Uint8Array(total)
|
||||
let offset = 0
|
||||
for (const chunk of chunks) {
|
||||
out.set(chunk, offset)
|
||||
offset += chunk.byteLength
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a JSON request body of at most `maxBytes`. The media type must be
|
||||
* exactly application/json: anything else (e.g. `text/plain; application/json`,
|
||||
* which browsers send cross-site without a preflight) is refused.
|
||||
*/
|
||||
export async function readJson(request: Request, maxBytes: number): Promise<unknown> {
|
||||
const type = (request.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase()
|
||||
if (type !== 'application/json') throw new HttpError(415, 'unsupported_media_type', 'Send application/json')
|
||||
const bytes = await readBodyCapped(request, maxBytes)
|
||||
try {
|
||||
return JSON.parse(new TextDecoder().decode(bytes))
|
||||
} catch {
|
||||
|
||||
@@ -16,7 +16,7 @@ import { AccessError, requireService, type AccessEnv } from './access'
|
||||
import { canonicalize } from './canonical'
|
||||
import { catalogUrls, deletedGames, type DataSource } from './data'
|
||||
import { ACCESS_CONFIG, missingConfig, type WorkerEnv } from './env'
|
||||
import { HttpError, errorResponse, isObject, json, sha256Hex } from './http'
|
||||
import { HttpError, errorResponse, isObject, json, readBodyCapped, sha256Hex } from './http'
|
||||
import { autoFlags, classify, type QueueStore } from './queue'
|
||||
|
||||
const MAX_BYTES = 64 * 1024
|
||||
@@ -80,9 +80,7 @@ export async function handleIngest(request: Request, deps: IngestDeps): Promise<
|
||||
const key = request.headers.get('idempotency-key') ?? ''
|
||||
if (!KEY.test(key)) throw new HttpError(400, 'idempotency_key_required')
|
||||
|
||||
if (Number(request.headers.get('content-length') ?? '0') > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
|
||||
const raw = await request.text()
|
||||
if (new TextEncoder().encode(raw).byteLength > MAX_BYTES) throw new HttpError(413, 'payload_too_large')
|
||||
const raw = new TextDecoder().decode(await readBodyCapped(request, MAX_BYTES))
|
||||
const bodyHash = await sha256Hex(raw)
|
||||
|
||||
const previous = await deps.store.idempotent(client, key)
|
||||
|
||||
@@ -46,7 +46,12 @@ export function classify(inputs: string[], ctx: ClassifyContext): Classified[] {
|
||||
if (ctx.catalog.has(canonical)) return { input, canonical, outcome: 'duplicate_catalog', store: false }
|
||||
const status = ctx.known.get(canonical)
|
||||
if (status === 'rejected') return { input, canonical, outcome: 'rejected', store: false }
|
||||
if (status && status !== 'failed') return { input, canonical, outcome: 'duplicate_pending', store: false }
|
||||
// 'ingested' rows whose game has since been removed (it is in the deleted
|
||||
// log) are reviewed again, flagged, like any previously deleted game.
|
||||
const cameBack = status === 'ingested' && ctx.deleted.has(canonical)
|
||||
if (status && status !== 'failed' && !cameBack) {
|
||||
return { input, canonical, outcome: 'duplicate_pending', store: false }
|
||||
}
|
||||
const deleted = ctx.deleted.get(canonical)
|
||||
if (deleted) {
|
||||
return {
|
||||
@@ -141,7 +146,7 @@ export class QueueStore {
|
||||
return out
|
||||
}
|
||||
|
||||
/** Insert new candidates; an existing `failed` row is reopened as pending. */
|
||||
/** Insert new candidates; an existing `failed` or `ingested` row is reopened as pending. */
|
||||
async insert(items: NewCandidate[], now = new Date()): Promise<void> {
|
||||
if (items.length === 0) return
|
||||
const stmt = this.db.prepare(
|
||||
@@ -152,7 +157,7 @@ export class QueueStore {
|
||||
image_url = coalesce(excluded.image_url, candidates.image_url),
|
||||
flags = excluded.flags, note = excluded.note, submitter = excluded.submitter,
|
||||
discovered_at = excluded.discovered_at, decided_at = NULL, decided_by = NULL
|
||||
WHERE candidates.status = 'failed'`,
|
||||
WHERE candidates.status IN ('failed', 'ingested')`,
|
||||
)
|
||||
await this.db.batch(
|
||||
items.map((c) =>
|
||||
|
||||
+16
-9
@@ -20,6 +20,7 @@ import { commitWithRetry, listDir, readFile, type CommitInput, type GitHubEnv }
|
||||
|
||||
const TEMP_LINK = 'scripts/temp_link.json'
|
||||
const DELETED_LOG = 'scripts/deleted_games.json'
|
||||
const UNBLOCKED = 'scripts/state/unblocked.json'
|
||||
const CHUNK_FILE = /^game_info_\d+\.json$/
|
||||
|
||||
type Change = Omit<CommitInput, 'expectedHeadOid'>
|
||||
@@ -58,17 +59,23 @@ export function withQueued(queueText: string | null, urls: string[]): { text: st
|
||||
return { text: dumps([...queue, ...added]), added }
|
||||
}
|
||||
|
||||
/** deleted_games.json without `urls` (a game the maintainer allows back in). */
|
||||
export function withoutDeleted(deletedText: string | null, urls: Set<string>): { text: string; removed: number } {
|
||||
const log = deletedText === null ? [] : (JSON.parse(deletedText) as DeletedEntry[])
|
||||
const kept = log.filter((e) => !urls.has(e.url))
|
||||
return { text: dumps(kept), removed: log.length - kept.length }
|
||||
/**
|
||||
* scripts/state/unblocked.json with `urls` added: removed games the maintainer
|
||||
* allows back in. update_info.py scrapes them despite the deleted log and drops
|
||||
* the log entry only when the game is really re-added (a still-paid or dead
|
||||
* game keeps its public removal record).
|
||||
*/
|
||||
export function withUnblocked(currentText: string | null, urls: string[]): { text: string; added: number } {
|
||||
const parsed: unknown = currentText === null ? [] : JSON.parse(currentText)
|
||||
const current = Array.isArray(parsed) ? parsed.filter((u): u is string => typeof u === 'string') : []
|
||||
const next = [...new Set([...current, ...urls])].sort()
|
||||
return { text: dumps(next), added: next.length - current.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue `urls` for the ingest workflow (the push to temp_link.json triggers
|
||||
* update.yml). With `unblock`, also drop those URLs from the deleted log,
|
||||
* which update_info.py otherwise refuses to re-add.
|
||||
* update.yml). With `unblock`, also let those previously removed games
|
||||
* through the ingest (see withUnblocked).
|
||||
*/
|
||||
export async function queueForIngest(
|
||||
env: GitHubEnv,
|
||||
@@ -82,8 +89,8 @@ export async function queueForIngest(
|
||||
added = queued.added
|
||||
if (queued.added.length > 0) writes.set(TEMP_LINK, queued.text)
|
||||
if (unblock.length > 0) {
|
||||
const log = withoutDeleted(await readFile(env, DELETED_LOG, head), new Set(unblock))
|
||||
if (log.removed > 0) writes.set(DELETED_LOG, log.text)
|
||||
const allowed = withUnblocked(await readFile(env, UNBLOCKED, head), unblock)
|
||||
if (allowed.added > 0) writes.set(UNBLOCKED, allowed.text)
|
||||
}
|
||||
if (writes.size === 0) return null
|
||||
return {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { adminApi, type AdminContext, type RepoOps } from '../admin'
|
||||
import { resetDataCache } from '../data'
|
||||
import { r2Key } from '../img'
|
||||
import { QueueStore } from '../queue'
|
||||
import { withoutDeleted, withQueued } from '../repo'
|
||||
import { withQueued, withUnblocked } from '../repo'
|
||||
import { testDb } from './d1'
|
||||
import { fakeData, ORIGIN } from './fakes'
|
||||
|
||||
@@ -183,13 +183,10 @@ describe('repo builders', () => {
|
||||
expect(withQueued(null, ['https://dev.itch.io/x']).text).toBe('[\n "https://dev.itch.io/x"\n]')
|
||||
})
|
||||
|
||||
it('drops unblocked URLs from the deleted log', () => {
|
||||
const log = JSON.stringify([
|
||||
{ url: 'https://dev.itch.io/a', name: 'A', reason: 'r', deleted_at: 't' },
|
||||
{ url: 'https://dev.itch.io/b', name: 'B', reason: 'r', deleted_at: 't' },
|
||||
])
|
||||
const { text, removed } = withoutDeleted(log, new Set(['https://dev.itch.io/a']))
|
||||
expect(removed).toBe(1)
|
||||
expect(JSON.parse(text).map((e: { url: string }) => e.url)).toEqual(['https://dev.itch.io/b'])
|
||||
it('adds unblocked URLs to the sorted allow-list once', () => {
|
||||
const { text, added } = withUnblocked('["https://dev.itch.io/b"]', ['https://dev.itch.io/a', 'https://dev.itch.io/b'])
|
||||
expect(added).toBe(1)
|
||||
expect(JSON.parse(text)).toEqual(['https://dev.itch.io/a', 'https://dev.itch.io/b'])
|
||||
expect(withUnblocked(null, ['https://dev.itch.io/x']).text).toBe('[\n "https://dev.itch.io/x"\n]')
|
||||
})
|
||||
})
|
||||
@@ -114,7 +114,7 @@ describe('editCatalog', () => {
|
||||
})
|
||||
|
||||
describe('queueForIngest', () => {
|
||||
it('appends to temp_link.json and, when unblocking, prunes the deleted log', async () => {
|
||||
it('appends to temp_link.json and, when unblocking, allow-lists the game (the log entry stays)', async () => {
|
||||
const commits = fakeGitHub({
|
||||
'scripts/temp_link.json': dumps(['https://dev.itch.io/queued']),
|
||||
'scripts/deleted_games.json': dumps([{ url: 'https://dev.itch.io/b', name: 'b', reason: 'r', deleted_at: 't' }]),
|
||||
@@ -123,7 +123,8 @@ describe('queueForIngest', () => {
|
||||
expect(sha).toBe('new-sha')
|
||||
expect(added).toEqual(['https://dev.itch.io/b'])
|
||||
expect(JSON.parse(commits[0].additions['scripts/temp_link.json'])).toEqual(['https://dev.itch.io/queued', 'https://dev.itch.io/b'])
|
||||
expect(JSON.parse(commits[0].additions['scripts/deleted_games.json'])).toEqual([])
|
||||
expect(JSON.parse(commits[0].additions['scripts/state/unblocked.json'])).toEqual(['https://dev.itch.io/b'])
|
||||
expect(commits[0].additions['scripts/deleted_games.json']).toBeUndefined()
|
||||
})
|
||||
|
||||
it('makes no commit when everything is already queued', async () => {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
/** Regressions from the Phase 4 review (see the commit that added this file). */
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { serveAdminApp, handleAdminApi } from '../admin'
|
||||
import { applyAdminChange, dumps, upsertCountHistory } from '../catalog'
|
||||
import type { WorkerEnv } from '../env'
|
||||
import { readBodyCapped, readJson } from '../http'
|
||||
import { classify, QueueStore } from '../queue'
|
||||
import { testDb } from './d1'
|
||||
|
||||
describe('request bodies', () => {
|
||||
it('stops reading a body without Content-Length once it passes the limit', async () => {
|
||||
let pulled = 0
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
pulled += 1024
|
||||
controller.enqueue(new Uint8Array(1024))
|
||||
if (pulled > 1_000_000) controller.close()
|
||||
},
|
||||
})
|
||||
const request = new Request('https://x/api', { method: 'POST', body, duplex: 'half' } as RequestInit)
|
||||
await expect(readBodyCapped(request, 4096)).rejects.toMatchObject({ status: 413 })
|
||||
expect(pulled).toBeLessThan(16 * 1024)
|
||||
})
|
||||
|
||||
it('accepts only the application/json media type', async () => {
|
||||
const post = (type: string) =>
|
||||
new Request('https://x/api', { method: 'POST', headers: { 'content-type': type }, body: '{}' })
|
||||
await expect(readJson(post('application/json; charset=utf-8'), 100)).resolves.toEqual({})
|
||||
await expect(readJson(post('text/plain; application/json'), 100)).rejects.toMatchObject({ status: 415 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('local admin bypass', () => {
|
||||
const env = { DEV_ADMIN_EMAIL: 'dev@localhost', SITE_ORIGIN: 'https://freeitchgames.win' } as unknown as WorkerEnv
|
||||
|
||||
it('refuses cross-origin writes even on localhost', async () => {
|
||||
const url = new URL('http://localhost:8787/api/admin/queue/decide')
|
||||
const request = new Request(url, {
|
||||
method: 'POST',
|
||||
headers: { origin: 'https://evil.example', 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'reject', urls: ['https://dev.itch.io/a'] }),
|
||||
})
|
||||
expect((await handleAdminApi(request, url, env, null)).status).toBe(403)
|
||||
})
|
||||
|
||||
it('still serves reads and same-origin writes locally', async () => {
|
||||
const shellUrl = new URL('http://localhost:8787/admin/')
|
||||
const assets = { fetch: async () => new Response('<html></html>') } as unknown as Fetcher
|
||||
expect((await serveAdminApp(new Request(shellUrl), shellUrl, { ...env, ASSETS: assets } as WorkerEnv)).status).toBe(200)
|
||||
})
|
||||
})
|
||||
|
||||
describe('count history', () => {
|
||||
it('does not change the caller rows and rewrites today when it already has a row', () => {
|
||||
const history = [
|
||||
{ date: '2026-09-17', total: 2 },
|
||||
{ date: '2026-09-18', total: 3 },
|
||||
]
|
||||
const before = JSON.stringify(history)
|
||||
const out = upsertCountHistory(history, 2, new Date('2026-09-18T10:00:00Z'))
|
||||
expect(JSON.stringify(history)).toBe(before)
|
||||
expect(out.at(-1)).toEqual({ date: '2026-09-18', total: 2 })
|
||||
|
||||
const games = ['a', 'b', 'c'].map((s) => ({ url: `https://dev.itch.io/${s}`, name: s }))
|
||||
const change = applyAdminChange(
|
||||
{
|
||||
chunks: [games],
|
||||
index: { total_games: 3, max_per_file: 500, last_updated: 'x', files: [{ name: 'game_info_001.json', count: 3 }] },
|
||||
history,
|
||||
deleted: [],
|
||||
},
|
||||
new Map(),
|
||||
[{ url: 'https://dev.itch.io/a', reason: 'r' }],
|
||||
new Date('2026-09-18T10:00:00Z'),
|
||||
)
|
||||
expect(change.writes.get('data_game/count_history.json')).toBe(
|
||||
dumps([
|
||||
{ date: '2026-09-17', total: 2 },
|
||||
{ date: '2026-09-18', total: 2 },
|
||||
]),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('games that come back after being ingested and removed', () => {
|
||||
let db: D1Database
|
||||
let dispose: () => Promise<void>
|
||||
beforeAll(async () => {
|
||||
;({ db, dispose } = await testDb())
|
||||
}, 60_000)
|
||||
afterAll(() => dispose())
|
||||
|
||||
it('are reviewed again, flagged, instead of being blocked for good', async () => {
|
||||
const url = 'https://dev.itch.io/back'
|
||||
const deleted = new Map([[url, { reason: 'Game became paid', deleted_at: '2026-09-01T00:00:00Z' }]])
|
||||
const [result] = classify([url], { catalog: new Set(), deleted, known: new Map([[url, 'ingested']]) })
|
||||
expect(result.outcome).toBe('previously_deleted')
|
||||
expect(result.store).toBe(true)
|
||||
// Still in the catalog: still a duplicate, not reopened.
|
||||
const listed = classify([url], { catalog: new Set([url]), deleted: new Map(), known: new Map([[url, 'ingested']]) })
|
||||
expect(listed[0].outcome).toBe('duplicate_catalog')
|
||||
|
||||
const store = new QueueStore(db)
|
||||
await store.insert([{ url, source: 'rss' }])
|
||||
await store.setStatus([url], 'ingested', 'reconcile')
|
||||
await store.insert([{ url, source: 'rss', flags: ['previously_deleted'] }])
|
||||
const [row] = await store.get([url])
|
||||
expect(row.status).toBe('pending')
|
||||
expect(JSON.parse(row.flags)).toEqual(['previously_deleted'])
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user