Files
free-games-itchio-list/webapp/worker/repo.ts
T
poli0981andClaude Opus 5.5 999d0887ee Fix the SonarQube Cloud findings; release 4.4.2 (#155)
The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud.

Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh.

Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check.

Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41.

[skip-discuss]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 18:16:12 +07:00

282 lines
11 KiB
TypeScript

/**
* Admin writes to the repo, each as one commit built from the files at the
* current head of main (commitWithRetry re-reads and rebuilds on a race).
* Pure builders are exported for tests; the GitHub reads stay thin.
* Commit messages never name the maintainer (their email stays in the
* private D1 audit log): the repo and its history are public.
*/
import { canonicalize } from './canonical'
import {
applyAdminChange,
dumps,
type CatalogFiles,
type CatalogIndex,
type CountPoint,
type DeletedEntry,
type FieldEdit,
type Game,
} from './catalog'
import { GITHUB_CONFIG, missingConfig, type WorkerEnv } from './env'
import { commitWithRetry, headOid, listDir, readFile, type CommitInput, type GitHubEnv } from './github'
import { compareCodeUnits } from './text'
const TEMP_LINK = 'scripts/temp_link.json'
const DELETED_LOG = 'scripts/deleted_games.json'
const UNBLOCKED = 'scripts/state/unblocked.json'
const INGEST_STATE = 'scripts/state/ingest_state.json'
const INGEST_OUTCOMES = 'scripts/state/ingest_outcomes.json'
const CHUNK_FILE = /^game_info_\d+\.json$/
type Change = Omit<CommitInput, 'expectedHeadOid'>
/** Who asked for a queue commit: the maintainer on the admin page, or its automatic rules (cron). */
export type QueueTag = 'admin' | 'auto'
/** Writes to the repo; injectable so tests don't talk to GitHub. */
export interface RepoOps {
queueForIngest(urls: string[], unblock: string[], tag?: QueueTag): Promise<{ sha: string | null; added: string[] }>
editCatalog(edits: Map<string, FieldEdit>, removals: { url: string; reason: string }[]): Promise<CatalogEditResult>
ingestView(): Promise<IngestView>
}
/** The repo writer, or null when the GitHub App is not configured (read-only admin, no automation). */
export function repoOps(env: WorkerEnv): RepoOps | null {
if (missingConfig(env, GITHUB_CONFIG).length > 0) return null
const gh = env as WorkerEnv & GitHubEnv
return {
queueForIngest: (urls, unblock, tag) => queueForIngest(gh, urls, unblock, tag),
editCatalog: (edits, removals) => editCatalog(gh, edits, removals),
ingestView: () => readIngestView(gh),
}
}
async function readJsonFile<T>(env: GitHubEnv, path: string, head: string, fallback: T): Promise<T> {
const text = await readFile(env, path, head)
return text === null ? fallback : (JSON.parse(text) as T)
}
/** The catalog files at `head`, chunks in the same order as data_store.py. */
async function readCatalog(env: GitHubEnv, head: string): Promise<CatalogFiles> {
const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort(compareCodeUnits)
const [chunks, index, history, deleted] = await Promise.all([
Promise.all(names.map((n) => readJsonFile<Game[]>(env, `data_game/${n}`, head, []))),
readJsonFile<CatalogIndex | null>(env, 'data_game/index.json', head, null),
readJsonFile<CountPoint[]>(env, 'data_game/count_history.json', head, []),
readJsonFile<DeletedEntry[]>(env, DELETED_LOG, head, []),
])
return { chunks, index, history, deleted }
}
/**
* temp_link.json with `urls` appended (skipping ones already queued in any
* spelling). Unknown entries are kept as they are; apply_patch.py cleans them.
*/
export function withQueued(queueText: string | null, urls: string[]): { text: string; added: string[] } {
const parsed: unknown = queueText === null ? [] : JSON.parse(queueText)
const queue: unknown[] = Array.isArray(parsed) ? parsed : []
const present = new Set(queue.map((u) => (typeof u === 'string' ? (canonicalize(u) ?? u) : '')))
const added: string[] = []
for (const url of urls) {
if (present.has(url)) continue
present.add(url)
added.push(url)
}
return { text: dumps([...queue, ...added]), added }
}
/**
* scripts/state/unblocked.json with `urls` added: removed games the maintainer
* allows back in. update_info.py scrapes them despite the deleted log and drops
* the log entry only when the game is really re-added (a still-paid or dead
* game keeps its public removal record).
*/
export function withUnblocked(currentText: string | null, urls: string[]): { text: string; added: number } {
const parsed: unknown = currentText === null ? [] : JSON.parse(currentText)
const current = Array.isArray(parsed) ? parsed.filter((u): u is string => typeof u === 'string') : []
const next = [...new Set([...current, ...urls])].sort(compareCodeUnits)
return { text: dumps(next), added: next.length - current.length }
}
const QUEUE_BODY: Record<QueueTag, string> = {
admin: 'Via the admin page.',
auto: "By the admin page's automatic rules (auto-approve, watched collections).",
}
/**
* Queue `urls` for the ingest workflow (the push to temp_link.json triggers
* update.yml). With `unblock`, also let those previously removed games
* through the ingest (see withUnblocked).
*/
export async function queueForIngest(
env: GitHubEnv,
urls: string[],
unblock: string[],
tag: QueueTag = 'admin',
): Promise<{ sha: string | null; added: string[] }> {
let added: string[] = []
const sha = await commitWithRetry(env, async (head): Promise<Change | null> => {
const writes = new Map<string, string>()
const queued = withQueued(await readFile(env, TEMP_LINK, head), urls)
added = queued.added
if (queued.added.length > 0) writes.set(TEMP_LINK, queued.text)
let unblocked = 0
if (unblock.length > 0) {
const allowed = withUnblocked(await readFile(env, UNBLOCKED, head), unblock)
unblocked = allowed.added
if (allowed.added > 0) writes.set(UNBLOCKED, allowed.text)
}
if (writes.size === 0) return null
return {
headline:
added.length > 0
? `Queue ${added.length} game(s) for ingest [${tag}]`
: `Unblock ${unblocked} removed game(s) [${tag}]`,
body: `${QUEUE_BODY[tag]}\n\n${(added.length > 0 ? added : unblock).join('\n')}`,
writes,
}
})
return { sha, added }
}
export interface CatalogEditResult {
sha: string | null
summary: { edited: number; removed: number }
/** Per URL: edited, already as asked, removed, or not in the catalog. */
edited: string[]
unchanged: string[]
removed: string[]
missing: string[]
/** Thumbnails of removed games (their resized copies are deleted from R2). */
thumbnails: string[]
}
/** Edit maintainer fields and/or remove games in one commit. */
export async function editCatalog(
env: GitHubEnv,
edits: Map<string, FieldEdit>,
removals: { url: string; reason: string }[],
now: () => Date = () => new Date(),
): Promise<CatalogEditResult> {
let result: CatalogEditResult = {
sha: null,
summary: { edited: 0, removed: 0 },
edited: [],
unchanged: [],
removed: [],
missing: [],
thumbnails: [],
}
const sha = await commitWithRetry(env, async (head): Promise<Change | null> => {
const files = await readCatalog(env, head)
const removing = new Set(removals.map((r) => r.url))
const thumbnails = files.chunks
.flat()
.filter((g) => removing.has(g.url) && typeof g.thumbnail === 'string' && g.thumbnail.startsWith('https://'))
.map((g) => g.thumbnail as string)
const change = applyAdminChange(files, edits, removals, now())
result = { sha: null, summary: change.summary, ...change.urls, thumbnails }
if (change.writes.size === 0 && change.deletes.length === 0) return null
const parts = [
change.summary.edited ? `edit ${change.summary.edited}` : '',
change.summary.removed ? `remove ${change.summary.removed}` : '',
].filter(Boolean)
const editedUrls = new Set(change.urls.edited)
const removedUrls = new Set(change.urls.removed)
const lines = [
...[...edits].filter(([url]) => editedUrls.has(url)).map(([url, edit]) => `edit ${url} ${JSON.stringify(edit)}`),
...removals.filter((r) => removedUrls.has(r.url)).map((r) => `remove ${r.url} (${r.reason})`),
]
return {
headline: `Catalog: ${parts.join(', ')} [admin]`,
body: `Via the admin page.\n\n${lines.join('\n')}`,
writes: change.writes,
deletes: change.deletes,
}
})
return { ...result, sha }
}
/**
* What the ingest pipeline knows about queued games, at one commit:
* the queue itself, failed attempts (still retried) and final outcomes.
*/
export interface IngestView {
head: string
/** Canonical URLs still in scripts/temp_link.json. */
queued: Set<string>
attempts: Map<string, { attempts: number; error: string | null }>
outcomes: Map<string, { outcome: string; at: string }>
}
function parsed(text: string | null): unknown {
if (text === null) return null
try {
return JSON.parse(text)
} catch {
return null
}
}
function isRecord(v: unknown): v is Record<string, unknown> {
return typeof v === 'object' && v !== null && !Array.isArray(v)
}
/** The three pipeline files, read defensively (a malformed file counts as empty). */
export function parseIngestView(
head: string,
queueText: string | null,
stateText: string | null,
outcomesText: string | null,
): IngestView {
const queue = parsed(queueText)
const queued = new Set<string>()
if (Array.isArray(queue)) {
for (const item of queue) {
const raw = isRecord(item) ? item.url : item
const url = typeof raw === 'string' ? canonicalize(raw) : null
if (url) queued.add(url)
}
}
const attempts = new Map<string, { attempts: number; error: string | null }>()
const state = parsed(stateText)
if (isRecord(state)) {
for (const [url, entry] of Object.entries(state)) {
if (isRecord(entry) && typeof entry.attempts === 'number') {
attempts.set(url, { attempts: entry.attempts, error: typeof entry.last_error === 'string' ? entry.last_error : null })
}
}
}
const outcomes = new Map<string, { outcome: string; at: string }>()
const outcomeMap = parsed(outcomesText)
if (isRecord(outcomeMap)) {
for (const [url, entry] of Object.entries(outcomeMap)) {
if (isRecord(entry) && typeof entry.outcome === 'string' && typeof entry.at === 'string') {
outcomes.set(url, { outcome: entry.outcome, at: entry.at })
}
}
}
return { head, queued, attempts, outcomes }
}
// Per isolate, keyed by the head commit; only a resolved value is kept
// (gotcha 15: never share an in-flight promise across requests).
let viewCache: IngestView | undefined
export function resetIngestViewCache(): void {
viewCache = undefined
}
/** The ingest view at the current head: one GitHub call while main has not moved, four when it has. */
export async function readIngestView(env: GitHubEnv): Promise<IngestView> {
const head = await headOid(env)
if (viewCache?.head === head) return viewCache
const [queue, state, outcomes] = await Promise.all([
readFile(env, TEMP_LINK, head),
readFile(env, INGEST_STATE, head),
readFile(env, INGEST_OUTCOMES, head),
])
const view = parseIngestView(head, queue, state, outcomes)
viewCache = view
return view
}