Files
free-games-itchio-list/scripts/update_info.py
T
poli0981andClaude Opus 5.5 999d0887ee Fix the SonarQube Cloud findings; release 4.4.2 (#155)
The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud.

Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh.

Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check.

Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41.

[skip-discuss]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 18:16:12 +07:00

223 lines
9.0 KiB
Python

"""
update_info.py — Scrape queued game links into a pipeline patch.
Flow:
1. Read URLs from scripts/temp_link.json (+ INPUT_URL from workflow_dispatch)
2. Canonicalize; drop invalid URLs, duplicates, games already in the catalog
and games in scripts/deleted_games.json (unless the admin restored them:
scripts/state/unblocked.json)
3. Fetch each page → keep only free games (with an `added_at` stamp); a
password-protected page ends the link like a dead one
4. Emit a patch (see patch.py). URLs that finished (added / paid / dead /
skipped) are dropped from the queue; transient failures stay queued and
are retried; a link is given up after MAX_ATTEMPTS failures at least
RETRY_GAP apart (tracked in scripts/state/ingest_state.json), so a burst
of back-to-back runs during a short itch.io outage cannot use them all up.
A page still throttled (429) after the retry is not the link's fault and
counts as no attempt.
A dispatched INPUT_URL is processed first and also written to the queue
(`queue_add`), so it survives a retry, a deadline or a rate-limit stop.
A second spelling of a queued game is left alone: the first copy's
outcome removes every spelling (apply_patch compares canonical forms).
5. How each link ended (added, paid, dead, private, duplicate, deleted,
gave_up) goes to `ingest_outcomes` → scripts/state/ingest_outcomes.json,
which the admin page reads to follow a batch it queued.
apply_patch.py then applies the patch to the latest `main`, so links queued
while this ran (admin, cron) are never overwritten.
Usage: python scripts/update_info.py --out patch.json [--deadline-min 50]
"""
import argparse
import os
import sys
import time
from datetime import datetime, timedelta
sys.path.insert(0, os.path.dirname(__file__))
from canonical import canonicalize
from data_store import get_all_urls, load_all_games
from json_io import load_json
from patch import new_patch, write_patch
from refresh import fetch_with_backoff
from scraper import NA, Pacer, create_session, is_password_protected, now_iso, parse_game
TEMP_LINK = "scripts/temp_link.json"
DELETED_LOG = "scripts/deleted_games.json"
INGEST_STATE = "scripts/state/ingest_state.json"
# Removed games the maintainer asked to bring back (admin Restore / approve with
# override). They are scraped despite the deleted log; the log entry goes only
# once the game is really re-added, so a still-paid game keeps its record.
UNBLOCKED = "scripts/state/unblocked.json"
MAX_ATTEMPTS = 3
RETRY_GAP = timedelta(hours=6)
DEAD_CODES = {404, 410}
def _ts(iso: str) -> datetime:
return datetime.fromisoformat(iso.replace("Z", "+00:00"))
def failed_attempt(previous: dict, outcome: str, now: str) -> dict:
"""The ingest_state entry after a failed fetch at `now`.
Only failures at least RETRY_GAP after the last counted one add an
attempt; a link queued again after we gave up starts a fresh round.
"""
if previous.get("gave_up") or not previous.get("attempts"):
return {"attempts": 1, "last_attempt": now, "last_error": outcome}
last = previous.get("last_attempt")
if last and _ts(now) - _ts(last) < RETRY_GAP:
return {**previous, "last_error": outcome}
return {"attempts": previous["attempts"] + 1, "last_attempt": now, "last_error": outcome}
def record_outcome(patch: dict, url: str, outcome: str) -> None:
"""How a queued link ended, for the admin page's tracking of the batches it queued."""
patch["ingest_outcomes"][url] = {"outcome": outcome, "at": now_iso()}
def main(argv: list[str] | None = None) -> None:
ap = argparse.ArgumentParser(description="Scrape queued links into a patch.")
ap.add_argument("--out", required=True, help="patch file to write")
ap.add_argument("--deadline-min", type=float, default=50.0)
args = ap.parse_args(argv)
queue: list = load_json(TEMP_LINK)
if not isinstance(queue, list):
queue = []
patch = new_patch("ingest")
input_url = os.environ.get("INPUT_URL", "").strip()
if input_url:
# First, so a backlog can't starve it; queued, so it outlives this run.
queue = [input_url, *queue]
patch["queue_add"].append(canonicalize(input_url) or input_url)
print(f"Added from workflow input: {input_url}")
stats = dict.fromkeys(
(
"queued",
"added",
"paid",
"dead",
"private",
"duplicate",
"deleted",
"invalid",
"retry",
"gave_up",
),
0,
)
stats["queued"] = len(queue)
patch["stats"] = stats
if not queue:
print("No links to process (temp_link.json empty and no INPUT_URL).")
write_patch(args.out, patch)
return
catalog_urls = get_all_urls(load_all_games())
deleted_urls = {e.get("url") for e in load_json(DELETED_LOG)}
unblocked = set(load_json(UNBLOCKED, default=[]))
ingest_state: dict = load_json(INGEST_STATE, default={})
todo: list[str] = []
seen: set[str] = set()
for raw in queue:
url = canonicalize(raw) if isinstance(raw, str) else None
if url is None:
print(f"Skip invalid: {raw!r}")
stats["invalid"] += 1
if isinstance(raw, str): # non-strings are dropped by apply_patch itself
patch["queue_remove"].append(raw)
continue
if url in seen:
continue # the first copy's final outcome drops every spelling
seen.add(url)
if url in catalog_urls:
print(f"Skip duplicate: {url}")
stats["duplicate"] += 1
record_outcome(patch, url, "duplicate")
if url in unblocked:
patch["unlog"].append(url)
elif url in deleted_urls and url not in unblocked:
print(f"Skip previously deleted: {url} (restore it from the admin page)")
stats["deleted"] += 1
record_outcome(patch, url, "deleted")
else:
todo.append(url)
continue
patch["queue_remove"].append(raw)
patch["ingest_state"][url] = None
session, pacer, started = create_session(), Pacer(), time.monotonic()
counters = {"http_429": 0, "consecutive_429": 0}
for idx, url in enumerate(todo, start=1):
if time.monotonic() - started > args.deadline_min * 60:
print("Deadline reached — the rest stays queued for the next run.")
break
print(f"[{idx}/{len(todo)}] Scraping: {url}")
page = fetch_with_backoff(session, url, pacer, counters)
if page is None:
print("::warning::itch.io rate limit — the rest stays queued for the next run.")
break
if page.status == 429:
print(" → throttled; stays queued, no attempt counted.")
stats["retry"] += 1
continue
outcome = ""
if page.status == 200 and page.soup is not None and is_password_protected(page.soup):
outcome = "private" # final: nobody can open it, so retrying won't help
elif page.status == 200 and page.soup is not None:
record = parse_game(page.soup, url)
if record["name"] == NA:
outcome = "error: no title"
elif not record.pop("is_free"):
outcome = "paid"
else:
record["added_at"] = now_iso()
patch["additions"].append(record)
outcome = "added"
elif page.status in DEAD_CODES:
outcome = "dead"
else:
outcome = f"error: HTTP {page.status or page.error}"
if outcome.startswith("error"):
entry = failed_attempt(ingest_state.get(url, {}), outcome, now_iso())
if entry["attempts"] >= MAX_ATTEMPTS:
print(f" → {outcome}; giving up after {entry['attempts']} attempts.")
patch["queue_remove"].append(url)
patch["ingest_state"][url] = {**entry, "gave_up": entry["last_attempt"]}
stats["gave_up"] += 1
record_outcome(patch, url, "gave_up")
if url in unblocked:
patch["unblock_remove"].append(url)
else:
print(f" → {outcome}; stays queued ({entry['attempts']}/{MAX_ATTEMPTS} attempts).")
patch["ingest_state"][url] = entry
stats["retry"] += 1
continue
print(f" → {outcome}")
stats[outcome] += 1
record_outcome(patch, url, outcome)
patch["queue_remove"].append(url)
patch["ingest_state"][url] = None
if url in unblocked:
patch["unlog" if outcome == "added" else "unblock_remove"].append(url)
write_patch(args.out, patch)
print(
f"\nDone — {stats['added']} added, {stats['paid']} paid, {stats['dead']} dead, "
f"{stats['private']} password-protected, "
f"{stats['duplicate']} duplicates, {stats['deleted']} previously deleted, "
f"{stats['invalid']} invalid, {stats['retry']} to retry, {stats['gave_up']} given up."
)
if __name__ == "__main__":
main()