Files
poli0981andClaude Fable 5.2 0e0e442de5 chore: maintenance mode, Qodana analysis; 4.6.1 (#188) [skip-discuss]
The project moves to maintenance mode (stable, feature-complete): docs, templates and badges (EN + VI). Qodana analysis in CI for the TypeScript / JavaScript, Python and Rust projects, with the findings of its first reports fixed. Version 4.6.1.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-11 21:44:32 +07:00

524 lines
20 KiB
Python

"""
refresh.py — Rotating health check for the whole catalog (one GET per game).
Replaces check_paid / check_alive / update_reviews / update_status (which each
crawled all 2,600+ pages, overran their job timeouts and lost everything) and
force_update (now `--full`).
Each run checks a slice of the catalog — games with a pending strike first,
then the least recently checked — about 1/7 of the catalog by default, so
every game is re-checked weekly. From that single page it derives:
- alive: 404/410 → "dead" strike
- price: paid → "paid" strike
- access: itch.io's password form instead of the game → "private" strike
- rating, rating_count, status, ai_disclosure / ai_content / accessibility
(+ backfills release_date / thumbnail / updated_at when missing)
A game is removed only when the same strike is seen again at least
MIN_STRIKE_GAP after it was first seen (a single 404 or a flaky price widget
no longer deletes a game). If a run would remove, or newly strike, more than
max(10, 10%) of the games it checked, all its removals are withheld (marked
`withheld_at`, so no later short run removes them either) and the run is
flagged: that pattern means itch.io changed its markup, not that hundreds of
games went paid overnight. A run with --allow-mass-removal applies them; a
healthy check clears them.
Nothing is written to the catalog here. The run emits a patch (see patch.py)
that apply_patch.py applies to the latest `main`. The patch is checkpointed
every CHECKPOINT_EVERY games and the run stops cleanly before --deadline-min,
so a scan that fails or hits its step timeout still hands over its progress
(refresh.yml applies it; a manually cancelled run applies nothing). A
checkpoint never carries removals — those games keep their matured strike
and are first in line for the next run.
Usage:
python scripts/refresh.py --out patch.json [--budget N] [--deadline-min 45]
python scripts/refresh.py --full [--url URL ...] --out patch.json # re-scrape all fields
"""
import argparse
import copy
import math
import os
import sys
import time
from datetime import datetime, timedelta
sys.path.insert(0, os.path.dirname(__file__))
from canonical import canonicalize
from data_store import load_all_games
from json_io import load_json
from patch import new_patch, write_patch
from scraper import (
NA,
Pacer,
Page,
ai_and_accessibility,
create_session,
extract_thumbnail,
fetch_page,
find_title,
has_info_panel,
is_free_game,
is_password_protected,
now_iso,
parse_game,
parse_info_table,
)
STATE_FILE = "scripts/state/refresh_state.json"
# User-editable annotations: never overwritten by any scrape.
PRESERVE_FIELDS = ("safe_virus", "notes", "nsfw")
# A full re-scrape keeps these when the page no longer shows them: itch.io shows
# the "Published" row only for a few weeks, and a game's cover doesn't go away.
KEEP_KNOWN_FIELDS = ("release_date", "thumbnail")
DEAD_CODES = {404, 410}
CHECKPOINT_EVERY = 50
MAX_CONSECUTIVE_429 = 3
DEFAULT_CYCLE_DAYS = 7
MIN_STRIKE_GAP = timedelta(hours=20)
MASS_FLOOR = 10
MASS_SHARE = 0.10
REASON_PAID = "Game became paid"
REASON_PRIVATE = "Game page is password-protected"
def reason_dead(code: int) -> str:
return f"Game page no longer exists (HTTP {code})"
def removal_kind(reason: str) -> str:
"""The strike kind ("dead", "paid", "private") a removal reason was written for."""
return {REASON_PAID: "paid", REASON_PRIVATE: "private"}.get(reason, "dead")
# ---------------------------------------------------------------------------
# Target selection
# ---------------------------------------------------------------------------
def default_budget(total: int) -> int:
return max(1, math.ceil(total / DEFAULT_CYCLE_DAYS))
def _ts(iso: str) -> datetime:
return datetime.fromisoformat(iso.replace("Z", "+00:00"))
def strike_matured(entry: dict, now: str) -> bool:
"""Whether the entry's strike was first seen at least MIN_STRIKE_GAP before `now`."""
first = entry.get("strike_at")
return bool(entry.get("strike") and first) and _ts(now) - _ts(first) >= MIN_STRIKE_GAP
def mass_limit(checked: int) -> int:
"""Most removals (or new strikes) one run may produce before it is flagged."""
return max(MASS_FLOOR, math.ceil(MASS_SHARE * checked))
def select_targets(
games: list[dict],
state: dict,
budget: int,
now: str,
full: bool = False,
allow_mass_removal: bool = False,
) -> list[dict]:
"""Matured strikes first, then least recently checked.
Never-checked games sort first; ties keep catalog order (stable sort).
Full mode rotates on its own `full` timestamp and ignores strikes.
Games withheld by the mass-change guard (`withheld_at`) would otherwise
hold the front of the line until someone acts: without
`allow_mass_removal` (the run that confirms them) they take at most half
of the budget, so the rotation keeps moving while the alert repeats.
"""
stamp = "full" if full else "checked"
matured: list[dict] = []
withheld: list[dict] = []
if not full:
for g in games:
entry = state.get(g["url"], {})
if strike_matured(entry, now):
(withheld if entry.get("withheld_at") else matured).append(g)
if allow_mass_removal:
matured, withheld = withheld + matured, []
else:
withheld = withheld[: max(1, budget // 2)]
first = matured + withheld
first_urls = {g["url"] for g in first}
rest = sorted(
(g for g in games if g["url"] not in first_urls),
key=lambda game: state.get(game["url"], {}).get(stamp, ""),
)
return (first + rest)[:budget]
# ---------------------------------------------------------------------------
# Per-game decision (pure: no I/O)
# ---------------------------------------------------------------------------
def refresh_updates(game: dict, soup) -> dict:
"""Changed scraper-owned fields for a normal (non-full) refresh."""
updates: dict = {}
if not has_info_panel(soup):
return updates
info = parse_info_table(soup)
current = {
"status": info.get("Status", NA),
"rating": info.get("Rating", NA),
"rating_count": info.get("RatingCount", NA),
**ai_and_accessibility(info),
}
for field, value in current.items():
if game.get(field) != value:
updates[field] = value
if game.get("release_date", NA) == NA:
published = info.get("Release date") or info.get("Published")
if published and published != NA:
updates["release_date"] = published
if info.get("Updated") and info["Updated"] != game.get("updated_at"):
updates["updated_at"] = info["Updated"]
if game.get("thumbnail", NA) == NA:
thumb = extract_thumbnail(soup)
if thumb != NA:
updates["thumbnail"] = thumb
return updates
def full_updates(game: dict, soup) -> dict:
"""Every scraper-owned field that differs from a fresh scrape."""
fresh = parse_game(soup, game["url"])
fresh.pop("is_free", None)
for field in KEEP_KNOWN_FIELDS:
if fresh.get(field) == NA and game.get(field, NA) != NA:
fresh[field] = game[field]
return {
k: v
for k, v in fresh.items()
if k != "url" and k not in PRESERVE_FIELDS and game.get(k) != v
}
def check_game(
game: dict, page: Page, entry: dict, now: str, full: bool = False
) -> tuple[str, dict, dict | None, dict]:
"""Decide what one fetched page (fetched at `now`) means for `game`.
Returns (outcome, updates, removal-or-None, new state entry).
"""
entry = dict(entry)
def strike(kind: str, reason: str) -> tuple[str, dict, dict | None, dict]:
if entry.get("strike") == kind and strike_matured(entry, now):
removal = {
"url": game["url"],
"name": game.get("name", NA),
"reason": reason,
"deleted_at": now,
}
return f"removed_{kind}", {}, removal, {}
if entry.get("strike") != kind:
entry["strike"] = kind
entry["strike_at"] = now
entry.pop("withheld_at", None)
entry["checked"] = now
if full:
entry["full"] = now # don't head every following full batch
return f"strike_{kind}", {}, None, entry
if page.status is None or (page.status != 200 and page.status not in DEAD_CODES):
entry["fails"] = entry.get("fails", 0) + 1
return "error", {}, None, entry
if page.status in DEAD_CODES:
return strike("dead", reason_dead(page.status))
soup = page.soup
if soup is not None and is_password_protected(soup):
return strike("private", REASON_PRIVATE)
if soup is None or find_title(soup) == NA:
entry["fails"] = entry.get("fails", 0) + 1
return "parse_fail", {}, None, entry
if not is_free_game(soup):
return strike("paid", REASON_PAID)
for key in ("strike", "strike_at", "withheld_at", "fails"):
entry.pop(key, None)
entry["checked"] = now
if full:
entry["full"] = now
updates = full_updates(game, soup)
else:
updates = refresh_updates(game, soup)
return ("changed" if updates else "ok"), updates, None, entry
# ---------------------------------------------------------------------------
# Run
# ---------------------------------------------------------------------------
def fetch_with_backoff(session, url: str, pacer: Pacer, counters: dict) -> Page | None:
"""GET with one retry after a 429/503. None = stop the run (rate limited)."""
page = Page(status=None)
for _ in range(2):
pacer.wait()
page = fetch_page(session, url)
if page.status not in (429, 503):
counters["consecutive_429"] = 0
return page
counters["http_429"] += 1
counters["consecutive_429"] += 1
if counters["consecutive_429"] >= MAX_CONSECUTIVE_429:
return None
pacer.back_off(page.retry_after or 60.0)
return page
def _write_outputs(stats: dict) -> None:
out = os.environ.get("GITHUB_OUTPUT")
if out:
with open(out, "a", encoding="utf-8") as f:
f.write(f"rate_limited={'true' if stats['rate_limited'] else 'false'}\n")
f.write(f"mass_change={'true' if stats['mass_change'] else 'false'}\n")
for key in ("checked", "changed", "removed"):
f.write(f"{key}={stats[key]}\n")
summary = os.environ.get("GITHUB_STEP_SUMMARY")
if summary:
rows = [
("Mode", stats["mode"]),
("Targets", stats["targets"]),
("Checked", stats["checked"]),
("Fields changed (games)", stats["changed"]),
("New strikes", stats["strikes"]),
(
"Removed (dead / paid / private)",
f"{stats['removed_dead']} / {stats['removed_paid']} / {stats['removed_private']}",
),
("Removals withheld (mass-change guard)", stats["withheld"]),
("Errors / parse failures", f"{stats['errors']} / {stats['parse_fail']}"),
("HTTP 429/503", stats["http_429"]),
("Stopped", stats["stopped"] or "completed"),
("Duration", f"{float(stats['duration_s']) / 60:.1f} min"),
]
with open(summary, "a", encoding="utf-8") as f:
f.write("### Catalog refresh\n\n| | |\n|---|---|\n")
f.writelines(f"| {k} | {v} |\n" for k, v in rows)
def keep_struck(patch: dict, state: dict, removals: list[dict], stats: dict, mark: bool) -> None:
"""Turn `removals` back into state entries that keep their matured strike.
The games stay first in line for the next run; `mark` records that the
removal was withheld by the mass-change guard. A full run also moves the
game's `full` stamp, like strike() does, so it doesn't head every batch.
"""
urls = {r["url"] for r in removals}
for removal in removals:
entry = {**state.get(removal["url"], {}), "checked": removal["deleted_at"]}
if mark:
entry["withheld_at"] = removal["deleted_at"]
if patch.get("kind") == "full":
entry["full"] = removal["deleted_at"]
patch["refresh_state"][removal["url"]] = entry
stats["removed"] -= 1
stats["removed_" + removal_kind(removal["reason"])] -= 1
patch["removals"] = [r for r in patch["removals"] if r["url"] not in urls]
def mark_suspect_strikes(patch: dict) -> int:
"""Mark every strike this run recorded as withheld (a spike run's strikes
are suspect), so no later run removes them without allow_mass_removal."""
marked = 0
for entry in patch["refresh_state"].values():
if entry and entry.get("strike") and not entry.get("withheld_at"):
entry["withheld_at"] = entry["checked"]
marked += 1
return marked
def guard_mass_change(patch: dict, state: dict, stats: dict, allow_removal: bool) -> None:
"""Withhold removals that look like a parser fault (unless allow_removal).
- more strikes than mass_limit(checked) in this run (a spike): the run's
removals are withheld and all its strikes are marked, so their removal
also waits for an explicit allow_mass_removal run;
- more removals than mass_limit(checked): the run's removals are withheld;
- a game already withheld by an earlier run stays withheld, whatever the
size of this run (a short or single-URL run must not slip it through).
"""
limit = mass_limit(stats["checked"])
spike = stats["strikes"] > limit
if spike:
stats["mass_change"] = True
print(
f"::error::{stats['strikes']} strikes out of {stats['checked']} checked games "
f"(limit {limit}) — check whether itch.io changed its page markup."
)
if not allow_removal:
mark_suspect_strikes(patch)
if allow_removal or not patch["removals"]:
return
if spike or stats["removed"] > limit:
held = list(patch["removals"])
why = "a strike spike" if spike else f"{stats['removed']} removals (limit {limit})"
else:
held = [r for r in patch["removals"] if state.get(r["url"], {}).get("withheld_at")]
why = "games withheld by an earlier run"
if not held:
return
stats["mass_change"] = True
stats["withheld"] += len(held)
print(
f"::error::Withholding {len(held)} removal(s) because of {why}. "
"If they are real, dispatch the refresh with allow_mass_removal."
)
keep_struck(patch, state, held, stats, mark=True)
def checkpoint_copy(patch: dict, state: dict, allow_removal: bool) -> dict:
"""The patch as a partial run may apply it: progress, strikes, no removals.
If the run so far already looks like a spike, its strikes (and deferred
removals) are marked as withheld, as the final guard would do.
"""
snap = copy.deepcopy(patch)
stats = snap["stats"]
spike = not allow_removal and stats["strikes"] > mass_limit(stats["checked"])
if spike:
mark_suspect_strikes(snap)
keep_struck(snap, state, list(snap["removals"]), stats, mark=spike)
stats["stopped"] = stats["stopped"] or "checkpoint"
return snap
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
ap.add_argument("--out", required=True, help="patch file to write")
ap.add_argument("--budget", type=int, default=0, help="games to check (0 = 1/7 of catalog)")
ap.add_argument("--deadline-min", type=float, default=45.0)
ap.add_argument("--full", action="store_true", help="re-scrape every scraper-owned field")
ap.add_argument("--url", action="append", default=[], help="only these game URLs")
ap.add_argument(
"--allow-mass-removal",
action="store_true",
help="apply removals even when the mass-change guard trips",
)
args = ap.parse_args(argv)
games = load_all_games()
state: dict = load_json(STATE_FILE, default={})
run_start, started = now_iso(), time.monotonic()
if args.url:
wanted = {canonicalize(u) or u.strip() for u in args.url if u.strip()}
targets = [g for g in games if g["url"] in wanted]
missing = wanted - {g["url"] for g in targets}
if missing:
print(f"URL(s) not in the catalog: {', '.join(sorted(missing))}")
return 1
else:
budget = args.budget or default_budget(len(games))
targets = select_targets(
games,
state,
budget,
run_start,
full=args.full,
allow_mass_removal=args.allow_mass_removal,
)
kind = "full" if args.full else "refresh"
patch = new_patch(kind)
stats = dict.fromkeys(
(
"checked",
"changed",
"strikes",
"removed",
"removed_dead",
"removed_paid",
"removed_private",
"errors",
"parse_fail",
"http_429",
"withheld",
"duration_s",
),
0,
)
stats.update(mode=kind, targets=len(targets), rate_limited=False, mass_change=False, stopped="")
patch["stats"] = stats
counters = {"http_429": 0, "consecutive_429": 0}
print(
f"Refreshing {len(targets)} of {len(games)} games ({kind}); "
f"deadline {args.deadline_min:g} min."
)
session, pacer = create_session(), Pacer()
for i, game in enumerate(targets, start=1):
elapsed = time.monotonic() - started
per_game = elapsed / (i - 1) if i > 1 else 6.0
if elapsed + 1.5 * per_game > args.deadline_min * 60:
stats["stopped"] = "deadline"
print(f"Deadline reached after {i - 1} games — stopping cleanly.")
break
url = game["url"]
page = fetch_with_backoff(session, url, pacer, counters)
stats["http_429"] = counters["http_429"]
if page is None:
stats["rate_limited"], stats["stopped"] = True, "rate_limited"
print(
f"::warning::itch.io rate-limited {MAX_CONSECUTIVE_429}x in a row — "
"saving progress and stopping."
)
break
if page.status in (429, 503):
# Still throttled after one retry: not the game's fault, leave its state alone.
print(f"[{i}/{len(targets)}] {'throttled':<13} {url}")
continue
outcome, updates, removal, entry = check_game(
game, page, state.get(url, {}), now_iso(), full=args.full
)
stats["checked"] += 1
if updates:
patch["updates"][url] = updates
stats["changed"] += 1
if removal:
patch["removals"].append(removal)
patch["refresh_state"][url] = None
stats["removed"] += 1
stats["removed_" + outcome.split("_", 1)[1]] += 1
else:
patch["refresh_state"][url] = entry
if outcome.startswith("strike_"):
stats["strikes"] += 1
elif outcome in ("error", "parse_fail"):
stats["errors" if outcome == "error" else "parse_fail"] += 1
detail = f" {sorted(updates)}" if updates else ""
print(f"[{i}/{len(targets)}] {outcome:<13} {url}{detail}")
if i % CHECKPOINT_EVERY == 0:
rate = i / max(time.monotonic() - started, 1e-6)
print(f" … {i} checked, {rate * 60:.1f}/min, {counters['http_429']}x 429 — checkpoint")
stats["duration_s"] = round(time.monotonic() - started)
write_patch(args.out, checkpoint_copy(patch, state, args.allow_mass_removal))
guard_mass_change(patch, state, stats, allow_removal=args.allow_mass_removal)
stats["duration_s"] = round(time.monotonic() - started)
write_patch(args.out, patch)
_write_outputs(stats)
print(
f"\nDone — {stats['checked']} checked, {stats['changed']} changed, "
f"{stats['strikes']} new strikes, {stats['removed']} removed, "
f"{stats['errors'] + stats['parse_fail']} errors, {stats['http_429']}x 429."
)
return 0
if __name__ == "__main__":
sys.exit(main())