Files
free-games-itchio-list/scripts/apply_patch.py
T
poli0981andClaude Opus 5 41a8f8461f fix: Phase 4 review findings (admin, ingest, suggest)
Worker
- The local DEV_ADMIN_EMAIL bypass keeps the same-origin check for
  writes, and JSON bodies must be exactly application/json (a cross-site
  `text/plain; application/json` POST no longer gets through without a
  preflight).
- Request bodies are capped while they stream (a chunked upload without
  Content-Length is never buffered past 4 KB / 64 KB).
- count_history.json is rewritten when today's row already exists
  (upsertCountHistory no longer mutates the rows it was given).
- A game that entered the catalog through the queue, was removed and turns
  up again is reviewed again (flagged previously removed) instead of being
  blocked as "already pending" for good.
- Restoring / approving a removed game no longer deletes its public
  removal record up front: the Worker allow-lists it in
  scripts/state/unblocked.json, update_info.py scrapes it despite the
  deleted log, and apply_patch.py drops the log entry only when the game is
  really re-added (a still-paid or dead game keeps its record).

Admin app
- The editor compares against what the repository holds (published data
  or what was saved in this session), so saved changes can be reverted and
  are never dropped silently.
- An Access session that expires mid-session shows a sign-in banner from
  any request, without losing what is on the page.
- Restore refreshes the queue; knip now checks the admin entry.

Suggest page
- The form doesn't let the browser's URL check block links without
  https:// (the page and the server accept them).
- A failed config request shows an error with Retry instead of a
  permanently disabled form or a false "closed" message.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 17:55:02 +07:00

236 lines
8.9 KiB
Python

"""
apply_patch.py — Apply a pipeline patch to the current working tree.
Called by bash/commit_push.sh right after `git reset --hard origin/main`, so it
always works on the latest data. Idempotent: applying the same patch twice
(e.g. after a rejected push) gives the same result.
- additions: appended when the URL is not already in the catalog
- updates: applied to existing games; editable fields are never touched
- removals: game removed + logged in scripts/deleted_games.json, unless a
newer check of that game (another run) has landed since
- *_state: merged into scripts/state/*.json (null drops an entry); for
refresh_state the newer `checked` wins and the newest `full`
stamp from either side is kept
- unlog / unblock_remove: an unblocked (restored) game that was re-added
leaves the deleted log and scripts/state/unblocked.json; one whose
ingest ended otherwise leaves only the allow-list
- queue_add / queue_remove: appended to / dropped from scripts/temp_link.json,
which is re-read here so URLs queued meanwhile survive. Entries
that are not URL strings are dropped on every apply.
Safety: unless ALLOW_MASS_REMOVAL=1, a refresh patch may remove at most
max(10, 10%) of the games it checked (refresh.py's own limit, re-checked here
in case a scan never reached its guard), and any patch at most max(25, 5%) of
the catalog.
Validation (validate.py) runs last; any error exits 1 so nothing is pushed.
Usage: python scripts/apply_patch.py PATCH.json
"""
import math
import os
import sys
sys.path.insert(0, os.path.dirname(__file__))
from canonical import canonicalize
from data_store import load_all_games, save_all_games
from json_io import dedup_deleted, load_json, save_json, save_state_map
from patch import is_empty, load_patch
from refresh import mass_limit
from validate import validate
PRESERVE_FIELDS = ("safe_virus", "notes", "nsfw")
DELETED_LOG = "scripts/deleted_games.json"
TEMP_LINK = "scripts/temp_link.json"
REFRESH_STATE = "scripts/state/refresh_state.json"
INGEST_STATE = "scripts/state/ingest_state.json"
UNBLOCKED = "scripts/state/unblocked.json"
MASS_REMOVAL_FLOOR = 25
MASS_REMOVAL_SHARE = 0.05
def mass_removal_limit(patch: dict, total: int) -> int:
limit = max(MASS_REMOVAL_FLOOR, math.ceil(MASS_REMOVAL_SHARE * total))
checked = patch.get("stats", {}).get("checked")
if patch.get("kind") in ("refresh", "full") and isinstance(checked, int):
limit = min(limit, mass_limit(checked))
return limit
def _queue_key(url: str) -> str:
return canonicalize(url) or url
def clean_queue(raw) -> list[str]:
"""temp_link.json as a list of strings ({"url": ...} objects are unwrapped)."""
if not isinstance(raw, list):
print("::warning::temp_link.json is not a JSON array — resetting it")
return []
queue: list[str] = []
for item in raw:
if isinstance(item, dict) and isinstance(item.get("url"), str):
item = item["url"]
if isinstance(item, str):
queue.append(item)
else:
print(f"::warning::dropping a queue entry that is not a URL: {item!r}")
return queue
def _merge_refresh_entry(current: dict | None, entry: dict) -> dict:
"""Newest check wins; the newest `full` stamp is kept from either side.
Overlapping scans start from the same state, so an older observation (an
error snapshot with a stale strike, say) must never undo a newer one, in
whichever order their applies land.
"""
current = current or {}
base = current if current.get("checked", "") > entry.get("checked", "") else entry
full = max(current.get("full", ""), entry.get("full", ""))
return {**base, "full": full} if full else base
def apply(patch: dict) -> dict:
"""Apply `patch` to the files under the current directory; returns counts."""
games = load_all_games()
by_url = {g["url"]: g for g in games}
refresh_state = load_json(REFRESH_STATE, default={})
counts = dict.fromkeys(("added", "updated", "removed", "stale", "queued", "unqueued"), 0)
for record in patch.get("additions", []):
url = record.get("url")
if url and url not in by_url:
games.append(record)
by_url[url] = record
counts["added"] += 1
for url, fields in patch.get("updates", {}).items():
game = by_url.get(url)
if game is None:
continue
changed = False
for key, value in fields.items():
if key in PRESERVE_FIELDS or key == "url":
continue
if game.get(key) != value:
game[key] = value
changed = True
counts["updated"] += changed
# A removal decided by a scan is dropped when another run has checked the
# game since (overlapping scans read the same starting state).
removals, stale = [], set()
for removal in patch.get("removals", []):
checked = refresh_state.get(removal["url"], {}).get("checked", "")
if checked > removal["deleted_at"]:
stale.add(removal["url"])
else:
removals.append(removal)
counts["stale"] = len(stale)
removed_urls = {r["url"] for r in removals}
if removed_urls:
kept = [g for g in games if g["url"] not in removed_urls]
counts["removed"] = len(games) - len(kept)
games = kept
log = load_json(DELETED_LOG)
save_json(DELETED_LOG, dedup_deleted(log + removals))
save_all_games(games)
live_urls = {g["url"] for g in games}
for path, key, state in (
(REFRESH_STATE, "refresh_state", refresh_state),
(INGEST_STATE, "ingest_state", None),
):
delta = patch.get(key, {})
if not delta:
continue
if state is None:
state = load_json(path, default={})
for url, entry in delta.items():
if entry is None:
if url not in stale:
state.pop(url, None)
elif key == "refresh_state":
state[url] = _merge_refresh_entry(state.get(url), entry)
else:
state[url] = entry
if key == "refresh_state":
state = {u: e for u, e in state.items() if u in live_urls}
save_state_map(path, state)
unlog = {u for u in patch.get("unlog", []) if isinstance(u, str)}
finished = unlog | {u for u in patch.get("unblock_remove", []) if isinstance(u, str)}
if unlog:
log = load_json(DELETED_LOG)
kept = [e for e in log if e.get("url") not in unlog]
if len(kept) != len(log):
save_json(DELETED_LOG, kept)
if finished and os.path.exists(UNBLOCKED):
allowed = load_json(UNBLOCKED)
remaining = sorted(u for u in allowed if u not in finished)
if remaining != allowed:
save_json(UNBLOCKED, remaining)
raw_queue = load_json(TEMP_LINK)
queue = clean_queue(raw_queue)
present = {_queue_key(u) for u in queue}
appended = []
for url in patch.get("queue_add", []):
if isinstance(url, str) and _queue_key(url) not in present:
queue.append(url)
appended.append(url)
present.add(_queue_key(url))
drop = {_queue_key(u) for u in patch.get("queue_remove", []) if isinstance(u, str)}
remaining = [u for u in queue if _queue_key(u) not in drop]
counts["unqueued"] = len(queue) - len(remaining)
counts["queued"] = sum(1 for u in appended if u in remaining)
if remaining != raw_queue:
save_json(TEMP_LINK, remaining)
return counts
def main(argv: list[str] | None = None) -> int:
argv = sys.argv[1:] if argv is None else argv
if len(argv) != 1:
print(__doc__)
return 2
patch = load_patch(argv[0])
if is_empty(patch):
print("Patch is empty — nothing to apply.")
return 0
removals = patch.get("removals", [])
if removals and os.environ.get("ALLOW_MASS_REMOVAL") != "1":
total = len(load_all_games())
limit = mass_removal_limit(patch, total)
if len(removals) > limit:
print(
f"::error::The patch removes {len(removals)} of {total} games "
f"(limit {limit}) — refusing. This usually means itch.io changed "
"its page markup. Set ALLOW_MASS_REMOVAL=1 to apply it."
)
return 1
counts = apply(patch)
print(
f"apply_patch ({patch.get('kind')}): {counts['added']} added, "
f"{counts['updated']} updated, {counts['removed']} removed "
f"({counts['stale']} stale removals skipped), {counts['queued']} queued, "
f"{counts['unqueued']} dropped from the queue."
)
errors, warnings = validate()
for w in warnings:
print(f"::warning::{w}")
for e in errors:
print(f"::error::{e}")
if errors:
print("Validation failed — refusing to commit.")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())