Files
poli0981andClaude Opus 5.5 d8991dada4 fix(pipeline): act on SonarCloud's findings in scripts/
- Linear parsers (python:S8786): the store page's "unsupported" language
  rows and its tags, and the discovery sweep's search rows, no longer use
  regexes that backtrack quadratically on long input. New tests, written
  against the old regexes first, pin what they read, and a differential
  fuzz (200,000 random inputs) found no case where old and new disagree.
- Raw invisible and bidi characters in normalize.py and two test files are
  written as escapes (text:S6389); every string value is unchanged.
- scrape_store_fields() states when there is no usable page (scraped = None)
  instead of deriving it into a separate flag (pythonbugs:S2259).
- cmd_list() in delete_game.py and edit_game.py returns nothing; the
  dispatch returns 0 (python:S3516). remove_games.main() keeps its int:
  delete_game.py and its tests use the exit code.
- The Steam client mounts its retrying adapter for https only
  (python:S5332); nothing is fetched over http.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:31:48 +07:00

194 lines
6.8 KiB
Python

#!/usr/bin/env python3
"""
Remove games from the catalogue - the command-line twin of /admin/remove.
Removing a game is not a one-time deletion. It adds a line to
data/blocklist.jsonl, a standing instruction that save_main() re-imposes on
every write (core/blocklist.py): the game leaves data/, is logged once in
scripts/removed_games.jsonl, and nothing - a queued link, a refetch, the
discovery sweep - can bring it back. Restoring is done in /admin (Remove ->
Removed -> Restore), because it also has to clear the review queue's record.
python scripts/delete_game.py 730 --reason not_a_game
python scripts/delete_game.py https://store.steampowered.com/app/730/ 440 \\
--reason duplicate --note "same game as 2357570" --apply
python scripts/delete_game.py --list
python scripts/delete_game.py --check # validate the file (runs in CI)
Writes the same bytes /admin writes (held equal by
scripts/tests/fixtures/blocklist_golden.jsonl). The reason and the note are
PUBLIC: the blocklist is a file in a public repository. Without --apply,
data/ changes on the next pipeline run (pushing the blocklist starts one).
Run from the repository root (DATA_DIR is relative).
"""
import argparse
import json
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.blocklist import (
REASONS, entry_problem, load_blocklist, make_entry, read_lines, serialize_entry,
)
from core.constants import BLOCKLIST_JSONL
from core.data_store import extract_appid, load_main, now_iso
# Same cap as /admin (web/shared/removal.ts MAX_REMOVE).
MAX_REMOVE = 100
MAX_NOTE = 300
def safe_stdout():
"""Game names are routinely non-Latin; a cp1252 console would mangle them."""
for stream in (sys.stdout, sys.stderr):
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, OSError):
pass
def _appid_of(raw: str):
raw = raw.strip()
return raw if raw.isdigit() else extract_appid(raw)
def _append(lines: list[str], path: str = BLOCKLIST_JSONL) -> None:
"""Append whole lines, never re-serialising the ones already there."""
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
needs_newline = False
if os.path.isfile(path) and os.path.getsize(path) > 0:
with open(path, "rb") as f:
f.seek(-1, os.SEEK_END)
needs_newline = f.read(1) != b"\n"
with open(path, "a", encoding="utf-8", newline="\n") as f:
if needs_newline:
f.write("\n")
for line in lines:
f.write(line + "\n")
def cmd_check(path: str = BLOCKLIST_JSONL) -> int:
problems = 0
seen: dict[str, int] = {}
for lineno, line in enumerate(read_lines(path), 1):
try:
entry = json.loads(line)
except json.JSONDecodeError as e:
print(f"{path}:{lineno}: not JSON: {e}")
problems += 1
continue
problem = entry_problem(entry)
if problem:
print(f"{path}:{lineno}: {problem}")
problems += 1
continue
if entry["appid"] in seen:
# Harmless (the last line wins) but never written by /admin.
print(f"{path}:{lineno}: note: {entry['appid']} also on line {seen[entry['appid']]}")
seen[entry["appid"]] = lineno
print(f"{path}: {len(seen)} blocked game(s), {problems} problem(s).")
return 1 if problems else 0
def cmd_list() -> None:
entries = load_blocklist()
if not entries:
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
return
for e in entries.values():
note = f" - {e['note']}" if e.get("note") else ""
print(f"{e['appid']:>10} {e['blocked_at']} {e['reason']:<13} {e.get('name', '')}{note}")
print(f"{len(entries)} blocked game(s).")
def cmd_add(targets: list[str], reason: str, note: str, apply: bool) -> int:
if reason not in REASONS:
print(f"--reason must be one of: {', '.join(REASONS)}")
return 2
note = (note or "").strip()
if len(note) > MAX_NOTE:
print(f"--note is longer than {MAX_NOTE} characters.")
return 2
if reason == "other" and not note:
print("--reason other needs a --note saying why.")
return 2
appids: list[str] = []
for raw in targets:
appid = _appid_of(raw)
if not appid:
print(f"Not a Steam appid or store link: {raw}")
return 2
if appid not in appids:
appids.append(appid)
if len(appids) > MAX_REMOVE:
print(f"At most {MAX_REMOVE} games at once, as in /admin.")
return 2
blocked = load_blocklist()
catalogue = {}
for game in load_main():
aid = extract_appid(game.get("link", ""))
if aid:
catalogue[aid] = game
stamp = now_iso()
lines = []
for appid in appids:
if appid in blocked:
print(f" skip {appid}: already blocked since {blocked[appid]['blocked_at']}")
continue
game = catalogue.get(appid)
if game is None:
# Every blocklist line must describe a game the catalogue held, so
# that each removed-games log line describes a real removal.
print(f" skip {appid}: not in data/ (nothing to remove)")
continue
entry = make_entry(appid, str(game.get("name", "")), reason, note, stamp,
link=game.get("link"))
lines.append(serialize_entry(entry))
print(f" block {appid}: {entry['name']}")
if not lines:
print("Nothing to add.")
return 0
_append(lines)
print(f"Added {len(lines)} line(s) to {BLOCKLIST_JSONL}.")
if apply:
import remove_games
return remove_games.main([])
print("data/ changes on the next pipeline run (or run again with --apply).")
return 0
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("targets", nargs="*", help="Steam appids or store links")
ap.add_argument("--reason", choices=list(REASONS), help="why the games are removed (public)")
ap.add_argument("--note", default="", help="optional detail, at most 300 characters (public)")
ap.add_argument("--apply", action="store_true", help="take them out of data/ now")
ap.add_argument("--list", action="store_true", help="print the blocklist")
ap.add_argument("--check", action="store_true", help="validate data/blocklist.jsonl; exit 1 on a problem")
args = ap.parse_args(argv)
safe_stdout()
if args.check:
return cmd_check()
if args.list:
cmd_list()
return 0
if not args.targets:
ap.print_help()
return 2
if not args.reason:
print("--reason is required when removing games.")
return 2
return cmd_add(args.targets, args.reason, args.note, args.apply)
if __name__ == "__main__":
sys.exit(main())