feat(pipeline): data/blocklist.jsonl - save_main drops and logs removed games

A game the maintainer removes is now a standing instruction in Git, like an
override, instead of a one-time deletion that the browser extension or the
next ingest could undo.

- core/blocklist.py reads data/blocklist.jsonl (one line per game, written by
  /admin and by delete_game.py). save_main() applies it FIRST on every write,
  for all 14 callers, without touching the caller's list, and whatever
  apply_human_overrides says.
- A dropped game is logged once in scripts/removed_games.jsonl as
  status_code "admin_removed", built from the blocklist line alone with
  removed_at = blocked_at, so writing it again changes no byte. A later purge
  line for the same game keeps winning, as dedup_removed() decides.
- More than 500 drops in one save raises BlocklistError before anything is
  written: a damaged file fails the job instead of emptying the catalogue.
- remove_games.py applies the blocklist now and heals a missing log line.
  bash/ingest.sh runs it before ingest_new.py, and ingest-new.yml also
  triggers on blocklist pushes and checks out the tip of main. One workflow
  on purpose: data-write keeps one pending run, so a second workflow and the
  ingest would keep evicting each other.
- ingest_new.py skips blocked links before any Steam request, and a link
  that is not text ({"link": 730}) no longer crashes the run (and every
  three-hourly retry after it).
- Discovery counts blocked appids as known.
- delete_game.py is a non-interactive CLI writing the same bytes as /admin
  (golden fixture), with --list and --check; check-overrides.yml runs --check.
- mark-dead-games.yml commits through bash/commit_push.sh, like every other
  data job: it rebases and retries, and commits the log with the shards.
- dependabot.yml ignores Svelte-era framework majors and @types/node majors
  instead of the React-era names.

With an empty blocklist every path is a no-op.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-13 12:33:53 +07:00
1 parent e70e1f770e
commit 986077d3f1
18 files changed
+1055 -61

No files matched your search

+12 -11
View File
@@ -1,10 +1,8 @@
# NOTE: committing this file REPLACES the Dependabot configuration currently
# set through repo settings. Existing open Dependabot PRs (e.g. #111) are
# superseded and will be recreated against these groups on the next run.
# set through repo settings.
#
# web/worker/ is intentionally absent: it does not exist yet. Add a fifth
# npm entry for it as part of the Cloudflare Worker work — that package will
# hold the repo-write credential, so it must be covered.
# web/worker/ and web/admin/ need no entry of their own: they are built from
# web/package.json, which the /web entry below covers.
version: 2
updates:
@@ -36,19 +34,22 @@ updates:
patterns: ['@tauri-apps/*']
ignore:
# Framework majors are migrations with their own staged PRs, not
# something to take as an unattended bump.
- dependency-name: react
# something to take as an unattended bump. (SvelteKit 3 needs the
# cookie override in package.json re-keyed or dropped; see CLAUDE.md.)
- dependency-name: '@sveltejs/kit'
update-types: [version-update:semver-major]
- dependency-name: react-dom
- dependency-name: '@sveltejs/adapter-static'
update-types: [version-update:semver-major]
- dependency-name: react-router-dom
- dependency-name: svelte
update-types: [version-update:semver-major]
- dependency-name: tailwindcss
update-types: [version-update:semver-major]
- dependency-name: typescript
update-types: [version-update:semver-major]
# lucide-react ships breaking icon renames on 0.x minors.
- dependency-name: lucide-react
# The type definitions follow the Node the build runs on (24, in
# web-ci.yml and the release workflows), not the newest Node.
- dependency-name: '@types/node'
update-types: [version-update:semver-major]
# ── Desktop / Android shell ───────────────────────────────────────────────
- package-ecosystem: cargo
+22
View File
@@ -18,6 +18,13 @@ on:
# no enforcement in either direction.
- 'web/worker/routes/edit.ts'
- 'web/worker/lib/override-doc.ts'
# The blocklist is the same kind of standing human instruction, with the
# same two writers (/admin and a CLI) that must produce identical bytes.
- 'data/blocklist.jsonl'
- 'scripts/core/blocklist.py'
- 'scripts/delete_game.py'
- 'web/shared/removal.ts'
- 'scripts/tests/fixtures/blocklist_golden.jsonl'
- '.github/workflows/check-overrides.yml'
push:
branches: [main]
@@ -30,6 +37,13 @@ on:
# no enforcement in either direction.
- 'web/worker/routes/edit.ts'
- 'web/worker/lib/override-doc.ts'
# The blocklist is the same kind of standing human instruction, with the
# same two writers (/admin and a CLI) that must produce identical bytes.
- 'data/blocklist.jsonl'
- 'scripts/core/blocklist.py'
- 'scripts/delete_game.py'
- 'web/shared/removal.ts'
- 'scripts/tests/fixtures/blocklist_golden.jsonl'
- '.github/workflows/check-overrides.yml'
workflow_dispatch:
@@ -65,6 +79,14 @@ jobs:
- name: Install web deps
working-directory: web
run: npm ci
- name: Validate the blocklist
# A line load_blocklist() cannot honour blocks nothing - silently.
run: python scripts/delete_game.py --check
- name: Assert CLI/Worker override parity
working-directory: web
run: npx vitest run worker/lib/override-doc.test.ts
- name: Assert CLI/Worker blocklist parity
# web-ci only runs for web/**; a Python-only change to the line format
# must fail somewhere too.
working-directory: web
run: npx vitest run shared/removal.test.ts
+11 -1
View File
@@ -3,7 +3,12 @@ on :
# branches filter is REQUIRED: without it every v*/desktop-v* tag push
# matches the paths filter (tag pushes compare against an empty base),
# fires the ingest on a detached HEAD, and dies with exit 128 on push.
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl' ] }
#
# data/blocklist.jsonl is here too: this job also applies removals
# (bash/ingest.sh runs scripts/remove_games.py first). A separate removal
# workflow would share `data-write` with this one, and each would evict the
# other's pending run.
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl', 'data/blocklist.jsonl' ] }
# Orphan recovery, not a poll. `data-write` keeps at most ONE run pending and
# cancels the older one, so a push here that queues behind a ~115-minute data
# job is silently evicted the moment any OTHER data-write workflow queues --
@@ -23,6 +28,11 @@ jobs :
steps :
-
uses : actions/checkout@v6
# The tip of main when the job STARTS, not the commit that triggered
# it. A run that waited behind a long data job would otherwise work on
# shards that job has since rewritten, and its push would fail to
# rebase.
with : { ref : main }
-
uses : actions/setup-python@v6
with : { python-version : '3.12' }
+10 -11
View File
@@ -28,6 +28,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
# The tip of main when the job starts, not when it was queued: a run
# that waited behind a long data job would otherwise rewrite stale shards.
with:
ref: main
- uses: actions/setup-python@v5
with:
@@ -49,14 +53,9 @@ jobs:
- name: Commit changes
if: ${{ inputs.dry_run != true }}
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add data/ scripts/dead_games.log
if git diff --cached --quiet; then
echo "No changes."
else
git commit -m "chore(data): mark dead games"
git push
fi
# Through commit_push.sh like every other data job: it rebases and
# retries when main moved during the run (a bare `git push` threw the
# run away), and it commits the whole tree, so scripts/removed_games.jsonl
# goes with the shards when save_main() applied a removal
# (scripts/core/blocklist.py).
run: bash ./bash/commit_push.sh "chore(data): mark dead games"
+17 -1
View File
@@ -1,5 +1,21 @@
#!/bin/bash
# The ingest job, which is also the removal job.
#
# remove_games.py applies data/blocklist.jsonl (games the maintainer removed in
# /admin or with delete_game.py) before ingest_new.py adds the queued links.
# One workflow does both on purpose: `data-write` keeps ONE pending run and a
# newer one evicts it, so a separate removal workflow and the ingest would keep
# cancelling each other. Whichever run survives does both.
set -euo pipefail
pip install --quiet -r requirements.txt
python scripts/remove_games.py
python scripts/ingest_new.py
bash "$(dirname "$0")/commit_push.sh" "Ingest games [$(date +'%Y-%m-%d %H:%M')]"
# Name the commit after what changed: the queue file is emptied only when
# ingest_new.py processed something.
if git diff --quiet -- scripts/temp_info.jsonl; then
msg="Apply removals [$(date +'%Y-%m-%d %H:%M')]"
else
msg="Ingest games [$(date +'%Y-%m-%d %H:%M')]"
fi
bash "$(dirname "$0")/commit_push.sh" "$msg"
View File
Whitespace-only changes.
+190
View File
@@ -0,0 +1,190 @@
"""
Games the maintainer took out of the catalogue, as a standing instruction.
data/blocklist.jsonl holds one line per blocked game. It is written by the
Worker (/admin/remove appends lines, Restore deletes one) and by
scripts/delete_game.py, and only READ here. Like data/overrides/, it is human
intent that every write re-imposes: save_main() drops a blocked record before
anything else happens to it, so no writer - the ingest of a link the browser
extension queued again, a full refetch, a purge - can put the game back. The
only way back is to delete the line (Restore in /admin), which also queues the
link for ingest_new.py in the same commit.
A drop is logged once in scripts/removed_games.jsonl, built from the blocklist
line alone. Its removed_at IS the line's blocked_at, so writing it again is a
byte-for-byte no-op and the log cannot churn (data_store.log_admin_removals).
Line format, keys in this order, compact separators (the Worker writes the
same bytes - web/shared/removal.ts, held equal by the golden fixture in
scripts/tests/fixtures/blocklist_golden.jsonl):
{"appid":"730","link":"https://store.steampowered.com/app/730/",
"name":"Counter-Strike 2","reason":"not_a_game","note":"","blocked_at":
"2026-10-13T12:00:00Z","by":"admin"}
Standard library only: scripts/core/* is stdlib-only, and the discovery
workflows import this without installing anything.
"""
import json
import os
import re
from typing import Iterable, Optional
from .constants import BLOCKLIST_JSONL
# The reason codes /admin offers, with the words the public log shows.
# web/shared/removal.ts REMOVAL_REASONS mirrors this table (a vitest reads it
# from this file and compares).
REASONS: dict[str, str] = {
"not_free": "No longer free to play",
"not_a_game": "Not a game (DLC, demo, tool or soundtrack)",
"duplicate": "Duplicate entry",
"broken": "Unplayable or shut down",
"inappropriate": "Does not fit the catalogue",
"other": "Other",
}
ENTRY_KEYS = ("appid", "link", "name", "reason", "note", "blocked_at", "by")
# Written into every line instead of a person: the file is public forever.
ATTRIBUTION = "admin"
STATUS_CODE = "admin_removed"
_APPID = re.compile(r"^\d{1,10}$")
# now_iso()'s shape exactly. dedup_removed() compares removed_at as a string,
# so a millisecond suffix would sort differently from the pipeline's own rows.
_ISO_SECONDS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
class BlocklistError(RuntimeError):
"""A blocklist that would remove an implausible number of games at once."""
def canonical_link(appid: str) -> str:
return f"https://store.steampowered.com/app/{appid}/"
def entry_problem(entry: object) -> Optional[str]:
"""Why a parsed line cannot be honoured, or None when it can."""
if not isinstance(entry, dict):
return "not an object"
appid = entry.get("appid")
if not isinstance(appid, str) or not _APPID.match(appid):
return "appid must be 1-10 digits, as a string"
blocked_at = entry.get("blocked_at")
if not isinstance(blocked_at, str) or not _ISO_SECONDS.match(blocked_at):
# Without a fixed timestamp the log line would be stamped anew on every
# write and rewritten forever.
return "blocked_at must look like 2026-10-13T12:00:00Z"
if entry.get("reason") not in REASONS:
return f"reason must be one of {', '.join(REASONS)}"
for key in ("link", "name", "note"):
if not isinstance(entry.get(key, ""), str):
return f"{key} must be a string"
return None
def read_lines(path: str = BLOCKLIST_JSONL) -> list[str]:
"""The file's non-empty lines, verbatim. [] when it does not exist."""
if not os.path.isfile(path):
return []
with open(path, "r", encoding="utf-8") as f:
return [line.rstrip("\n") for line in f if line.strip()]
def load_blocklist(path: str = BLOCKLIST_JSONL, quiet: bool = False) -> dict[str, dict]:
"""appid -> entry for every line that can be honoured.
Tolerant on purpose, like apply_overrides(): a bad line is reported and
skipped, never raised. A skipped line blocks nothing, so the worst a
damaged line can do is fail to remove one game - `delete_game.py --check`
(run in CI) is what makes it visible. The LAST valid line for an appid wins.
"""
out: dict[str, dict] = {}
for lineno, line in enumerate(read_lines(path), 1):
try:
entry = json.loads(line)
except json.JSONDecodeError as e:
if not quiet:
print(f" [blocklist] {path}:{lineno} bad JSON, ignored: {e}")
continue
problem = entry_problem(entry)
if problem:
if not quiet:
print(f" [blocklist] {path}:{lineno} ignored: {problem}")
continue
out[entry["appid"]] = entry
return out
def blocked_appids(path: str = BLOCKLIST_JSONL) -> set[str]:
return set(load_blocklist(path, quiet=True))
def make_entry(appid: str, name: str, reason: str, note: str, blocked_at: str,
link: Optional[str] = None) -> dict:
entry = {
"appid": appid,
"link": link or canonical_link(appid),
"name": clean_text(name, 300),
"reason": reason,
"note": clean_text(note, 300),
"blocked_at": blocked_at,
"by": ATTRIBUTION,
}
problem = entry_problem(entry)
if problem:
raise ValueError(problem)
return entry
def clean_text(value: str, limit: int) -> str:
"""A one-line note: control characters and line breaks (U+2028/U+2029
included) become spaces, runs of whitespace collapse, at most `limit`
characters. web/shared/removal.ts cleanText() is the same rule."""
out = []
for ch in value or "":
out.append(" " if (ord(ch) < 0x20 or 0x7F <= ord(ch) < 0xA0 or ch in "\u2028\u2029") else ch)
return " ".join("".join(out).split())[:limit]
def serialize_entry(entry: dict) -> str:
"""One line, without its newline: keys in ENTRY_KEYS order, compact
separators, non-ASCII kept as is - except U+2028/U+2029, escaped, because
str.splitlines() and some editors treat them as line breaks and would cut a
line in two. web/shared/removal.ts serializeEntry() writes the same bytes."""
ordered = {key: entry.get(key, "") for key in ENTRY_KEYS}
text = json.dumps(ordered, ensure_ascii=False, separators=(",", ":"))
return text.replace("\u2028", "\\u2028").replace("\u2029", "\\u2029")
def public_reason(entry: dict) -> str:
"""The reason the public removed-games log shows. The free-text note is
deliberately left out of it (it stays in the blocklist and the commit)."""
return f"Removed by the maintainer: {REASONS.get(entry.get('reason', ''), 'Other')}"
def removed_row(entry: dict) -> dict:
"""The scripts/removed_games.jsonl line for a blocked game. Built from the
blocklist line alone, so it is identical however often it is written."""
return {
"link": entry.get("link") or canonical_link(entry["appid"]),
"name": entry.get("name", ""),
"appid": entry["appid"],
"reason": public_reason(entry),
"status_code": STATUS_CODE,
"removed_at": entry["blocked_at"],
}
def split_blocked(records: Iterable[dict], blocked: dict[str, dict],
appid_of) -> tuple[list[dict], list[dict]]:
"""(kept, dropped). A record whose appid cannot be read is kept: this
module only ever removes what it can name."""
kept: list[dict] = []
dropped: list[dict] = []
for record in records:
appid = appid_of(record.get("link", "")) if isinstance(record, dict) else None
(dropped if appid and appid in blocked else kept).append(record)
return kept, dropped
+7
View File
@@ -11,6 +11,13 @@ DEAD_LINKS_LOG = "scripts/dead_links.log"
REMOVED_JSONL = "scripts/removed_games.jsonl"
GAMES_DIR = "games"
OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game
# Games the maintainer took out of the catalogue (core/blocklist.py). Never
# name it data_*: save_main() treats data/data_*.jsonl as shards and deletes
# the ones it did not write.
BLOCKLIST_JSONL = "data/blocklist.jsonl"
# More blocked games than this in ONE save is treated as a damaged file, not a
# decision: save_main() refuses before writing anything (5 x the /admin cap).
BLOCKLIST_MAX_DROP = 500
# ──────────── Sharding ────────────
MAX_RECORDS_PER_FILE = 800
+77 -3
View File
@@ -16,9 +16,10 @@ from datetime import datetime, timezone
from typing import Optional
from .constants import (
DATA_DIR, LEGACY_JSONL, TEMP_JSONL,
DATA_DIR, LEGACY_JSONL, TEMP_JSONL, REMOVED_JSONL,
MAX_RECORDS_PER_FILE, SHARD_PREFIX,
MANUAL_FIELDS, ARRAY_FIELDS, DESCRIPTION_MAX,
BLOCKLIST_MAX_DROP,
)
from .normalize import normalize_records
@@ -128,6 +129,68 @@ def dedup_removed(records: list[dict]) -> list[dict]:
return list(latest.values())
def log_admin_removals(rows: list[dict]) -> int:
"""Make sure the removed-games log carries these blocklist rows.
A row is added only when the log's latest line for that appid is not
already exactly it. Because each row is built from its blocklist line
(removed_at = blocked_at), writing it a second time is a no-op, and a later
line for the same game - a purge that logged it first - keeps winning, as
dedup_removed() already decides. The file is left untouched unless
something actually changes, so this can run on every save.
"""
if not rows:
return 0
existing = load_jsonl(REMOVED_JSONL)
latest = {}
for r in dedup_removed(existing):
latest[r.get("appid") or extract_appid(r.get("link", ""))] = r
missing = [row for row in rows if latest.get(row["appid"]) != row]
if not missing:
return 0
merged = dedup_removed(existing + missing)
if merged == existing:
return 0
os.makedirs(os.path.dirname(REMOVED_JSONL) or ".", exist_ok=True)
save_jsonl(REMOVED_JSONL, merged)
return len(missing)
def apply_blocklist(records: list[dict], max_drop: int = BLOCKLIST_MAX_DROP) -> list[dict]:
"""The records to WRITE: everything except games in data/blocklist.jsonl.
Returns a new list and never touches the caller's: several scripts hold
positions into theirs (ingest_new.py's appid -> index map) and some save
part-way through a loop. A drop is logged in scripts/removed_games.jsonl,
once. More than `max_drop` drops in one save raises BEFORE anything is
written - a damaged file must fail the job loudly, not empty the catalogue.
"""
from .blocklist import BlocklistError, load_blocklist, removed_row, split_blocked
blocked = load_blocklist()
if not blocked:
return records
kept, dropped = split_blocked(records, blocked, extract_appid)
if not dropped:
return records
if len(dropped) > max_drop:
raise BlocklistError(
f"data/blocklist.jsonl would remove {len(dropped)} games in one save "
f"(limit {max_drop}). Refusing to write anything. If that is intended, "
f"run scripts/remove_games.py --max-drop {len(dropped)}."
)
appids = []
for record in dropped:
appid = extract_appid(record.get("link", ""))
if appid not in appids:
appids.append(appid)
logged = log_admin_removals([removed_row(blocked[a]) for a in appids])
print(f" blocklist: left out {len(appids)} blocked game(s)"
+ (f", logged {logged} in {REMOVED_JSONL}" if logged else "")
+ f": {', '.join(appids[:10])}{' …' if len(appids) > 10 else ''}")
return kept
# ──────────── Shard helpers ────────────
def _shard_paths() -> list[str]:
@@ -200,16 +263,22 @@ def truncate_descriptions(records: list[dict]) -> int:
return n
def save_main(records: list[dict], apply_human_overrides: bool = True):
def save_main(records: list[dict], apply_human_overrides: bool = True,
max_blocked_drop: int = BLOCKLIST_MAX_DROP):
"""Save records to sharded JSONL files (MAX_RECORDS_PER_FILE per shard).
Human overrides are re-imposed here, immediately before the write. This is
the single choke point for the dataset - all 11 data-writing scripts call
the single choke point for the dataset - all 14 data-writing scripts call
save_main() and nothing else writes data/data_*.jsonl - so applying them
here is what makes a correction survive refetch_all.py and
normalize_genres.py --apply without either script knowing this layer
exists. See core/overrides.py.
The blocklist (core/blocklist.py) is applied FIRST, the same way: a game
the maintainer removed is left out of every write, whichever script made
it, and that is never switched off - not even with
apply_human_overrides=False. The caller's list is not modified.
apply_human_overrides=False exists for tooling that must see the
un-overridden values (an audit report). Never pass it from the pipeline.
@@ -218,6 +287,11 @@ def save_main(records: list[dict], apply_human_overrides: bool = True):
notes - so every writer gets them, the browser extension's ingest
included. Like the truncation, they never touch last_updated.
"""
# Before everything else, so no work is spent on records that are not
# written, and a blocked game's override reports as an orphan (its file
# stays, and applies again if the game is restored).
records = apply_blocklist(records, max_drop=max_blocked_drop)
# Before the truncation: a decoded "&quot;" is one character, not six.
normalized = normalize_records(records)
if normalized:
+189 -26
View File
@@ -1,30 +1,193 @@
#!/usr/bin/env python3
"""Delete game (interactive)."""
import sys, os
"""
Remove games from the catalogue - the command-line twin of /admin/remove.
Removing a game is not a one-time deletion. It adds a line to
data/blocklist.jsonl, a standing instruction that save_main() re-imposes on
every write (core/blocklist.py): the game leaves data/, is logged once in
scripts/removed_games.jsonl, and nothing - the browser extension, a refetch,
the discovery sweep - can bring it back. Restoring is done in /admin (Remove ->
Removed -> Restore), because it also has to clear the review queue's record.
python scripts/delete_game.py 730 --reason not_a_game
python scripts/delete_game.py https://store.steampowered.com/app/730/ 440 \
--reason duplicate --note "same game as 2357570" --apply
python scripts/delete_game.py --list
python scripts/delete_game.py --check # validate the file (runs in CI)
Writes the same bytes /admin writes (held equal by
scripts/tests/fixtures/blocklist_golden.jsonl). The reason and the note are
PUBLIC: the blocklist is a file in a public repository. Without --apply,
data/ changes on the next pipeline run (pushing the blocklist starts one).
Run from the repository root (DATA_DIR is relative).
"""
import argparse
import json
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.data_store import load_main, save_main, extract_appid, normalize_link
def main():
games = load_main()
if not games: print("Empty."); return
q = input(f"Search ({len(games)} games): ").strip()
if not q: return
norm = normalize_link(q)
tid = extract_appid(norm) if norm else (q if q.isdigit() else None)
matches = [(i,g) for i,g in enumerate(games)
if (tid and extract_appid(g.get("link","")) == tid)
or q.lower() in g.get("name","").lower()]
if not matches: print("Not found."); return
for idx,(i,g) in enumerate(matches,1):
print(f" {idx}. {g.get('name','?')} ({g.get('link','?')})")
c = input(f"Delete (1-{len(matches)}/all/q): ").strip()
if c.lower()=="q": return
d = ({i for i,_ in matches} if c.lower()=="all"
else {matches[int(c)-1][0]} if c.isdigit() and 1<=int(c)<=len(matches) else set())
if not d: print("Invalid."); return
if input(f"Confirm {len(d)}? (y/n): ").strip().lower()!="y": return
games = [g for i,g in enumerate(games) if i not in d]
save_main(games)
print(f"✓ Deleted {len(d)}. {len(games)} left.")
from core.blocklist import (
REASONS, entry_problem, load_blocklist, make_entry, read_lines, serialize_entry,
)
from core.constants import BLOCKLIST_JSONL
from core.data_store import extract_appid, load_main, now_iso
if __name__ == "__main__": main()
# Same cap as /admin (web/shared/removal.ts MAX_REMOVE).
MAX_REMOVE = 100
MAX_NOTE = 300
def safe_stdout():
"""Game names are routinely non-Latin; a cp1252 console would mangle them."""
for stream in (sys.stdout, sys.stderr):
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, OSError):
pass
def _appid_of(raw: str):
raw = raw.strip()
return raw if raw.isdigit() else extract_appid(raw)
def _append(lines: list[str], path: str = BLOCKLIST_JSONL) -> None:
"""Append whole lines, never re-serialising the ones already there."""
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
needs_newline = False
if os.path.isfile(path) and os.path.getsize(path) > 0:
with open(path, "rb") as f:
f.seek(-1, os.SEEK_END)
needs_newline = f.read(1) != b"\n"
with open(path, "a", encoding="utf-8", newline="\n") as f:
if needs_newline:
f.write("\n")
for line in lines:
f.write(line + "\n")
def cmd_check(path: str = BLOCKLIST_JSONL) -> int:
problems = 0
seen: dict[str, int] = {}
for lineno, line in enumerate(read_lines(path), 1):
try:
entry = json.loads(line)
except json.JSONDecodeError as e:
print(f"{path}:{lineno}: not JSON: {e}")
problems += 1
continue
problem = entry_problem(entry)
if problem:
print(f"{path}:{lineno}: {problem}")
problems += 1
continue
if entry["appid"] in seen:
# Harmless (the last line wins) but never written by /admin.
print(f"{path}:{lineno}: note: {entry['appid']} also on line {seen[entry['appid']]}")
seen[entry["appid"]] = lineno
print(f"{path}: {len(seen)} blocked game(s), {problems} problem(s).")
return 1 if problems else 0
def cmd_list() -> int:
entries = load_blocklist()
if not entries:
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
return 0
for e in entries.values():
note = f" - {e['note']}" if e.get("note") else ""
print(f"{e['appid']:>10} {e['blocked_at']} {e['reason']:<13} {e.get('name', '')}{note}")
print(f"{len(entries)} blocked game(s).")
return 0
def cmd_add(targets: list[str], reason: str, note: str, apply: bool) -> int:
if reason not in REASONS:
print(f"--reason must be one of: {', '.join(REASONS)}")
return 2
note = (note or "").strip()
if len(note) > MAX_NOTE:
print(f"--note is longer than {MAX_NOTE} characters.")
return 2
if reason == "other" and not note:
print("--reason other needs a --note saying why.")
return 2
appids: list[str] = []
for raw in targets:
appid = _appid_of(raw)
if not appid:
print(f"Not a Steam appid or store link: {raw}")
return 2
if appid not in appids:
appids.append(appid)
if len(appids) > MAX_REMOVE:
print(f"At most {MAX_REMOVE} games at once, as in /admin.")
return 2
blocked = load_blocklist()
catalogue = {}
for game in load_main():
aid = extract_appid(game.get("link", ""))
if aid:
catalogue[aid] = game
stamp = now_iso()
lines = []
for appid in appids:
if appid in blocked:
print(f" skip {appid}: already blocked since {blocked[appid]['blocked_at']}")
continue
game = catalogue.get(appid)
if game is None:
# Every blocklist line must describe a game the catalogue held, so
# that each removed-games log line describes a real removal.
print(f" skip {appid}: not in data/ (nothing to remove)")
continue
entry = make_entry(appid, str(game.get("name", "")), reason, note, stamp,
link=game.get("link"))
lines.append(serialize_entry(entry))
print(f" block {appid}: {entry['name']}")
if not lines:
print("Nothing to add.")
return 0
_append(lines)
print(f"Added {len(lines)} line(s) to {BLOCKLIST_JSONL}.")
if apply:
import remove_games
return remove_games.main([])
print("data/ changes on the next pipeline run (or run again with --apply).")
return 0
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("targets", nargs="*", help="Steam appids or store links")
ap.add_argument("--reason", choices=list(REASONS), help="why the games are removed (public)")
ap.add_argument("--note", default="", help="optional detail, at most 300 characters (public)")
ap.add_argument("--apply", action="store_true", help="take them out of data/ now")
ap.add_argument("--list", action="store_true", help="print the blocklist")
ap.add_argument("--check", action="store_true", help="validate data/blocklist.jsonl; exit 1 on a problem")
args = ap.parse_args(argv)
safe_stdout()
if args.check:
return cmd_check()
if args.list:
return cmd_list()
if not args.targets:
ap.print_help()
return 2
if not args.reason:
print("--reason is required when removing games.")
return 2
return cmd_add(args.targets, args.reason, args.note, args.apply)
if __name__ == "__main__":
sys.exit(main())
+9 -2
View File
@@ -26,6 +26,7 @@ import urllib.request
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.blocklist import blocked_appids
from core.data_store import load_main, extract_appid
from core.steam_client import get_client
@@ -134,8 +135,14 @@ def parse_search_rows(results_html):
def catalogue_appids():
"""Appids already in data/."""
out = set()
"""Appids already in data/, plus the ones the maintainer removed.
A game in data/blocklist.jsonl was taken out on purpose; proposing it again
would only put it back in the review queue. The Worker refuses it too (its
removal recorded a rejected decision), so this is the second fence, and the
one that holds even when D1 and Git disagree.
"""
out = set(blocked_appids())
for rec in load_main():
aid = extract_appid(rec.get("link", ""))
if aid:
+27 -6
View File
@@ -8,6 +8,7 @@ so fetch_full() can skip its own fetch_app_details() call → eliminates double-
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.blocklist import blocked_appids
from core.constants import REMOVED_JSONL, MANUAL_FIELDS
from core.data_store import (
load_main, save_main, load_temp, clear_temp,
@@ -20,13 +21,25 @@ from core.steam_client import get_client
from core.health_checker import check_game_health, NETWORK_ERROR
def _link_text(value) -> str:
"""A queued link as text. `{"link": 730}` used to reach normalize_link() as
an int and crash the run before it saved or cleared the queue - so it
crashed again on every three-hourly retry. A number is an appid; anything
else that is not text becomes "" and is reported as an invalid line."""
if isinstance(value, bool):
return ""
if isinstance(value, int):
return str(value)
return value.strip() if isinstance(value, str) else ""
def parse_entries(records):
entries = []
for rec in records:
if isinstance(rec, str):
entries.append({"link": rec.strip()})
elif isinstance(rec, dict) and rec.get("link"):
entries.append(rec)
if isinstance(rec, (str, int)) and not isinstance(rec, bool):
entries.append({"link": _link_text(rec)})
elif isinstance(rec, dict) and rec.get("link") not in (None, ""):
entries.append({**rec, "link": _link_text(rec["link"])})
return entries
@@ -39,9 +52,12 @@ def main():
games = load_main()
index = build_index(games)
entries = parse_entries(temp)
# data/blocklist.jsonl: games the maintainer removed. save_main() would
# drop them anyway; skipping here also saves the Steam requests.
blocked = blocked_appids()
print(f"Found {len(entries)} entries\n")
added = skipped_dup = skipped_bad = skipped_net = 0
added = skipped_dup = skipped_bad = skipped_net = skipped_blocked = 0
rejected = []
client = get_client()
@@ -55,6 +71,10 @@ def main():
continue
appid = extract_appid(link)
if appid in blocked:
print(f" ⛔ Blocked ({appid}): removed by the maintainer, see data/blocklist.jsonl")
skipped_blocked += 1
continue
if appid in index:
print(f" ⏭ Dup ({appid})")
skipped_dup += 1
@@ -108,7 +128,8 @@ def main():
clear_temp()
print(f"\n{'─'*50}")
print(f"Added: {added} | Dup: {skipped_dup} | Bad: {skipped_bad} | Net: {skipped_net}")
print(f"Added: {added} | Dup: {skipped_dup} | Bad: {skipped_bad} | Net: {skipped_net}"
f" | Blocked: {skipped_blocked}")
print(f"Total: {len(games)}")
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""
Apply data/blocklist.jsonl now: take every blocked game out of data/ and log it.
save_main() already leaves blocked games out of EVERY write, so any data job
would do this on its own. This script is what makes it happen promptly: the
"Ingest New Game Links" workflow runs it first whenever /admin (or
delete_game.py) commits to the blocklist, and on its three-hourly schedule.
It also heals the log: a blocked game that is no longer in data/ but has no
line in scripts/removed_games.jsonl gets one. That covers a removal applied by
a job whose commit left the log out. Every blocklist line was written for a game
the catalogue held at the time (/admin and delete_game.py both refuse
anything else), so each logged line describes a real removal.
python scripts/remove_games.py # apply and log
python scripts/remove_games.py --max-drop 900 # a deliberate mass removal
Run from the repository root (DATA_DIR is relative).
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.blocklist import load_blocklist, removed_row
from core.constants import BLOCKLIST_JSONL, BLOCKLIST_MAX_DROP
from core.data_store import extract_appid, load_main, log_admin_removals, save_main
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--max-drop", type=int, default=BLOCKLIST_MAX_DROP,
help=f"refuse to remove more games than this in one run (default {BLOCKLIST_MAX_DROP})")
args = ap.parse_args(argv)
blocked = load_blocklist()
if not blocked:
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
return 0
games = load_main()
live = {extract_appid(g.get("link", "")) for g in games}
present = sorted(a for a in blocked if a in live)
if present:
# save_main() does the removal and logs each game it leaves out.
save_main(games, max_blocked_drop=args.max_drop)
print(f"Removed {len(present)} blocked game(s) from data/: {', '.join(present[:20])}"
+ (" …" if len(present) > 20 else ""))
else:
print("No blocked game is in data/.")
healed = log_admin_removals([removed_row(e) for a, e in blocked.items() if a not in live])
if healed:
print(f"Logged {healed} earlier removal(s) that were missing from scripts/removed_games.jsonl.")
print(f"Blocklist: {len(blocked)} game(s).")
return 0
if __name__ == "__main__":
sys.exit(main())
+5
View File
@@ -0,0 +1,5 @@
{"appid":"730","link":"https://store.steampowered.com/app/730/","name":"Counter-Strike 2","reason":"not_a_game","note":"","blocked_at":"2026-10-13T12:00:00Z","by":"admin"}
{"appid":"1172470","link":"https://store.steampowered.com/app/1172470/","name":"Apex Legends™","reason":"duplicate","note":"same game as 2357570","blocked_at":"2026-10-13T12:00:01Z","by":"admin"}
{"appid":"440","link":"https://store.steampowered.com/app/440/","name":"Ünïcödé — 中文 ゲーム","reason":"inappropriate","note":"He said \"hi\" \\ back / slash","blocked_at":"2026-10-13T12:00:02Z","by":"admin"}
{"appid":"10","link":"https://store.steampowered.com/app/10/","name":"🎮 Game\twith tab","reason":"other","note":"line one\nline two \u2028 sep \u2029 par \u0001 ctl  del","blocked_at":"2026-10-13T12:00:03Z","by":"admin"}
{"appid":"2357570","link":"https://store.steampowered.com/app/2357570/","name":"Overwatch®","reason":"broken","note":"Đã ngừng hoạt động <b>&amp;</b>","blocked_at":"2026-10-13T12:00:04Z","by":"admin"}
+197
View File
@@ -0,0 +1,197 @@
"""
data/blocklist.jsonl is a standing instruction, applied inside save_main() like
the overrides. These pin what makes that safe: a blocked game is left out of
every write without touching the caller's list, it is logged exactly once, a
second save changes no byte, a damaged file cannot empty the catalogue, and the
line format is the one /admin writes.
Run from the repository root: python -m pytest scripts/tests
"""
import json
import os
import pytest
from core import blocklist, data_store
from core.blocklist import BlocklistError, load_blocklist, make_entry, serialize_entry
GOLDEN = os.path.join(os.path.dirname(__file__), "fixtures", "blocklist_golden.jsonl")
def _game(appid: str, name: str = "") -> dict:
return {
"link": f"https://store.steampowered.com/app/{appid}/",
"name": name or f"Game {appid}",
"description": "",
"genre": "Action",
}
def _entry(appid: str, reason: str = "not_a_game", at: str = "2026-10-13T12:00:00Z", **kw) -> dict:
return make_entry(appid, kw.pop("name", f"Game {appid}"), reason, kw.pop("note", ""), at)
@pytest.fixture
def workdir(tmp_path, monkeypatch):
"""A scratch repository root (DATA_DIR and the log paths are relative)."""
monkeypatch.chdir(tmp_path)
os.makedirs("data")
os.makedirs("scripts")
stamps = iter(f"2026-01-01T00:00:{s:02d}Z" for s in range(60))
monkeypatch.setattr(data_store, "now_iso", lambda: next(stamps))
return tmp_path
def _block(*entries: dict) -> None:
with open(os.path.join("data", "blocklist.jsonl"), "a", encoding="utf-8", newline="\n") as f:
for e in entries:
f.write(serialize_entry(e) + "\n")
def _bytes(path: str) -> bytes:
return open(path, "rb").read() if os.path.isfile(path) else b""
def _snapshot() -> dict:
out = {}
for name in sorted(os.listdir("data")):
if name.endswith(".jsonl") or name == "index.json":
out[name] = _bytes(os.path.join("data", name))
out["removed"] = _bytes(os.path.join("scripts", "removed_games.jsonl"))
return out
def _removed() -> list[dict]:
return data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl"))
def _appids() -> list[str]:
return [data_store.extract_appid(g["link"]) for g in data_store.load_main()]
def test_without_a_blocklist_nothing_changes(workdir):
games = [_game("1"), _game("2")]
data_store.save_main(games, apply_human_overrides=False)
before = _snapshot()
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
assert _snapshot() == before
assert _removed() == []
def test_a_blocked_game_is_left_out_and_logged_once(workdir):
_block(_entry("2", name="Second"))
games = [_game("1"), _game("2", "Second"), _game("3")]
data_store.save_main(games, apply_human_overrides=False)
assert _appids() == ["1", "3"]
# The caller's list is not touched: scripts keep positions into it.
assert [g["link"] for g in games] == [_game(a)["link"] for a in ("1", "2", "3")]
assert _removed() == [{
"link": "https://store.steampowered.com/app/2/",
"name": "Second",
"appid": "2",
"reason": "Removed by the maintainer: Not a game (DLC, demo, tool or soundtrack)",
"status_code": "admin_removed",
"removed_at": "2026-10-13T12:00:00Z",
}]
index = json.load(open(os.path.join("data", "index.json"), encoding="utf-8"))
assert index["total"] == 2
def test_saving_again_changes_no_byte(workdir):
_block(_entry("2"))
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
before = _snapshot()
# The same list again (a script checkpointing), then a fresh load.
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
assert _snapshot() == before
assert len(_removed()) == 1
def test_a_blocked_game_not_in_data_writes_nothing_from_save_main(workdir):
_block(_entry("99"))
data_store.save_main([_game("1")], apply_human_overrides=False)
assert _removed() == []
def test_the_blocklist_applies_even_without_overrides(workdir):
_block(_entry("1"))
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=True)
assert _appids() == ["2"]
def test_too_many_drops_refuse_before_writing_anything(workdir):
data_store.save_main([_game("1"), _game("2"), _game("3")], apply_human_overrides=False)
before = _snapshot()
_block(_entry("1"), _entry("2"), _entry("3"))
with pytest.raises(BlocklistError):
data_store.save_main(data_store.load_main(), apply_human_overrides=False, max_blocked_drop=2)
after = _snapshot()
after.pop("blocklist.jsonl")
before.pop("blocklist.jsonl", None)
assert after == before
def test_damaged_and_incomplete_lines_block_nothing(workdir, capsys):
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
f.write("{not json\n")
f.write(json.dumps({"appid": "1", "reason": "not_free"}) + "\n") # no blocked_at
f.write(json.dumps({"appid": 2, "reason": "not_free", "blocked_at": "2026-10-13T12:00:00Z"}) + "\n")
f.write(json.dumps({"appid": "3", "reason": "nope", "blocked_at": "2026-10-13T12:00:00Z"}) + "\n")
f.write(json.dumps({"appid": "4", "reason": "not_free", "blocked_at": "2026-10-13T12:00:00.123Z"}) + "\n")
f.write(serialize_entry(_entry("5")) + "\n")
assert set(load_blocklist()) == {"5"}
data_store.save_main([_game(a) for a in "12345"], apply_human_overrides=False)
assert _appids() == ["1", "2", "3", "4"]
assert "ignored" in capsys.readouterr().out
def test_blocking_again_later_logs_the_new_removal(workdir):
_block(_entry("2", at="2026-10-13T12:00:00Z"))
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
# Restored (line deleted), re-added, blocked again later.
os.remove(os.path.join("data", "blocklist.jsonl"))
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
_block(_entry("2", at="2026-11-01T08:00:00Z"))
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
rows = _removed()
assert len(rows) == 1 and rows[0]["removed_at"] == "2026-11-01T08:00:00Z"
def test_a_later_purge_line_keeps_winning(workdir):
later = {"link": "https://store.steampowered.com/app/2/", "name": "Game 2", "appid": "2",
"reason": "No longer free-to-play", "status_code": "not_free",
"removed_at": "2026-12-01T00:00:00Z"}
data_store.save_jsonl(os.path.join("scripts", "removed_games.jsonl"), [later])
before = _bytes(os.path.join("scripts", "removed_games.jsonl"))
assert data_store.log_admin_removals([blocklist.removed_row(_entry("2"))]) == 0
assert _bytes(os.path.join("scripts", "removed_games.jsonl")) == before
def test_an_override_of_a_blocked_game_is_kept(workdir):
os.makedirs(os.path.join("data", "overrides"))
path = os.path.join("data", "overrides", "2.json")
with open(path, "w", encoding="utf-8") as fh:
json.dump({"schema": 1, "appid": "2", "fields": {"genre": {"value": "Puzzle", "was": "Action"}}}, fh)
_block(_entry("2"))
data_store.save_main([_game("1"), _game("2")])
assert os.path.isfile(path)
assert _appids() == ["1"]
def test_the_golden_lines_round_trip_byte_for_byte():
# web/shared/removal.test.ts reads the same file: both serialisers must
# reproduce every line exactly, escaping included.
for line in open(GOLDEN, encoding="utf-8").read().split("\n"):
if line:
assert serialize_entry(json.loads(line)) == line
def test_notes_are_one_line():
e = make_entry("1", "A
B", "other", " first\nsecond\t\u0001 third ", "2026-10-13T12:00:00Z")
assert e["note"] == "first second third"
assert e["name"] == "A B"
assert e["by"] == "admin"
with pytest.raises(ValueError):
make_entry("1", "x", "bogus", "", "2026-10-13T12:00:00Z")
+123
View File
@@ -0,0 +1,123 @@
"""
delete_game.py (the CLI twin of /admin/remove) and remove_games.py (what the
ingest workflow runs to apply the blocklist).
Run from the repository root: python -m pytest scripts/tests
"""
import json
import os
import pytest
import delete_game
import remove_games
from core import data_store
from core.blocklist import load_blocklist, removed_row
def _game(appid: str, name: str = "") -> dict:
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": name or f"Game {appid}",
"description": "", "genre": "Action"}
@pytest.fixture
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.makedirs("data")
os.makedirs("scripts")
monkeypatch.setattr(data_store, "now_iso", lambda: "2026-10-13T12:00:00Z")
monkeypatch.setattr(delete_game, "now_iso", lambda: "2026-10-13T12:00:00Z")
data_store.save_main([_game("10", "Ten"), _game("20", "Twenty"), _game("30")], apply_human_overrides=False)
return tmp_path
def _lines() -> list[str]:
path = os.path.join("data", "blocklist.jsonl")
return open(path, encoding="utf-8").read().splitlines() if os.path.isfile(path) else []
def _appids() -> list[str]:
return [data_store.extract_appid(g["link"]) for g in data_store.load_main()]
def test_adds_one_line_per_game_in_the_admin_format(repo):
assert delete_game.main(["10", "https://store.steampowered.com/app/20/", "--reason", "duplicate",
"--note", "same as 30"]) == 0
assert _lines() == [
'{"appid":"10","link":"https://store.steampowered.com/app/10/","name":"Ten","reason":"duplicate",'
'"note":"same as 30","blocked_at":"2026-10-13T12:00:00Z","by":"admin"}',
'{"appid":"20","link":"https://store.steampowered.com/app/20/","name":"Twenty","reason":"duplicate",'
'"note":"same as 30","blocked_at":"2026-10-13T12:00:00Z","by":"admin"}',
]
# Without --apply, data/ is untouched until the pipeline runs.
assert _appids() == ["10", "20", "30"]
def test_skips_what_is_blocked_or_absent_and_never_rewrites_lines(repo):
delete_game.main(["10", "--reason", "broken"])
first = _lines()
assert delete_game.main(["10", "99", "--reason", "broken"]) == 0
assert _lines() == first
def test_refuses_bad_input(repo):
assert delete_game.main(["10", "--reason", "other"]) == 2 # needs a note
assert delete_game.main(["not-a-game", "--reason", "broken"]) == 2
assert delete_game.main(["10"]) == 2 # no reason
assert _lines() == []
def test_apply_removes_and_logs(repo):
assert delete_game.main(["20", "--reason", "not_free", "--apply"]) == 0
assert _appids() == ["10", "30"]
rows = data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl"))
assert [(r["appid"], r["status_code"], r["removed_at"]) for r in rows] == [
("20", "admin_removed", "2026-10-13T12:00:00Z")]
def test_check_fails_on_a_line_that_would_block_nothing(repo, capsys):
delete_game.main(["10", "--reason", "broken"])
assert delete_game.main(["--check"]) == 0
with open(os.path.join("data", "blocklist.jsonl"), "a", encoding="utf-8") as f:
f.write(json.dumps({"appid": "20", "reason": "broken"}) + "\n")
assert delete_game.main(["--check"]) == 1
assert "blocked_at" in capsys.readouterr().out
def test_appends_after_a_file_without_a_final_newline(repo):
delete_game.main(["10", "--reason", "broken"])
path = os.path.join("data", "blocklist.jsonl")
text = open(path, encoding="utf-8").read().rstrip("\n")
open(path, "w", encoding="utf-8", newline="\n").write(text)
delete_game.main(["20", "--reason", "broken"])
assert len(_lines()) == 2 and set(load_blocklist()) == {"10", "20"}
def test_remove_games_applies_then_is_a_no_op(repo):
delete_game.main(["30", "--reason", "broken"])
assert remove_games.main([]) == 0
assert _appids() == ["10", "20"]
snapshot = {p: open(os.path.join("data", p), "rb").read() for p in os.listdir("data") if p != "overrides"}
log = open(os.path.join("scripts", "removed_games.jsonl"), "rb").read()
assert remove_games.main([]) == 0
assert {p: open(os.path.join("data", p), "rb").read() for p in os.listdir("data") if p != "overrides"} == snapshot
assert open(os.path.join("scripts", "removed_games.jsonl"), "rb").read() == log
def test_remove_games_heals_a_missing_log_line(repo):
delete_game.main(["30", "--reason", "broken", "--apply"])
# A job whose commit left the log out: the game is gone, the line is not.
os.remove(os.path.join("scripts", "removed_games.jsonl"))
assert remove_games.main([]) == 0
rows = data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl"))
assert rows == [removed_row(load_blocklist()["30"])]
def test_remove_games_respects_the_mass_drop_limit(repo):
delete_game.main(["10", "20", "30", "--reason", "broken"])
from core.blocklist import BlocklistError
with pytest.raises(BlocklistError):
remove_games.main(["--max-drop", "2"])
assert _appids() == ["10", "20", "30"]
assert remove_games.main(["--max-drop", "3"]) == 0
assert _appids() == []
+16
View File
@@ -14,3 +14,19 @@ def test_a_lookalike_host_is_not_taken_for_access():
# cloudflareaccess.com" (CodeQL py/incomplete-url-substring-sanitization).
out = discover_new._describe_access_redirect("https://evilcloudflareaccess.com/cdn-cgi/access/login/x")
assert out.startswith("unexpected redirect to")
def test_removed_games_count_as_known(tmp_path, monkeypatch):
# A game the maintainer took out must not be proposed again; the D1
# decision is one fence, the blocklist read here is the other.
import os
from core import data_store
from core.blocklist import make_entry, serialize_entry
monkeypatch.chdir(tmp_path)
os.makedirs("data")
data_store.save_main([{"link": "https://store.steampowered.com/app/1/", "name": "One"}],
apply_human_overrides=False)
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
f.write(serialize_entry(make_entry("2", "Two", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
assert discover_new.catalogue_appids() == {"1", "2"}
+80
View File
@@ -0,0 +1,80 @@
"""
ingest_new.py reads scripts/temp_info.jsonl, which two producers append to (the
browser extension and /admin). These pin the lines that used to crash or
slip through: a link that is not text, and a game the maintainer removed.
Steam is never called: check_game_health and fetch_full are replaced.
Run from the repository root: python -m pytest scripts/tests
"""
import json
import os
import pytest
import ingest_new
from core import data_store
from core.blocklist import make_entry, serialize_entry
from core.health_checker import HealthResult, OK
@pytest.fixture
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.makedirs("data")
os.makedirs("scripts")
data_store.save_main([{"link": "https://store.steampowered.com/app/1/", "name": "Existing"}],
apply_human_overrides=False)
calls: list[str] = []
def health(link, name="", client=None, details=None):
appid = data_store.extract_appid(link)
calls.append(appid)
return HealthResult(OK, appid, f"Game {appid}", link,
data=details if details is not None else {"type": "game", "name": f"Game {appid}"})
def fetch(game, client=None, prefetched_details=None):
game["name"] = (prefetched_details or {}).get("name", game.get("name", ""))
monkeypatch.setattr(ingest_new, "check_game_health", health)
monkeypatch.setattr(ingest_new, "fetch_full", fetch)
monkeypatch.setattr(ingest_new, "get_client", lambda: None)
return calls
def _queue(*lines) -> None:
with open(os.path.join("scripts", "temp_info.jsonl"), "w", encoding="utf-8", newline="\n") as f:
for line in lines:
f.write((line if isinstance(line, str) else json.dumps(line)) + "\n")
def _appids() -> list[str]:
return [data_store.extract_appid(g["link"]) for g in data_store.load_main()]
def test_parse_entries_turns_numbers_into_links_and_drops_nothing_silently():
out = ingest_new.parse_entries([730, "440 ", {"link": 570}, {"link": ["x"]}, {"link": True}, {"name": "x"}, True])
assert out == [
{"link": "730"},
{"link": "440"},
{"link": "570"},
{"link": ""}, # reported as an invalid line, not a crash
{"link": ""},
]
def test_a_numeric_link_no_longer_crashes_the_run(repo):
_queue({"link": 730}, {"link": ["not", "a", "link"]})
ingest_new.main()
assert _appids() == ["1", "730"]
# The queue is cleared, so the next three-hourly run does not crash again.
assert open(os.path.join("scripts", "temp_info.jsonl"), encoding="utf-8").read() == ""
def test_a_blocked_game_is_skipped_before_any_steam_request(repo):
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
f.write(serialize_entry(make_entry("440", "Blocked", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
_queue('{"link":"https://store.steampowered.com/app/440/"}', '"https://store.steampowered.com/app/570/"')
ingest_new.main()
assert repo == ["570"]
assert _appids() == ["1", "570"]