feat(pipeline): data/blocklist.jsonl - save_main drops and logs removed games
A game the maintainer removes is now a standing instruction in Git, like an
override, instead of a one-time deletion that the browser extension or the
next ingest could undo.
- core/blocklist.py reads data/blocklist.jsonl (one line per game, written by
/admin and by delete_game.py). save_main() applies it FIRST on every write,
for all 14 callers, without touching the caller's list, and whatever
apply_human_overrides says.
- A dropped game is logged once in scripts/removed_games.jsonl as
status_code "admin_removed", built from the blocklist line alone with
removed_at = blocked_at, so writing it again changes no byte. A later purge
line for the same game keeps winning, as dedup_removed() decides.
- More than 500 drops in one save raises BlocklistError before anything is
written: a damaged file fails the job instead of emptying the catalogue.
- remove_games.py applies the blocklist now and heals a missing log line.
bash/ingest.sh runs it before ingest_new.py, and ingest-new.yml also
triggers on blocklist pushes and checks out the tip of main. One workflow
on purpose: data-write keeps one pending run, so a second workflow and the
ingest would keep evicting each other.
- ingest_new.py skips blocked links before any Steam request, and a link
that is not text ({"link": 730}) no longer crashes the run (and every
three-hourly retry after it).
- Discovery counts blocked appids as known.
- delete_game.py is a non-interactive CLI writing the same bytes as /admin
(golden fixture), with --list and --check; check-overrides.yml runs --check.
- mark-dead-games.yml commits through bash/commit_push.sh, like every other
data job: it rebases and retries, and commits the log with the shards.
- dependabot.yml ignores Svelte-era framework majors and @types/node majors
instead of the React-era names.
With an empty blocklist every path is a no-op.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e70e1f770e
commit
986077d3f1
18 files changed
+1055
-61
No files matched your search
+12
-11
@@ -1,10 +1,8 @@
|
||||
# NOTE: committing this file REPLACES the Dependabot configuration currently
|
||||
# set through repo settings. Existing open Dependabot PRs (e.g. #111) are
|
||||
# superseded and will be recreated against these groups on the next run.
|
||||
# set through repo settings.
|
||||
#
|
||||
# web/worker/ is intentionally absent: it does not exist yet. Add a fifth
|
||||
# npm entry for it as part of the Cloudflare Worker work — that package will
|
||||
# hold the repo-write credential, so it must be covered.
|
||||
# web/worker/ and web/admin/ need no entry of their own: they are built from
|
||||
# web/package.json, which the /web entry below covers.
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
@@ -36,19 +34,22 @@ updates:
|
||||
patterns: ['@tauri-apps/*']
|
||||
ignore:
|
||||
# Framework majors are migrations with their own staged PRs, not
|
||||
# something to take as an unattended bump.
|
||||
- dependency-name: react
|
||||
# something to take as an unattended bump. (SvelteKit 3 needs the
|
||||
# cookie override in package.json re-keyed or dropped; see CLAUDE.md.)
|
||||
- dependency-name: '@sveltejs/kit'
|
||||
update-types: [version-update:semver-major]
|
||||
- dependency-name: react-dom
|
||||
- dependency-name: '@sveltejs/adapter-static'
|
||||
update-types: [version-update:semver-major]
|
||||
- dependency-name: react-router-dom
|
||||
- dependency-name: svelte
|
||||
update-types: [version-update:semver-major]
|
||||
- dependency-name: tailwindcss
|
||||
update-types: [version-update:semver-major]
|
||||
- dependency-name: typescript
|
||||
update-types: [version-update:semver-major]
|
||||
# lucide-react ships breaking icon renames on 0.x minors.
|
||||
- dependency-name: lucide-react
|
||||
# The type definitions follow the Node the build runs on (24, in
|
||||
# web-ci.yml and the release workflows), not the newest Node.
|
||||
- dependency-name: '@types/node'
|
||||
update-types: [version-update:semver-major]
|
||||
|
||||
# ── Desktop / Android shell ───────────────────────────────────────────────
|
||||
- package-ecosystem: cargo
|
||||
|
||||
@@ -18,6 +18,13 @@ on:
|
||||
# no enforcement in either direction.
|
||||
- 'web/worker/routes/edit.ts'
|
||||
- 'web/worker/lib/override-doc.ts'
|
||||
# The blocklist is the same kind of standing human instruction, with the
|
||||
# same two writers (/admin and a CLI) that must produce identical bytes.
|
||||
- 'data/blocklist.jsonl'
|
||||
- 'scripts/core/blocklist.py'
|
||||
- 'scripts/delete_game.py'
|
||||
- 'web/shared/removal.ts'
|
||||
- 'scripts/tests/fixtures/blocklist_golden.jsonl'
|
||||
- '.github/workflows/check-overrides.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
@@ -30,6 +37,13 @@ on:
|
||||
# no enforcement in either direction.
|
||||
- 'web/worker/routes/edit.ts'
|
||||
- 'web/worker/lib/override-doc.ts'
|
||||
# The blocklist is the same kind of standing human instruction, with the
|
||||
# same two writers (/admin and a CLI) that must produce identical bytes.
|
||||
- 'data/blocklist.jsonl'
|
||||
- 'scripts/core/blocklist.py'
|
||||
- 'scripts/delete_game.py'
|
||||
- 'web/shared/removal.ts'
|
||||
- 'scripts/tests/fixtures/blocklist_golden.jsonl'
|
||||
- '.github/workflows/check-overrides.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -65,6 +79,14 @@ jobs:
|
||||
- name: Install web deps
|
||||
working-directory: web
|
||||
run: npm ci
|
||||
- name: Validate the blocklist
|
||||
# A line load_blocklist() cannot honour blocks nothing - silently.
|
||||
run: python scripts/delete_game.py --check
|
||||
- name: Assert CLI/Worker override parity
|
||||
working-directory: web
|
||||
run: npx vitest run worker/lib/override-doc.test.ts
|
||||
- name: Assert CLI/Worker blocklist parity
|
||||
# web-ci only runs for web/**; a Python-only change to the line format
|
||||
# must fail somewhere too.
|
||||
working-directory: web
|
||||
run: npx vitest run shared/removal.test.ts
|
||||
@@ -3,7 +3,12 @@ on :
|
||||
# branches filter is REQUIRED: without it every v*/desktop-v* tag push
|
||||
# matches the paths filter (tag pushes compare against an empty base),
|
||||
# fires the ingest on a detached HEAD, and dies with exit 128 on push.
|
||||
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl' ] }
|
||||
#
|
||||
# data/blocklist.jsonl is here too: this job also applies removals
|
||||
# (bash/ingest.sh runs scripts/remove_games.py first). A separate removal
|
||||
# workflow would share `data-write` with this one, and each would evict the
|
||||
# other's pending run.
|
||||
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl', 'data/blocklist.jsonl' ] }
|
||||
# Orphan recovery, not a poll. `data-write` keeps at most ONE run pending and
|
||||
# cancels the older one, so a push here that queues behind a ~115-minute data
|
||||
# job is silently evicted the moment any OTHER data-write workflow queues --
|
||||
@@ -23,6 +28,11 @@ jobs :
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
# The tip of main when the job STARTS, not the commit that triggered
|
||||
# it. A run that waited behind a long data job would otherwise work on
|
||||
# shards that job has since rewritten, and its push would fail to
|
||||
# rebase.
|
||||
with : { ref : main }
|
||||
-
|
||||
uses : actions/setup-python@v6
|
||||
with : { python-version : '3.12' }
|
||||
|
||||
@@ -28,6 +28,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# The tip of main when the job starts, not when it was queued: a run
|
||||
# that waited behind a long data job would otherwise rewrite stale shards.
|
||||
with:
|
||||
ref: main
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
@@ -49,14 +53,9 @@ jobs:
|
||||
|
||||
- name: Commit changes
|
||||
if: ${{ inputs.dry_run != true }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add data/ scripts/dead_games.log
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes."
|
||||
else
|
||||
git commit -m "chore(data): mark dead games"
|
||||
git push
|
||||
fi
|
||||
|
||||
# Through commit_push.sh like every other data job: it rebases and
|
||||
# retries when main moved during the run (a bare `git push` threw the
|
||||
# run away), and it commits the whole tree, so scripts/removed_games.jsonl
|
||||
# goes with the shards when save_main() applied a removal
|
||||
# (scripts/core/blocklist.py).
|
||||
run: bash ./bash/commit_push.sh "chore(data): mark dead games"
|
||||
+17
-1
@@ -1,5 +1,21 @@
|
||||
#!/bin/bash
|
||||
# The ingest job, which is also the removal job.
|
||||
#
|
||||
# remove_games.py applies data/blocklist.jsonl (games the maintainer removed in
|
||||
# /admin or with delete_game.py) before ingest_new.py adds the queued links.
|
||||
# One workflow does both on purpose: `data-write` keeps ONE pending run and a
|
||||
# newer one evicts it, so a separate removal workflow and the ingest would keep
|
||||
# cancelling each other. Whichever run survives does both.
|
||||
set -euo pipefail
|
||||
pip install --quiet -r requirements.txt
|
||||
python scripts/remove_games.py
|
||||
python scripts/ingest_new.py
|
||||
bash "$(dirname "$0")/commit_push.sh" "Ingest games [$(date +'%Y-%m-%d %H:%M')]"
|
||||
|
||||
# Name the commit after what changed: the queue file is emptied only when
|
||||
# ingest_new.py processed something.
|
||||
if git diff --quiet -- scripts/temp_info.jsonl; then
|
||||
msg="Apply removals [$(date +'%Y-%m-%d %H:%M')]"
|
||||
else
|
||||
msg="Ingest games [$(date +'%Y-%m-%d %H:%M')]"
|
||||
fi
|
||||
bash "$(dirname "$0")/commit_push.sh" "$msg"
|
||||
Whitespace-only changes.
@@ -0,0 +1,190 @@
|
||||
"""
|
||||
Games the maintainer took out of the catalogue, as a standing instruction.
|
||||
|
||||
data/blocklist.jsonl holds one line per blocked game. It is written by the
|
||||
Worker (/admin/remove appends lines, Restore deletes one) and by
|
||||
scripts/delete_game.py, and only READ here. Like data/overrides/, it is human
|
||||
intent that every write re-imposes: save_main() drops a blocked record before
|
||||
anything else happens to it, so no writer - the ingest of a link the browser
|
||||
extension queued again, a full refetch, a purge - can put the game back. The
|
||||
only way back is to delete the line (Restore in /admin), which also queues the
|
||||
link for ingest_new.py in the same commit.
|
||||
|
||||
A drop is logged once in scripts/removed_games.jsonl, built from the blocklist
|
||||
line alone. Its removed_at IS the line's blocked_at, so writing it again is a
|
||||
byte-for-byte no-op and the log cannot churn (data_store.log_admin_removals).
|
||||
|
||||
Line format, keys in this order, compact separators (the Worker writes the
|
||||
same bytes - web/shared/removal.ts, held equal by the golden fixture in
|
||||
scripts/tests/fixtures/blocklist_golden.jsonl):
|
||||
|
||||
{"appid":"730","link":"https://store.steampowered.com/app/730/",
|
||||
"name":"Counter-Strike 2","reason":"not_a_game","note":"","blocked_at":
|
||||
"2026-10-13T12:00:00Z","by":"admin"}
|
||||
|
||||
Standard library only: scripts/core/* is stdlib-only, and the discovery
|
||||
workflows import this without installing anything.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from .constants import BLOCKLIST_JSONL
|
||||
|
||||
# The reason codes /admin offers, with the words the public log shows.
|
||||
# web/shared/removal.ts REMOVAL_REASONS mirrors this table (a vitest reads it
|
||||
# from this file and compares).
|
||||
REASONS: dict[str, str] = {
|
||||
"not_free": "No longer free to play",
|
||||
"not_a_game": "Not a game (DLC, demo, tool or soundtrack)",
|
||||
"duplicate": "Duplicate entry",
|
||||
"broken": "Unplayable or shut down",
|
||||
"inappropriate": "Does not fit the catalogue",
|
||||
"other": "Other",
|
||||
}
|
||||
|
||||
ENTRY_KEYS = ("appid", "link", "name", "reason", "note", "blocked_at", "by")
|
||||
|
||||
# Written into every line instead of a person: the file is public forever.
|
||||
ATTRIBUTION = "admin"
|
||||
|
||||
STATUS_CODE = "admin_removed"
|
||||
|
||||
_APPID = re.compile(r"^\d{1,10}$")
|
||||
# now_iso()'s shape exactly. dedup_removed() compares removed_at as a string,
|
||||
# so a millisecond suffix would sort differently from the pipeline's own rows.
|
||||
_ISO_SECONDS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
|
||||
|
||||
|
||||
class BlocklistError(RuntimeError):
|
||||
"""A blocklist that would remove an implausible number of games at once."""
|
||||
|
||||
|
||||
def canonical_link(appid: str) -> str:
|
||||
return f"https://store.steampowered.com/app/{appid}/"
|
||||
|
||||
|
||||
def entry_problem(entry: object) -> Optional[str]:
|
||||
"""Why a parsed line cannot be honoured, or None when it can."""
|
||||
if not isinstance(entry, dict):
|
||||
return "not an object"
|
||||
appid = entry.get("appid")
|
||||
if not isinstance(appid, str) or not _APPID.match(appid):
|
||||
return "appid must be 1-10 digits, as a string"
|
||||
blocked_at = entry.get("blocked_at")
|
||||
if not isinstance(blocked_at, str) or not _ISO_SECONDS.match(blocked_at):
|
||||
# Without a fixed timestamp the log line would be stamped anew on every
|
||||
# write and rewritten forever.
|
||||
return "blocked_at must look like 2026-10-13T12:00:00Z"
|
||||
if entry.get("reason") not in REASONS:
|
||||
return f"reason must be one of {', '.join(REASONS)}"
|
||||
for key in ("link", "name", "note"):
|
||||
if not isinstance(entry.get(key, ""), str):
|
||||
return f"{key} must be a string"
|
||||
return None
|
||||
|
||||
|
||||
def read_lines(path: str = BLOCKLIST_JSONL) -> list[str]:
|
||||
"""The file's non-empty lines, verbatim. [] when it does not exist."""
|
||||
if not os.path.isfile(path):
|
||||
return []
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return [line.rstrip("\n") for line in f if line.strip()]
|
||||
|
||||
|
||||
def load_blocklist(path: str = BLOCKLIST_JSONL, quiet: bool = False) -> dict[str, dict]:
|
||||
"""appid -> entry for every line that can be honoured.
|
||||
|
||||
Tolerant on purpose, like apply_overrides(): a bad line is reported and
|
||||
skipped, never raised. A skipped line blocks nothing, so the worst a
|
||||
damaged line can do is fail to remove one game - `delete_game.py --check`
|
||||
(run in CI) is what makes it visible. The LAST valid line for an appid wins.
|
||||
"""
|
||||
out: dict[str, dict] = {}
|
||||
for lineno, line in enumerate(read_lines(path), 1):
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError as e:
|
||||
if not quiet:
|
||||
print(f" [blocklist] {path}:{lineno} bad JSON, ignored: {e}")
|
||||
continue
|
||||
problem = entry_problem(entry)
|
||||
if problem:
|
||||
if not quiet:
|
||||
print(f" [blocklist] {path}:{lineno} ignored: {problem}")
|
||||
continue
|
||||
out[entry["appid"]] = entry
|
||||
return out
|
||||
|
||||
|
||||
def blocked_appids(path: str = BLOCKLIST_JSONL) -> set[str]:
|
||||
return set(load_blocklist(path, quiet=True))
|
||||
|
||||
|
||||
def make_entry(appid: str, name: str, reason: str, note: str, blocked_at: str,
|
||||
link: Optional[str] = None) -> dict:
|
||||
entry = {
|
||||
"appid": appid,
|
||||
"link": link or canonical_link(appid),
|
||||
"name": clean_text(name, 300),
|
||||
"reason": reason,
|
||||
"note": clean_text(note, 300),
|
||||
"blocked_at": blocked_at,
|
||||
"by": ATTRIBUTION,
|
||||
}
|
||||
problem = entry_problem(entry)
|
||||
if problem:
|
||||
raise ValueError(problem)
|
||||
return entry
|
||||
|
||||
|
||||
def clean_text(value: str, limit: int) -> str:
|
||||
"""A one-line note: control characters and line breaks (U+2028/U+2029
|
||||
included) become spaces, runs of whitespace collapse, at most `limit`
|
||||
characters. web/shared/removal.ts cleanText() is the same rule."""
|
||||
out = []
|
||||
for ch in value or "":
|
||||
out.append(" " if (ord(ch) < 0x20 or 0x7F <= ord(ch) < 0xA0 or ch in "\u2028\u2029") else ch)
|
||||
return " ".join("".join(out).split())[:limit]
|
||||
|
||||
|
||||
def serialize_entry(entry: dict) -> str:
|
||||
"""One line, without its newline: keys in ENTRY_KEYS order, compact
|
||||
separators, non-ASCII kept as is - except U+2028/U+2029, escaped, because
|
||||
str.splitlines() and some editors treat them as line breaks and would cut a
|
||||
line in two. web/shared/removal.ts serializeEntry() writes the same bytes."""
|
||||
ordered = {key: entry.get(key, "") for key in ENTRY_KEYS}
|
||||
text = json.dumps(ordered, ensure_ascii=False, separators=(",", ":"))
|
||||
return text.replace("\u2028", "\\u2028").replace("\u2029", "\\u2029")
|
||||
|
||||
|
||||
def public_reason(entry: dict) -> str:
|
||||
"""The reason the public removed-games log shows. The free-text note is
|
||||
deliberately left out of it (it stays in the blocklist and the commit)."""
|
||||
return f"Removed by the maintainer: {REASONS.get(entry.get('reason', ''), 'Other')}"
|
||||
|
||||
|
||||
def removed_row(entry: dict) -> dict:
|
||||
"""The scripts/removed_games.jsonl line for a blocked game. Built from the
|
||||
blocklist line alone, so it is identical however often it is written."""
|
||||
return {
|
||||
"link": entry.get("link") or canonical_link(entry["appid"]),
|
||||
"name": entry.get("name", ""),
|
||||
"appid": entry["appid"],
|
||||
"reason": public_reason(entry),
|
||||
"status_code": STATUS_CODE,
|
||||
"removed_at": entry["blocked_at"],
|
||||
}
|
||||
|
||||
|
||||
def split_blocked(records: Iterable[dict], blocked: dict[str, dict],
|
||||
appid_of) -> tuple[list[dict], list[dict]]:
|
||||
"""(kept, dropped). A record whose appid cannot be read is kept: this
|
||||
module only ever removes what it can name."""
|
||||
kept: list[dict] = []
|
||||
dropped: list[dict] = []
|
||||
for record in records:
|
||||
appid = appid_of(record.get("link", "")) if isinstance(record, dict) else None
|
||||
(dropped if appid and appid in blocked else kept).append(record)
|
||||
return kept, dropped
|
||||
@@ -11,6 +11,13 @@ DEAD_LINKS_LOG = "scripts/dead_links.log"
|
||||
REMOVED_JSONL = "scripts/removed_games.jsonl"
|
||||
GAMES_DIR = "games"
|
||||
OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game
|
||||
# Games the maintainer took out of the catalogue (core/blocklist.py). Never
|
||||
# name it data_*: save_main() treats data/data_*.jsonl as shards and deletes
|
||||
# the ones it did not write.
|
||||
BLOCKLIST_JSONL = "data/blocklist.jsonl"
|
||||
# More blocked games than this in ONE save is treated as a damaged file, not a
|
||||
# decision: save_main() refuses before writing anything (5 x the /admin cap).
|
||||
BLOCKLIST_MAX_DROP = 500
|
||||
|
||||
# ──────────── Sharding ────────────
|
||||
MAX_RECORDS_PER_FILE = 800
|
||||
|
||||
@@ -16,9 +16,10 @@ from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from .constants import (
|
||||
DATA_DIR, LEGACY_JSONL, TEMP_JSONL,
|
||||
DATA_DIR, LEGACY_JSONL, TEMP_JSONL, REMOVED_JSONL,
|
||||
MAX_RECORDS_PER_FILE, SHARD_PREFIX,
|
||||
MANUAL_FIELDS, ARRAY_FIELDS, DESCRIPTION_MAX,
|
||||
BLOCKLIST_MAX_DROP,
|
||||
)
|
||||
from .normalize import normalize_records
|
||||
|
||||
@@ -128,6 +129,68 @@ def dedup_removed(records: list[dict]) -> list[dict]:
|
||||
return list(latest.values())
|
||||
|
||||
|
||||
def log_admin_removals(rows: list[dict]) -> int:
|
||||
"""Make sure the removed-games log carries these blocklist rows.
|
||||
|
||||
A row is added only when the log's latest line for that appid is not
|
||||
already exactly it. Because each row is built from its blocklist line
|
||||
(removed_at = blocked_at), writing it a second time is a no-op, and a later
|
||||
line for the same game - a purge that logged it first - keeps winning, as
|
||||
dedup_removed() already decides. The file is left untouched unless
|
||||
something actually changes, so this can run on every save.
|
||||
"""
|
||||
if not rows:
|
||||
return 0
|
||||
existing = load_jsonl(REMOVED_JSONL)
|
||||
latest = {}
|
||||
for r in dedup_removed(existing):
|
||||
latest[r.get("appid") or extract_appid(r.get("link", ""))] = r
|
||||
missing = [row for row in rows if latest.get(row["appid"]) != row]
|
||||
if not missing:
|
||||
return 0
|
||||
merged = dedup_removed(existing + missing)
|
||||
if merged == existing:
|
||||
return 0
|
||||
os.makedirs(os.path.dirname(REMOVED_JSONL) or ".", exist_ok=True)
|
||||
save_jsonl(REMOVED_JSONL, merged)
|
||||
return len(missing)
|
||||
|
||||
|
||||
def apply_blocklist(records: list[dict], max_drop: int = BLOCKLIST_MAX_DROP) -> list[dict]:
|
||||
"""The records to WRITE: everything except games in data/blocklist.jsonl.
|
||||
|
||||
Returns a new list and never touches the caller's: several scripts hold
|
||||
positions into theirs (ingest_new.py's appid -> index map) and some save
|
||||
part-way through a loop. A drop is logged in scripts/removed_games.jsonl,
|
||||
once. More than `max_drop` drops in one save raises BEFORE anything is
|
||||
written - a damaged file must fail the job loudly, not empty the catalogue.
|
||||
"""
|
||||
from .blocklist import BlocklistError, load_blocklist, removed_row, split_blocked
|
||||
|
||||
blocked = load_blocklist()
|
||||
if not blocked:
|
||||
return records
|
||||
kept, dropped = split_blocked(records, blocked, extract_appid)
|
||||
if not dropped:
|
||||
return records
|
||||
if len(dropped) > max_drop:
|
||||
raise BlocklistError(
|
||||
f"data/blocklist.jsonl would remove {len(dropped)} games in one save "
|
||||
f"(limit {max_drop}). Refusing to write anything. If that is intended, "
|
||||
f"run scripts/remove_games.py --max-drop {len(dropped)}."
|
||||
)
|
||||
appids = []
|
||||
for record in dropped:
|
||||
appid = extract_appid(record.get("link", ""))
|
||||
if appid not in appids:
|
||||
appids.append(appid)
|
||||
logged = log_admin_removals([removed_row(blocked[a]) for a in appids])
|
||||
print(f" blocklist: left out {len(appids)} blocked game(s)"
|
||||
+ (f", logged {logged} in {REMOVED_JSONL}" if logged else "")
|
||||
+ f": {', '.join(appids[:10])}{' …' if len(appids) > 10 else ''}")
|
||||
return kept
|
||||
|
||||
|
||||
# ──────────── Shard helpers ────────────
|
||||
|
||||
def _shard_paths() -> list[str]:
|
||||
@@ -200,16 +263,22 @@ def truncate_descriptions(records: list[dict]) -> int:
|
||||
return n
|
||||
|
||||
|
||||
def save_main(records: list[dict], apply_human_overrides: bool = True):
|
||||
def save_main(records: list[dict], apply_human_overrides: bool = True,
|
||||
max_blocked_drop: int = BLOCKLIST_MAX_DROP):
|
||||
"""Save records to sharded JSONL files (MAX_RECORDS_PER_FILE per shard).
|
||||
|
||||
Human overrides are re-imposed here, immediately before the write. This is
|
||||
the single choke point for the dataset - all 11 data-writing scripts call
|
||||
the single choke point for the dataset - all 14 data-writing scripts call
|
||||
save_main() and nothing else writes data/data_*.jsonl - so applying them
|
||||
here is what makes a correction survive refetch_all.py and
|
||||
normalize_genres.py --apply without either script knowing this layer
|
||||
exists. See core/overrides.py.
|
||||
|
||||
The blocklist (core/blocklist.py) is applied FIRST, the same way: a game
|
||||
the maintainer removed is left out of every write, whichever script made
|
||||
it, and that is never switched off - not even with
|
||||
apply_human_overrides=False. The caller's list is not modified.
|
||||
|
||||
apply_human_overrides=False exists for tooling that must see the
|
||||
un-overridden values (an audit report). Never pass it from the pipeline.
|
||||
|
||||
@@ -218,6 +287,11 @@ def save_main(records: list[dict], apply_human_overrides: bool = True):
|
||||
notes - so every writer gets them, the browser extension's ingest
|
||||
included. Like the truncation, they never touch last_updated.
|
||||
"""
|
||||
# Before everything else, so no work is spent on records that are not
|
||||
# written, and a blocked game's override reports as an orphan (its file
|
||||
# stays, and applies again if the game is restored).
|
||||
records = apply_blocklist(records, max_drop=max_blocked_drop)
|
||||
|
||||
# Before the truncation: a decoded """ is one character, not six.
|
||||
normalized = normalize_records(records)
|
||||
if normalized:
|
||||
|
||||
+189
-26
@@ -1,30 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Delete game (interactive)."""
|
||||
import sys, os
|
||||
"""
|
||||
Remove games from the catalogue - the command-line twin of /admin/remove.
|
||||
|
||||
Removing a game is not a one-time deletion. It adds a line to
|
||||
data/blocklist.jsonl, a standing instruction that save_main() re-imposes on
|
||||
every write (core/blocklist.py): the game leaves data/, is logged once in
|
||||
scripts/removed_games.jsonl, and nothing - the browser extension, a refetch,
|
||||
the discovery sweep - can bring it back. Restoring is done in /admin (Remove ->
|
||||
Removed -> Restore), because it also has to clear the review queue's record.
|
||||
|
||||
python scripts/delete_game.py 730 --reason not_a_game
|
||||
python scripts/delete_game.py https://store.steampowered.com/app/730/ 440 \
|
||||
--reason duplicate --note "same game as 2357570" --apply
|
||||
python scripts/delete_game.py --list
|
||||
python scripts/delete_game.py --check # validate the file (runs in CI)
|
||||
|
||||
Writes the same bytes /admin writes (held equal by
|
||||
scripts/tests/fixtures/blocklist_golden.jsonl). The reason and the note are
|
||||
PUBLIC: the blocklist is a file in a public repository. Without --apply,
|
||||
data/ changes on the next pipeline run (pushing the blocklist starts one).
|
||||
|
||||
Run from the repository root (DATA_DIR is relative).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from core.data_store import load_main, save_main, extract_appid, normalize_link
|
||||
|
||||
def main():
|
||||
games = load_main()
|
||||
if not games: print("Empty."); return
|
||||
q = input(f"Search ({len(games)} games): ").strip()
|
||||
if not q: return
|
||||
norm = normalize_link(q)
|
||||
tid = extract_appid(norm) if norm else (q if q.isdigit() else None)
|
||||
matches = [(i,g) for i,g in enumerate(games)
|
||||
if (tid and extract_appid(g.get("link","")) == tid)
|
||||
or q.lower() in g.get("name","").lower()]
|
||||
if not matches: print("Not found."); return
|
||||
for idx,(i,g) in enumerate(matches,1):
|
||||
print(f" {idx}. {g.get('name','?')} ({g.get('link','?')})")
|
||||
c = input(f"Delete (1-{len(matches)}/all/q): ").strip()
|
||||
if c.lower()=="q": return
|
||||
d = ({i for i,_ in matches} if c.lower()=="all"
|
||||
else {matches[int(c)-1][0]} if c.isdigit() and 1<=int(c)<=len(matches) else set())
|
||||
if not d: print("Invalid."); return
|
||||
if input(f"Confirm {len(d)}? (y/n): ").strip().lower()!="y": return
|
||||
games = [g for i,g in enumerate(games) if i not in d]
|
||||
save_main(games)
|
||||
print(f"✓ Deleted {len(d)}. {len(games)} left.")
|
||||
from core.blocklist import (
|
||||
REASONS, entry_problem, load_blocklist, make_entry, read_lines, serialize_entry,
|
||||
)
|
||||
from core.constants import BLOCKLIST_JSONL
|
||||
from core.data_store import extract_appid, load_main, now_iso
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
# Same cap as /admin (web/shared/removal.ts MAX_REMOVE).
|
||||
MAX_REMOVE = 100
|
||||
MAX_NOTE = 300
|
||||
|
||||
|
||||
def safe_stdout():
|
||||
"""Game names are routinely non-Latin; a cp1252 console would mangle them."""
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def _appid_of(raw: str):
|
||||
raw = raw.strip()
|
||||
return raw if raw.isdigit() else extract_appid(raw)
|
||||
|
||||
|
||||
def _append(lines: list[str], path: str = BLOCKLIST_JSONL) -> None:
|
||||
"""Append whole lines, never re-serialising the ones already there."""
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
needs_newline = False
|
||||
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(-1, os.SEEK_END)
|
||||
needs_newline = f.read(1) != b"\n"
|
||||
with open(path, "a", encoding="utf-8", newline="\n") as f:
|
||||
if needs_newline:
|
||||
f.write("\n")
|
||||
for line in lines:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def cmd_check(path: str = BLOCKLIST_JSONL) -> int:
|
||||
problems = 0
|
||||
seen: dict[str, int] = {}
|
||||
for lineno, line in enumerate(read_lines(path), 1):
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"{path}:{lineno}: not JSON: {e}")
|
||||
problems += 1
|
||||
continue
|
||||
problem = entry_problem(entry)
|
||||
if problem:
|
||||
print(f"{path}:{lineno}: {problem}")
|
||||
problems += 1
|
||||
continue
|
||||
if entry["appid"] in seen:
|
||||
# Harmless (the last line wins) but never written by /admin.
|
||||
print(f"{path}:{lineno}: note: {entry['appid']} also on line {seen[entry['appid']]}")
|
||||
seen[entry["appid"]] = lineno
|
||||
print(f"{path}: {len(seen)} blocked game(s), {problems} problem(s).")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
def cmd_list() -> int:
|
||||
entries = load_blocklist()
|
||||
if not entries:
|
||||
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
|
||||
return 0
|
||||
for e in entries.values():
|
||||
note = f" - {e['note']}" if e.get("note") else ""
|
||||
print(f"{e['appid']:>10} {e['blocked_at']} {e['reason']:<13} {e.get('name', '')}{note}")
|
||||
print(f"{len(entries)} blocked game(s).")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_add(targets: list[str], reason: str, note: str, apply: bool) -> int:
|
||||
if reason not in REASONS:
|
||||
print(f"--reason must be one of: {', '.join(REASONS)}")
|
||||
return 2
|
||||
note = (note or "").strip()
|
||||
if len(note) > MAX_NOTE:
|
||||
print(f"--note is longer than {MAX_NOTE} characters.")
|
||||
return 2
|
||||
if reason == "other" and not note:
|
||||
print("--reason other needs a --note saying why.")
|
||||
return 2
|
||||
|
||||
appids: list[str] = []
|
||||
for raw in targets:
|
||||
appid = _appid_of(raw)
|
||||
if not appid:
|
||||
print(f"Not a Steam appid or store link: {raw}")
|
||||
return 2
|
||||
if appid not in appids:
|
||||
appids.append(appid)
|
||||
if len(appids) > MAX_REMOVE:
|
||||
print(f"At most {MAX_REMOVE} games at once, as in /admin.")
|
||||
return 2
|
||||
|
||||
blocked = load_blocklist()
|
||||
catalogue = {}
|
||||
for game in load_main():
|
||||
aid = extract_appid(game.get("link", ""))
|
||||
if aid:
|
||||
catalogue[aid] = game
|
||||
|
||||
stamp = now_iso()
|
||||
lines = []
|
||||
for appid in appids:
|
||||
if appid in blocked:
|
||||
print(f" skip {appid}: already blocked since {blocked[appid]['blocked_at']}")
|
||||
continue
|
||||
game = catalogue.get(appid)
|
||||
if game is None:
|
||||
# Every blocklist line must describe a game the catalogue held, so
|
||||
# that each removed-games log line describes a real removal.
|
||||
print(f" skip {appid}: not in data/ (nothing to remove)")
|
||||
continue
|
||||
entry = make_entry(appid, str(game.get("name", "")), reason, note, stamp,
|
||||
link=game.get("link"))
|
||||
lines.append(serialize_entry(entry))
|
||||
print(f" block {appid}: {entry['name']}")
|
||||
|
||||
if not lines:
|
||||
print("Nothing to add.")
|
||||
return 0
|
||||
_append(lines)
|
||||
print(f"Added {len(lines)} line(s) to {BLOCKLIST_JSONL}.")
|
||||
|
||||
if apply:
|
||||
import remove_games
|
||||
return remove_games.main([])
|
||||
print("data/ changes on the next pipeline run (or run again with --apply).")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("targets", nargs="*", help="Steam appids or store links")
|
||||
ap.add_argument("--reason", choices=list(REASONS), help="why the games are removed (public)")
|
||||
ap.add_argument("--note", default="", help="optional detail, at most 300 characters (public)")
|
||||
ap.add_argument("--apply", action="store_true", help="take them out of data/ now")
|
||||
ap.add_argument("--list", action="store_true", help="print the blocklist")
|
||||
ap.add_argument("--check", action="store_true", help="validate data/blocklist.jsonl; exit 1 on a problem")
|
||||
args = ap.parse_args(argv)
|
||||
safe_stdout()
|
||||
|
||||
if args.check:
|
||||
return cmd_check()
|
||||
if args.list:
|
||||
return cmd_list()
|
||||
if not args.targets:
|
||||
ap.print_help()
|
||||
return 2
|
||||
if not args.reason:
|
||||
print("--reason is required when removing games.")
|
||||
return 2
|
||||
return cmd_add(args.targets, args.reason, args.note, args.apply)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -26,6 +26,7 @@ import urllib.request
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.blocklist import blocked_appids
|
||||
from core.data_store import load_main, extract_appid
|
||||
from core.steam_client import get_client
|
||||
|
||||
@@ -134,8 +135,14 @@ def parse_search_rows(results_html):
|
||||
|
||||
|
||||
def catalogue_appids():
|
||||
"""Appids already in data/."""
|
||||
out = set()
|
||||
"""Appids already in data/, plus the ones the maintainer removed.
|
||||
|
||||
A game in data/blocklist.jsonl was taken out on purpose; proposing it again
|
||||
would only put it back in the review queue. The Worker refuses it too (its
|
||||
removal recorded a rejected decision), so this is the second fence, and the
|
||||
one that holds even when D1 and Git disagree.
|
||||
"""
|
||||
out = set(blocked_appids())
|
||||
for rec in load_main():
|
||||
aid = extract_appid(rec.get("link", ""))
|
||||
if aid:
|
||||
|
||||
+27
-6
@@ -8,6 +8,7 @@ so fetch_full() can skip its own fetch_app_details() call → eliminates double-
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.blocklist import blocked_appids
|
||||
from core.constants import REMOVED_JSONL, MANUAL_FIELDS
|
||||
from core.data_store import (
|
||||
load_main, save_main, load_temp, clear_temp,
|
||||
@@ -20,13 +21,25 @@ from core.steam_client import get_client
|
||||
from core.health_checker import check_game_health, NETWORK_ERROR
|
||||
|
||||
|
||||
def _link_text(value) -> str:
|
||||
"""A queued link as text. `{"link": 730}` used to reach normalize_link() as
|
||||
an int and crash the run before it saved or cleared the queue - so it
|
||||
crashed again on every three-hourly retry. A number is an appid; anything
|
||||
else that is not text becomes "" and is reported as an invalid line."""
|
||||
if isinstance(value, bool):
|
||||
return ""
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def parse_entries(records):
|
||||
entries = []
|
||||
for rec in records:
|
||||
if isinstance(rec, str):
|
||||
entries.append({"link": rec.strip()})
|
||||
elif isinstance(rec, dict) and rec.get("link"):
|
||||
entries.append(rec)
|
||||
if isinstance(rec, (str, int)) and not isinstance(rec, bool):
|
||||
entries.append({"link": _link_text(rec)})
|
||||
elif isinstance(rec, dict) and rec.get("link") not in (None, ""):
|
||||
entries.append({**rec, "link": _link_text(rec["link"])})
|
||||
return entries
|
||||
|
||||
|
||||
@@ -39,9 +52,12 @@ def main():
|
||||
games = load_main()
|
||||
index = build_index(games)
|
||||
entries = parse_entries(temp)
|
||||
# data/blocklist.jsonl: games the maintainer removed. save_main() would
|
||||
# drop them anyway; skipping here also saves the Steam requests.
|
||||
blocked = blocked_appids()
|
||||
print(f"Found {len(entries)} entries\n")
|
||||
|
||||
added = skipped_dup = skipped_bad = skipped_net = 0
|
||||
added = skipped_dup = skipped_bad = skipped_net = skipped_blocked = 0
|
||||
rejected = []
|
||||
client = get_client()
|
||||
|
||||
@@ -55,6 +71,10 @@ def main():
|
||||
continue
|
||||
|
||||
appid = extract_appid(link)
|
||||
if appid in blocked:
|
||||
print(f" ⛔ Blocked ({appid}): removed by the maintainer, see data/blocklist.jsonl")
|
||||
skipped_blocked += 1
|
||||
continue
|
||||
if appid in index:
|
||||
print(f" ⏭ Dup ({appid})")
|
||||
skipped_dup += 1
|
||||
@@ -108,7 +128,8 @@ def main():
|
||||
clear_temp()
|
||||
|
||||
print(f"\n{'─'*50}")
|
||||
print(f"Added: {added} | Dup: {skipped_dup} | Bad: {skipped_bad} | Net: {skipped_net}")
|
||||
print(f"Added: {added} | Dup: {skipped_dup} | Bad: {skipped_bad} | Net: {skipped_net}"
|
||||
f" | Blocked: {skipped_blocked}")
|
||||
print(f"Total: {len(games)}")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Apply data/blocklist.jsonl now: take every blocked game out of data/ and log it.
|
||||
|
||||
save_main() already leaves blocked games out of EVERY write, so any data job
|
||||
would do this on its own. This script is what makes it happen promptly: the
|
||||
"Ingest New Game Links" workflow runs it first whenever /admin (or
|
||||
delete_game.py) commits to the blocklist, and on its three-hourly schedule.
|
||||
|
||||
It also heals the log: a blocked game that is no longer in data/ but has no
|
||||
line in scripts/removed_games.jsonl gets one. That covers a removal applied by
|
||||
a job whose commit left the log out. Every blocklist line was written for a game
|
||||
the catalogue held at the time (/admin and delete_game.py both refuse
|
||||
anything else), so each logged line describes a real removal.
|
||||
|
||||
python scripts/remove_games.py # apply and log
|
||||
python scripts/remove_games.py --max-drop 900 # a deliberate mass removal
|
||||
|
||||
Run from the repository root (DATA_DIR is relative).
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.blocklist import load_blocklist, removed_row
|
||||
from core.constants import BLOCKLIST_JSONL, BLOCKLIST_MAX_DROP
|
||||
from core.data_store import extract_appid, load_main, log_admin_removals, save_main
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--max-drop", type=int, default=BLOCKLIST_MAX_DROP,
|
||||
help=f"refuse to remove more games than this in one run (default {BLOCKLIST_MAX_DROP})")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
blocked = load_blocklist()
|
||||
if not blocked:
|
||||
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
|
||||
return 0
|
||||
|
||||
games = load_main()
|
||||
live = {extract_appid(g.get("link", "")) for g in games}
|
||||
present = sorted(a for a in blocked if a in live)
|
||||
|
||||
if present:
|
||||
# save_main() does the removal and logs each game it leaves out.
|
||||
save_main(games, max_blocked_drop=args.max_drop)
|
||||
print(f"Removed {len(present)} blocked game(s) from data/: {', '.join(present[:20])}"
|
||||
+ (" …" if len(present) > 20 else ""))
|
||||
else:
|
||||
print("No blocked game is in data/.")
|
||||
|
||||
healed = log_admin_removals([removed_row(e) for a, e in blocked.items() if a not in live])
|
||||
if healed:
|
||||
print(f"Logged {healed} earlier removal(s) that were missing from scripts/removed_games.jsonl.")
|
||||
print(f"Blocklist: {len(blocked)} game(s).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,5 @@
|
||||
{"appid":"730","link":"https://store.steampowered.com/app/730/","name":"Counter-Strike 2","reason":"not_a_game","note":"","blocked_at":"2026-10-13T12:00:00Z","by":"admin"}
|
||||
{"appid":"1172470","link":"https://store.steampowered.com/app/1172470/","name":"Apex Legends™","reason":"duplicate","note":"same game as 2357570","blocked_at":"2026-10-13T12:00:01Z","by":"admin"}
|
||||
{"appid":"440","link":"https://store.steampowered.com/app/440/","name":"Ünïcödé — 中文 ゲーム","reason":"inappropriate","note":"He said \"hi\" \\ back / slash","blocked_at":"2026-10-13T12:00:02Z","by":"admin"}
|
||||
{"appid":"10","link":"https://store.steampowered.com/app/10/","name":"🎮 Game\twith tab","reason":"other","note":"line one\nline two \u2028 sep \u2029 par \u0001 ctl del","blocked_at":"2026-10-13T12:00:03Z","by":"admin"}
|
||||
{"appid":"2357570","link":"https://store.steampowered.com/app/2357570/","name":"Overwatch®","reason":"broken","note":"Đã ngừng hoạt động <b>&</b>","blocked_at":"2026-10-13T12:00:04Z","by":"admin"}
|
||||
@@ -0,0 +1,197 @@
|
||||
"""
|
||||
data/blocklist.jsonl is a standing instruction, applied inside save_main() like
|
||||
the overrides. These pin what makes that safe: a blocked game is left out of
|
||||
every write without touching the caller's list, it is logged exactly once, a
|
||||
second save changes no byte, a damaged file cannot empty the catalogue, and the
|
||||
line format is the one /admin writes.
|
||||
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from core import blocklist, data_store
|
||||
from core.blocklist import BlocklistError, load_blocklist, make_entry, serialize_entry
|
||||
|
||||
GOLDEN = os.path.join(os.path.dirname(__file__), "fixtures", "blocklist_golden.jsonl")
|
||||
|
||||
|
||||
def _game(appid: str, name: str = "") -> dict:
|
||||
return {
|
||||
"link": f"https://store.steampowered.com/app/{appid}/",
|
||||
"name": name or f"Game {appid}",
|
||||
"description": "",
|
||||
"genre": "Action",
|
||||
}
|
||||
|
||||
|
||||
def _entry(appid: str, reason: str = "not_a_game", at: str = "2026-10-13T12:00:00Z", **kw) -> dict:
|
||||
return make_entry(appid, kw.pop("name", f"Game {appid}"), reason, kw.pop("note", ""), at)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workdir(tmp_path, monkeypatch):
|
||||
"""A scratch repository root (DATA_DIR and the log paths are relative)."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
os.makedirs("scripts")
|
||||
stamps = iter(f"2026-01-01T00:00:{s:02d}Z" for s in range(60))
|
||||
monkeypatch.setattr(data_store, "now_iso", lambda: next(stamps))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _block(*entries: dict) -> None:
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "a", encoding="utf-8", newline="\n") as f:
|
||||
for e in entries:
|
||||
f.write(serialize_entry(e) + "\n")
|
||||
|
||||
|
||||
def _bytes(path: str) -> bytes:
|
||||
return open(path, "rb").read() if os.path.isfile(path) else b""
|
||||
|
||||
|
||||
def _snapshot() -> dict:
|
||||
out = {}
|
||||
for name in sorted(os.listdir("data")):
|
||||
if name.endswith(".jsonl") or name == "index.json":
|
||||
out[name] = _bytes(os.path.join("data", name))
|
||||
out["removed"] = _bytes(os.path.join("scripts", "removed_games.jsonl"))
|
||||
return out
|
||||
|
||||
|
||||
def _removed() -> list[dict]:
|
||||
return data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl"))
|
||||
|
||||
|
||||
def _appids() -> list[str]:
|
||||
return [data_store.extract_appid(g["link"]) for g in data_store.load_main()]
|
||||
|
||||
|
||||
def test_without_a_blocklist_nothing_changes(workdir):
|
||||
games = [_game("1"), _game("2")]
|
||||
data_store.save_main(games, apply_human_overrides=False)
|
||||
before = _snapshot()
|
||||
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
|
||||
assert _snapshot() == before
|
||||
assert _removed() == []
|
||||
|
||||
|
||||
def test_a_blocked_game_is_left_out_and_logged_once(workdir):
|
||||
_block(_entry("2", name="Second"))
|
||||
games = [_game("1"), _game("2", "Second"), _game("3")]
|
||||
data_store.save_main(games, apply_human_overrides=False)
|
||||
|
||||
assert _appids() == ["1", "3"]
|
||||
# The caller's list is not touched: scripts keep positions into it.
|
||||
assert [g["link"] for g in games] == [_game(a)["link"] for a in ("1", "2", "3")]
|
||||
assert _removed() == [{
|
||||
"link": "https://store.steampowered.com/app/2/",
|
||||
"name": "Second",
|
||||
"appid": "2",
|
||||
"reason": "Removed by the maintainer: Not a game (DLC, demo, tool or soundtrack)",
|
||||
"status_code": "admin_removed",
|
||||
"removed_at": "2026-10-13T12:00:00Z",
|
||||
}]
|
||||
index = json.load(open(os.path.join("data", "index.json"), encoding="utf-8"))
|
||||
assert index["total"] == 2
|
||||
|
||||
|
||||
def test_saving_again_changes_no_byte(workdir):
|
||||
_block(_entry("2"))
|
||||
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
|
||||
before = _snapshot()
|
||||
# The same list again (a script checkpointing), then a fresh load.
|
||||
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
|
||||
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
|
||||
assert _snapshot() == before
|
||||
assert len(_removed()) == 1
|
||||
|
||||
|
||||
def test_a_blocked_game_not_in_data_writes_nothing_from_save_main(workdir):
|
||||
_block(_entry("99"))
|
||||
data_store.save_main([_game("1")], apply_human_overrides=False)
|
||||
assert _removed() == []
|
||||
|
||||
|
||||
def test_the_blocklist_applies_even_without_overrides(workdir):
|
||||
_block(_entry("1"))
|
||||
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=True)
|
||||
assert _appids() == ["2"]
|
||||
|
||||
|
||||
def test_too_many_drops_refuse_before_writing_anything(workdir):
|
||||
data_store.save_main([_game("1"), _game("2"), _game("3")], apply_human_overrides=False)
|
||||
before = _snapshot()
|
||||
_block(_entry("1"), _entry("2"), _entry("3"))
|
||||
with pytest.raises(BlocklistError):
|
||||
data_store.save_main(data_store.load_main(), apply_human_overrides=False, max_blocked_drop=2)
|
||||
after = _snapshot()
|
||||
after.pop("blocklist.jsonl")
|
||||
before.pop("blocklist.jsonl", None)
|
||||
assert after == before
|
||||
|
||||
|
||||
def test_damaged_and_incomplete_lines_block_nothing(workdir, capsys):
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write("{not json\n")
|
||||
f.write(json.dumps({"appid": "1", "reason": "not_free"}) + "\n") # no blocked_at
|
||||
f.write(json.dumps({"appid": 2, "reason": "not_free", "blocked_at": "2026-10-13T12:00:00Z"}) + "\n")
|
||||
f.write(json.dumps({"appid": "3", "reason": "nope", "blocked_at": "2026-10-13T12:00:00Z"}) + "\n")
|
||||
f.write(json.dumps({"appid": "4", "reason": "not_free", "blocked_at": "2026-10-13T12:00:00.123Z"}) + "\n")
|
||||
f.write(serialize_entry(_entry("5")) + "\n")
|
||||
assert set(load_blocklist()) == {"5"}
|
||||
data_store.save_main([_game(a) for a in "12345"], apply_human_overrides=False)
|
||||
assert _appids() == ["1", "2", "3", "4"]
|
||||
assert "ignored" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_blocking_again_later_logs_the_new_removal(workdir):
|
||||
_block(_entry("2", at="2026-10-13T12:00:00Z"))
|
||||
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
|
||||
# Restored (line deleted), re-added, blocked again later.
|
||||
os.remove(os.path.join("data", "blocklist.jsonl"))
|
||||
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
|
||||
_block(_entry("2", at="2026-11-01T08:00:00Z"))
|
||||
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
|
||||
rows = _removed()
|
||||
assert len(rows) == 1 and rows[0]["removed_at"] == "2026-11-01T08:00:00Z"
|
||||
|
||||
|
||||
def test_a_later_purge_line_keeps_winning(workdir):
|
||||
later = {"link": "https://store.steampowered.com/app/2/", "name": "Game 2", "appid": "2",
|
||||
"reason": "No longer free-to-play", "status_code": "not_free",
|
||||
"removed_at": "2026-12-01T00:00:00Z"}
|
||||
data_store.save_jsonl(os.path.join("scripts", "removed_games.jsonl"), [later])
|
||||
before = _bytes(os.path.join("scripts", "removed_games.jsonl"))
|
||||
assert data_store.log_admin_removals([blocklist.removed_row(_entry("2"))]) == 0
|
||||
assert _bytes(os.path.join("scripts", "removed_games.jsonl")) == before
|
||||
|
||||
|
||||
def test_an_override_of_a_blocked_game_is_kept(workdir):
|
||||
os.makedirs(os.path.join("data", "overrides"))
|
||||
path = os.path.join("data", "overrides", "2.json")
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump({"schema": 1, "appid": "2", "fields": {"genre": {"value": "Puzzle", "was": "Action"}}}, fh)
|
||||
_block(_entry("2"))
|
||||
data_store.save_main([_game("1"), _game("2")])
|
||||
assert os.path.isfile(path)
|
||||
assert _appids() == ["1"]
|
||||
|
||||
|
||||
def test_the_golden_lines_round_trip_byte_for_byte():
|
||||
# web/shared/removal.test.ts reads the same file: both serialisers must
|
||||
# reproduce every line exactly, escaping included.
|
||||
for line in open(GOLDEN, encoding="utf-8").read().split("\n"):
|
||||
if line:
|
||||
assert serialize_entry(json.loads(line)) == line
|
||||
|
||||
|
||||
def test_notes_are_one_line():
|
||||
e = make_entry("1", "A
B", "other", " first\nsecond\t\u0001 third ", "2026-10-13T12:00:00Z")
|
||||
assert e["note"] == "first second third"
|
||||
assert e["name"] == "A B"
|
||||
assert e["by"] == "admin"
|
||||
with pytest.raises(ValueError):
|
||||
make_entry("1", "x", "bogus", "", "2026-10-13T12:00:00Z")
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
delete_game.py (the CLI twin of /admin/remove) and remove_games.py (what the
|
||||
ingest workflow runs to apply the blocklist).
|
||||
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import delete_game
|
||||
import remove_games
|
||||
from core import data_store
|
||||
from core.blocklist import load_blocklist, removed_row
|
||||
|
||||
|
||||
def _game(appid: str, name: str = "") -> dict:
|
||||
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": name or f"Game {appid}",
|
||||
"description": "", "genre": "Action"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
os.makedirs("scripts")
|
||||
monkeypatch.setattr(data_store, "now_iso", lambda: "2026-10-13T12:00:00Z")
|
||||
monkeypatch.setattr(delete_game, "now_iso", lambda: "2026-10-13T12:00:00Z")
|
||||
data_store.save_main([_game("10", "Ten"), _game("20", "Twenty"), _game("30")], apply_human_overrides=False)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _lines() -> list[str]:
|
||||
path = os.path.join("data", "blocklist.jsonl")
|
||||
return open(path, encoding="utf-8").read().splitlines() if os.path.isfile(path) else []
|
||||
|
||||
|
||||
def _appids() -> list[str]:
|
||||
return [data_store.extract_appid(g["link"]) for g in data_store.load_main()]
|
||||
|
||||
|
||||
def test_adds_one_line_per_game_in_the_admin_format(repo):
|
||||
assert delete_game.main(["10", "https://store.steampowered.com/app/20/", "--reason", "duplicate",
|
||||
"--note", "same as 30"]) == 0
|
||||
assert _lines() == [
|
||||
'{"appid":"10","link":"https://store.steampowered.com/app/10/","name":"Ten","reason":"duplicate",'
|
||||
'"note":"same as 30","blocked_at":"2026-10-13T12:00:00Z","by":"admin"}',
|
||||
'{"appid":"20","link":"https://store.steampowered.com/app/20/","name":"Twenty","reason":"duplicate",'
|
||||
'"note":"same as 30","blocked_at":"2026-10-13T12:00:00Z","by":"admin"}',
|
||||
]
|
||||
# Without --apply, data/ is untouched until the pipeline runs.
|
||||
assert _appids() == ["10", "20", "30"]
|
||||
|
||||
|
||||
def test_skips_what_is_blocked_or_absent_and_never_rewrites_lines(repo):
|
||||
delete_game.main(["10", "--reason", "broken"])
|
||||
first = _lines()
|
||||
assert delete_game.main(["10", "99", "--reason", "broken"]) == 0
|
||||
assert _lines() == first
|
||||
|
||||
|
||||
def test_refuses_bad_input(repo):
|
||||
assert delete_game.main(["10", "--reason", "other"]) == 2 # needs a note
|
||||
assert delete_game.main(["not-a-game", "--reason", "broken"]) == 2
|
||||
assert delete_game.main(["10"]) == 2 # no reason
|
||||
assert _lines() == []
|
||||
|
||||
|
||||
def test_apply_removes_and_logs(repo):
|
||||
assert delete_game.main(["20", "--reason", "not_free", "--apply"]) == 0
|
||||
assert _appids() == ["10", "30"]
|
||||
rows = data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl"))
|
||||
assert [(r["appid"], r["status_code"], r["removed_at"]) for r in rows] == [
|
||||
("20", "admin_removed", "2026-10-13T12:00:00Z")]
|
||||
|
||||
|
||||
def test_check_fails_on_a_line_that_would_block_nothing(repo, capsys):
|
||||
delete_game.main(["10", "--reason", "broken"])
|
||||
assert delete_game.main(["--check"]) == 0
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps({"appid": "20", "reason": "broken"}) + "\n")
|
||||
assert delete_game.main(["--check"]) == 1
|
||||
assert "blocked_at" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_appends_after_a_file_without_a_final_newline(repo):
|
||||
delete_game.main(["10", "--reason", "broken"])
|
||||
path = os.path.join("data", "blocklist.jsonl")
|
||||
text = open(path, encoding="utf-8").read().rstrip("\n")
|
||||
open(path, "w", encoding="utf-8", newline="\n").write(text)
|
||||
delete_game.main(["20", "--reason", "broken"])
|
||||
assert len(_lines()) == 2 and set(load_blocklist()) == {"10", "20"}
|
||||
|
||||
|
||||
def test_remove_games_applies_then_is_a_no_op(repo):
|
||||
delete_game.main(["30", "--reason", "broken"])
|
||||
assert remove_games.main([]) == 0
|
||||
assert _appids() == ["10", "20"]
|
||||
snapshot = {p: open(os.path.join("data", p), "rb").read() for p in os.listdir("data") if p != "overrides"}
|
||||
log = open(os.path.join("scripts", "removed_games.jsonl"), "rb").read()
|
||||
assert remove_games.main([]) == 0
|
||||
assert {p: open(os.path.join("data", p), "rb").read() for p in os.listdir("data") if p != "overrides"} == snapshot
|
||||
assert open(os.path.join("scripts", "removed_games.jsonl"), "rb").read() == log
|
||||
|
||||
|
||||
def test_remove_games_heals_a_missing_log_line(repo):
|
||||
delete_game.main(["30", "--reason", "broken", "--apply"])
|
||||
# A job whose commit left the log out: the game is gone, the line is not.
|
||||
os.remove(os.path.join("scripts", "removed_games.jsonl"))
|
||||
assert remove_games.main([]) == 0
|
||||
rows = data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl"))
|
||||
assert rows == [removed_row(load_blocklist()["30"])]
|
||||
|
||||
|
||||
def test_remove_games_respects_the_mass_drop_limit(repo):
|
||||
delete_game.main(["10", "20", "30", "--reason", "broken"])
|
||||
from core.blocklist import BlocklistError
|
||||
with pytest.raises(BlocklistError):
|
||||
remove_games.main(["--max-drop", "2"])
|
||||
assert _appids() == ["10", "20", "30"]
|
||||
assert remove_games.main(["--max-drop", "3"]) == 0
|
||||
assert _appids() == []
|
||||
@@ -14,3 +14,19 @@ def test_a_lookalike_host_is_not_taken_for_access():
|
||||
# cloudflareaccess.com" (CodeQL py/incomplete-url-substring-sanitization).
|
||||
out = discover_new._describe_access_redirect("https://evilcloudflareaccess.com/cdn-cgi/access/login/x")
|
||||
assert out.startswith("unexpected redirect to")
|
||||
|
||||
|
||||
def test_removed_games_count_as_known(tmp_path, monkeypatch):
|
||||
# A game the maintainer took out must not be proposed again; the D1
|
||||
# decision is one fence, the blocklist read here is the other.
|
||||
import os
|
||||
from core import data_store
|
||||
from core.blocklist import make_entry, serialize_entry
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
data_store.save_main([{"link": "https://store.steampowered.com/app/1/", "name": "One"}],
|
||||
apply_human_overrides=False)
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(serialize_entry(make_entry("2", "Two", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
|
||||
assert discover_new.catalogue_appids() == {"1", "2"}
|
||||
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
ingest_new.py reads scripts/temp_info.jsonl, which two producers append to (the
|
||||
browser extension and /admin). These pin the lines that used to crash or
|
||||
slip through: a link that is not text, and a game the maintainer removed.
|
||||
|
||||
Steam is never called: check_game_health and fetch_full are replaced.
|
||||
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import ingest_new
|
||||
from core import data_store
|
||||
from core.blocklist import make_entry, serialize_entry
|
||||
from core.health_checker import HealthResult, OK
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
os.makedirs("scripts")
|
||||
data_store.save_main([{"link": "https://store.steampowered.com/app/1/", "name": "Existing"}],
|
||||
apply_human_overrides=False)
|
||||
calls: list[str] = []
|
||||
|
||||
def health(link, name="", client=None, details=None):
|
||||
appid = data_store.extract_appid(link)
|
||||
calls.append(appid)
|
||||
return HealthResult(OK, appid, f"Game {appid}", link,
|
||||
data=details if details is not None else {"type": "game", "name": f"Game {appid}"})
|
||||
|
||||
def fetch(game, client=None, prefetched_details=None):
|
||||
game["name"] = (prefetched_details or {}).get("name", game.get("name", ""))
|
||||
|
||||
monkeypatch.setattr(ingest_new, "check_game_health", health)
|
||||
monkeypatch.setattr(ingest_new, "fetch_full", fetch)
|
||||
monkeypatch.setattr(ingest_new, "get_client", lambda: None)
|
||||
return calls
|
||||
|
||||
|
||||
def _queue(*lines) -> None:
|
||||
with open(os.path.join("scripts", "temp_info.jsonl"), "w", encoding="utf-8", newline="\n") as f:
|
||||
for line in lines:
|
||||
f.write((line if isinstance(line, str) else json.dumps(line)) + "\n")
|
||||
|
||||
|
||||
def _appids() -> list[str]:
|
||||
return [data_store.extract_appid(g["link"]) for g in data_store.load_main()]
|
||||
|
||||
|
||||
def test_parse_entries_turns_numbers_into_links_and_drops_nothing_silently():
|
||||
out = ingest_new.parse_entries([730, "440 ", {"link": 570}, {"link": ["x"]}, {"link": True}, {"name": "x"}, True])
|
||||
assert out == [
|
||||
{"link": "730"},
|
||||
{"link": "440"},
|
||||
{"link": "570"},
|
||||
{"link": ""}, # reported as an invalid line, not a crash
|
||||
{"link": ""},
|
||||
]
|
||||
|
||||
|
||||
def test_a_numeric_link_no_longer_crashes_the_run(repo):
|
||||
_queue({"link": 730}, {"link": ["not", "a", "link"]})
|
||||
ingest_new.main()
|
||||
assert _appids() == ["1", "730"]
|
||||
# The queue is cleared, so the next three-hourly run does not crash again.
|
||||
assert open(os.path.join("scripts", "temp_info.jsonl"), encoding="utf-8").read() == ""
|
||||
|
||||
|
||||
def test_a_blocked_game_is_skipped_before_any_steam_request(repo):
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(serialize_entry(make_entry("440", "Blocked", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
|
||||
_queue('{"link":"https://store.steampowered.com/app/440/"}', '"https://store.steampowered.com/app/570/"')
|
||||
ingest_new.main()
|
||||
assert repo == ["570"]
|
||||
assert _appids() == ["1", "570"]
|
||||
Reference in new issue
Block a user