Files
poli0981andClaude Opus 5.5 3d0e22cd76 fix(pipeline): read anti-cheat from Steam's own disclosure; retire the extension's paths
None of the 4,704 games added since September had an anti-cheat. The only
code that read the anti-cheat a developer declares on Steam was the browser
extension (archived; last push 2026-06-25); the pipeline matched appdetails'
categories, which only ever say "Valve Anti-Cheat enabled".

- core/anticheat.py (new, pure): the store page's div.anticheat_section
  (its HEADING decides kernel, never the class; a Boot Protection row is a
  flag) and GetItems' extra_details.anticheat; labels from
  ANTI_CHEAT_PATTERNS, else Steam's name cut at 40 characters; same_family()
  ("EAC" = "Easy Anti-Cheat", "ACE" = "Anti-Cheat Expert (ACE)"); and
  apply_anti_cheat(), the one rule.
- fetch_full() fills, never replaces, from the page it already fetches (or
  GetItems for a gated page), else the VAC category, now found in any
  position (with a stored "" only the first category was checked).
- refresh_anticheat.py (new): GetItems for every game (~160 requests), a
  store page only for a disclosure without a name. Steam wins; an override
  wins over Steam and is not fought (OverrideGuard); no disclosure never
  clears; a failed batch is unknown. Runs at the end of the daily Refresh
  Store Data job with its own commit; only=anticheat runs it alone. That job
  now checks out main.
- The queue takes links only: merge_extension_data(), EXTENSION_FIELDS,
  ARRAY_FIELDS and the schema.ts mirrors go. translate_notes() becomes
  clean_notes(): the extension's leading "Has paid DLC" and "[hover-add]"
  are dropped (517 notes, once, never news).
- refetch_all.py keeps "-" instead of resetting it to ""; a retired override
  whose was equals its value no longer bumps last_updated on every save.
- "denuvo" joins Denuvo AC on both sides; anti-cheat.test.ts holds the TS
  patterns and enum to constants.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 09:38:39 +07:00

150 lines
5.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
Ingest new games v2.2.
Key optimization: health_checker now returns cached API data in HealthResult.data,
so fetch_full() can skip its own fetch_app_details() call → eliminates double-fetch.
"""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.blocklist import blocked_appids
from core.constants import REMOVED_JSONL
from core.data_store import (
load_main, save_main, load_temp, clear_temp,
load_jsonl, save_jsonl, dedup_removed,
normalize_link, extract_appid, build_index,
make_skeleton, now_iso,
)
from core.fetcher import fetch_full
from core.overrides import OverrideGuard
from core.steam_client import get_client
from core.health_checker import check_game_health, NETWORK_ERROR
def _link_text(value) -> str:
"""A queued link as text. `{"link": 730}` used to reach normalize_link() as
an int and crash the run before it saved or cleared the queue - so it
crashed again on every three-hourly retry. A number is an appid; anything
else that is not text becomes "" and is reported as an invalid line."""
if isinstance(value, bool):
return ""
if isinstance(value, int):
return str(value)
return value.strip() if isinstance(value, str) else ""
def parse_entries(records):
"""The queued links, as [{"link": text}]. A line is a store link and
nothing else: /admin writes {"link": url}, and any other key on a line -
the retired browser extension sent a whole record - is ignored, so a
queued line can never decide a field. The game comes from Steam."""
entries = []
for rec in records:
if isinstance(rec, (str, int)) and not isinstance(rec, bool):
entries.append({"link": _link_text(rec)})
elif isinstance(rec, dict) and rec.get("link") not in (None, ""):
entries.append({"link": _link_text(rec["link"])})
return entries
def main():
temp = load_temp()
if not temp:
print("temp_info.jsonl is empty.")
return
games = load_main()
index = build_index(games)
entries = parse_entries(temp)
# data/blocklist.jsonl: games the maintainer removed. save_main() would
# drop them anyway; skipping here also saves the Steam requests.
blocked = blocked_appids()
print(f"Found {len(entries)} entries\n")
added = skipped_dup = skipped_bad = skipped_net = skipped_blocked = 0
rejected = []
client = get_client()
# A restored game keeps its overrides: fetch_full() must not write a
# manual field one of them would undo.
guard = OverrideGuard.load()
for entry in entries:
link = normalize_link(entry.get("link", ""))
if not link:
print(f" ✗ Invalid: {entry.get('link', '?')[:60]}")
rejected.append({"link": entry.get("link", ""), "reason": "Invalid format",
"status_code": "invalid_format", "removed_at": now_iso()})
skipped_bad += 1
continue
appid = extract_appid(link)
if appid in blocked:
print(f" ⛔ Blocked ({appid}): removed by the maintainer, see data/blocklist.jsonl")
skipped_blocked += 1
continue
if appid in index:
print(f" ⏭ Dup ({appid})")
skipped_dup += 1
continue
# Health check (also fetches appdetails → cached in result.data)
print(f" 🔍 {appid}...", end=" ")
health = check_game_health(link, "", client)
if health.status == NETWORK_ERROR:
print("⚠ network – skip")
skipped_net += 1
continue
if health.should_remove:
print(f"❌ {health.reason}")
rejected.append({"link": link, "name": health.name, "appid": appid,
"reason": health.reason, "status_code": health.status,
"removed_at": now_iso()})
skipped_bad += 1
continue
# The catalogue is games. Discovery only proposes appdetails type
# "game", but this queue also takes a link a pull request adds and -
# with auto-approve on - candidates nobody looked at, so a free DLC,
# demo or tool is refused HERE, where Steam's own answer is in hand.
# Not in health_checker: Purge uses it, and would start deleting records.
app_type = str((health.data or {}).get("type") or "").strip().lower()
if app_type and app_type != "game":
print(f"❌ not a game (Steam type: {app_type})")
rejected.append({"link": link, "name": health.name, "appid": appid,
"reason": f"Not a game (Steam type: {app_type})",
"status_code": "not_a_game", "removed_at": now_iso()})
skipped_bad += 1
continue
print("✅")
# Build the record from Steam (reusing the health check's appdetails,
# so no second call). Human values arrive through overrides, which
# save_main() re-imposes.
game = make_skeleton(link)
print(f" ➕ Fetching ({appid})...")
fetch_full(game, client=client, prefetched_details=health.data, guard=guard)
games.append(game)
index[appid] = len(games) - 1
added += 1
dev = game.get("developer", [])
print(f" → {game.get('name', '?')} [{game.get('genre', '?')}] "
f"dev={dev[0] if dev else '?'} tags={len(game.get('tags', []))}")
save_main(games)
if rejected:
existing = load_jsonl(REMOVED_JSONL)
save_jsonl(REMOVED_JSONL, dedup_removed(existing + rejected))
clear_temp()
print(f"\n{'─'*50}")
print(f"Added: {added} | Dup: {skipped_dup} | Bad: {skipped_bad} | Net: {skipped_net}"
f" | Blocked: {skipped_blocked}")
print(f"Total: {len(games)}")
if __name__ == "__main__":
main()