Files
poli0981andClaude Opus 5.5 6f6eb3a71f feat(discovery): propose free games in every language
The store search pinned supportedlang=english, which left out every free
game that does not list English: 1,592 of 18,790 on 2026-10-30, most of
them Chinese, Japanese or Korean only (5118700 among them). Without the
parameter Steam filters on no language at all, with l=english or an
Accept-Language: ja header alike.

- fetch_search_page() sends no supportedlang, and sends l=english so the
  row text ("Coming soon", month names) stays English by parameter.
- Candidates carry their languages (appdetails_languages()), a dry run
  says which list no English, and every run counts them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 17:28:10 +07:00

476 lines
18 KiB
Python

#!/usr/bin/env python3
"""Discover newly-released free-to-play games and propose them for review.
This script PROPOSES ONLY. It never writes to data/, never commits, and cannot
add a game to the catalogue: it POSTs candidates to the admin Worker, which
stores them as `pending` for a human to approve. That separation is the point —
discovery is automated, publication is not.
Deliberately NOT imported: anything that writes the dataset. See the CI guard in
.github/workflows/discover-new.yml.
Usage:
python scripts/discover_new.py --mode daily
python scripts/discover_new.py --mode daily --dry-run
"""
import argparse
import base64
import html
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.blocklist import blocked_appids
from core.data_store import load_main, extract_appid
from core.steam_client import get_client
# Steam caps the search page at 100 rows regardless of what we ask for.
PAGE_SIZE = 100
# A page spans about a week of releases (13 a day in October 2026, every
# language), so five pages is over a month of insurance. If the cron silently
# stops for a few weeks, the next successful run still catches everything with
# no intervention.
DEFAULT_MAX_PAGES = 5
DEFAULT_STOP_AFTER_KNOWN = 2
# The Worker's own cap; batches larger than this are rejected.
POST_BATCH = 100
# Steam knows about 30 languages; the cap only bounds what a publisher's
# string can put into the payload.
MAX_LANGUAGES = 40
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
# NOT `col search_released`: the live markup is
# `<div class="search_released responsive_secondrow">`. The `col ` prefix
# never matched, so every row came back with an empty date and the cheap
# "Coming soon" skip below silently never fired.
_RE_RELEASED = re.compile(r'<div class="search_released[^"]*">(.*?)</div>', re.S)
_RE_TAGS = re.compile(r"<[^>]+>")
_RE_BR = re.compile(r"<br\s*/?>", re.I)
# Types that are not games. `category1=998` already filters most of these out,
# but appdetails is the authoritative source and occasionally disagrees.
ALLOWED_TYPES = {"game"}
class Candidate:
__slots__ = ("appid", "link", "name", "header_image", "release_date",
"app_type", "is_free", "payload")
def __init__(self, appid, data):
self.appid = appid
self.link = f"https://store.steampowered.com/app/{appid}/"
self.name = (data.get("name") or "")[:200]
self.header_image = (data.get("header_image") or "")[:500]
rd = data.get("release_date") or {}
self.release_date = (rd.get("date") or "")[:60]
self.app_type = (data.get("type") or "")[:30]
self.is_free = bool(data.get("is_free"))
self.payload = {
"short_description": (data.get("short_description") or "")[:600],
"genres": [g.get("description") for g in (data.get("genres") or [])][:12],
"platforms": data.get("platforms") or {},
"developers": (data.get("developers") or [])[:6],
"publishers": (data.get("publishers") or [])[:6],
# Discovery proposes games in every language; this is what tells a
# reviewer which ones a reader of English cannot play.
"languages": appdetails_languages(data.get("supported_languages")),
}
def to_wire(self):
return {
"appid": self.appid,
"link": self.link,
"name": self.name,
"header_image": self.header_image,
"release_date": self.release_date,
"app_type": self.app_type,
"is_free": self.is_free,
"health_status": "ok",
"payload": self.payload,
}
def safe_stdout():
"""Stop a non-Latin game name from killing the run on a cp1252 console.
Game names are publisher-controlled and routinely contain CJK, Cyrillic or
emoji. On a Windows console that is a UnicodeEncodeError from print(), which
aborts the sweep AFTER every Steam request has been paid for. GitHub runners
are UTF-8 so this only bites locally -- which is exactly where the script is
developed.
"""
for stream in (sys.stdout, sys.stderr):
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, OSError):
pass
def _row_anchors(results_html):
"""(appid, inner HTML) of each search row: an `<a ...>` whose tag carries
data-ds-appid (the last one, if twice), up to the next `</a>`.
One forward pass. It reads exactly what
`<a[^>]*data-ds-appid="([^"]+)"[^>]*>(.*?)</a>` did, without that regex's
backtracking (SonarCloud python:S8786): a tag ends at its first `>`, and
the scan goes on after a row's `</a>`, or after the tag of an anchor that
is not a row.
"""
pos = 0
while True:
start = results_html.find("<a", pos)
if start < 0:
return
close = results_html.find(">", start)
if close < 0:
return
appids = _RE_APPID.findall(results_html, start, close)
if not appids:
pos = close + 1
continue
end = results_html.find("</a>", close + 1)
if end < 0:
return
yield appids[-1], results_html[close + 1:end]
pos = end + len("</a>")
def parse_search_rows(results_html):
"""[(appid, name, released_text)] from one search page.
Kept local rather than in core/scraper.py: that module is imported by
fetcher.py, which drives every data-WRITING script, so a parsing bug there
would break refetch_all/update_data/ingest_new. Here the blast radius is
discovery only.
"""
rows = []
for appid, inner in _row_anchors(results_html):
if "," in appid or not appid.isdigit():
continue # bundle/package id
title_m = _RE_TITLE.search(inner)
rel_m = _RE_RELEASED.search(inner)
# Steam HTML-escapes titles: "Flip &amp; Shop" must not reach a review
# screen (or a repo file) with the entity intact.
name = html.unescape(_RE_TAGS.sub("", title_m.group(1)).strip()) if title_m else ""
released = _RE_TAGS.sub("", rel_m.group(1)).strip() if rel_m else ""
rows.append((appid, name, released))
if not rows:
# Fallback for a markup change: at least recover the appids so the sweep
# degrades to "fetch details for everything" rather than silently
# finding nothing.
rows = [(a, "", "") for a in _RE_APPID.findall(results_html)
if a.isdigit()]
return rows
def appdetails_languages(text):
"""The language names in appdetails' `supported_languages`, in its order.
Steam sends one HTML string, named in the request's language (English
here): "English<strong>*</strong>, Japanese<br><strong>*</strong>languages
with full audio support". The asterisks mark full audio and the line break
starts that footnote. A game that declares no language has no key at all:
an empty list, which is not the same thing as "no English".
"""
if not isinstance(text, str):
return []
first = _RE_BR.split(text, maxsplit=1)[0]
plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "")
return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES]
def language_note(languages):
"""What a dry run says about a candidate's languages: nothing for a game
that lists English."""
if "English" in languages:
return ""
if not languages:
return "no languages declared"
more = ", ..." if len(languages) > 3 else ""
return "no English: " + ", ".join(languages[:3]) + more
def without_english(candidates):
"""How many candidates do not list English, for the run's summary."""
return sum(1 for c in candidates if "English" not in c.payload["languages"])
def catalogue_appids():
"""Appids already in data/, plus the ones the maintainer removed.
A game in data/blocklist.jsonl was taken out on purpose; proposing it again
would only put it back in the review queue. The Worker refuses it too (its
removal recorded a rejected decision), so this is the second fence, and the
one that holds even when D1 and Git disagree.
"""
out = set(blocked_appids())
for rec in load_main():
aid = extract_appid(rec.get("link", ""))
if aid:
out.add(aid)
return out
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Surface Access redirects instead of silently following them.
urlopen follows the 302 to the Access login page and hands back HTTP 200
text/html, which reads as "the endpoint returned the wrong content type"
when the real fact is "Access refused the credential". Stopping at the
redirect keeps the actual failure visible.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
_OPENER = urllib.request.build_opener(_NoRedirect)
def _describe_access_redirect(location):
"""Explain an Access login redirect, using its own meta JWT."""
host = urllib.parse.urlparse(location).hostname or ""
# With the dot: "evilcloudflareaccess.com" also ends in "cloudflareaccess.com",
# and would have been explained as Access refusing the service token.
if not host.endswith(".cloudflareaccess.com"):
return f"unexpected redirect to {location[:120]}"
# The login URL carries a `meta` JWT whose claims say what Access decided.
# service_token_status is the one that matters: false means the service
# token was not accepted for this application.
detail = ""
try:
meta = urllib.parse.parse_qs(urllib.parse.urlparse(location).query).get("meta", [""])[0]
payload = meta.split(".")[1]
payload += "=" * (-len(payload) % 4)
claims = json.loads(base64.urlsafe_b64decode(payload))
detail = (f" [aud={str(claims.get('aud'))[:12]}... "
f"service_token_status={claims.get('service_token_status')} "
f"auth_status={claims.get('auth_status')}]")
except Exception:
pass
return chr(10).join([
"Cloudflare Access redirected to its LOGIN PAGE instead of accepting "
"the service token" + detail + ".",
" The application covering this path exists, but its policy does not "
"admit service tokens.",
" Fix: give that application a policy with Action = 'Service Auth' and "
"a 'Service Token' rule naming the token.",
" An 'Allow' policy carrying only an Emails rule serves the "
"interactive login page to a service token - exactly this symptom.",
])
def worker_get(base, path, token_id, token_secret, timeout=30):
req = urllib.request.Request(
base.rstrip("/") + path,
headers={
"CF-Access-Client-Id": token_id,
"CF-Access-Client-Secret": token_secret,
"Accept": "application/json",
"User-Agent": "f2p-discover",
},
)
try:
resp = _OPENER.open(req, timeout=timeout)
except urllib.error.HTTPError as e:
if e.code in (301, 302, 303, 307, 308):
raise RuntimeError(_describe_access_redirect(e.headers.get("Location", "")))
raise
with resp:
ctype = resp.headers.get("Content-Type", "")
raw = resp.read()
if "application/json" not in ctype:
raise RuntimeError(
f"expected JSON from {path}, got {ctype or 'no content-type'} "
f"(HTTP {resp.status}) - is the Access service token configured?"
)
return json.loads(raw)
def worker_post(base, path, payload, token_id, token_secret, timeout=60):
body = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
base.rstrip("/") + path,
data=body,
headers={
"CF-Access-Client-Id": token_id,
"CF-Access-Client-Secret": token_secret,
"Content-Type": "application/json",
"Accept": "application/json",
"User-Agent": "f2p-discover",
},
method="POST",
)
with urllib.request.urlopen(req, timeout=timeout) as resp:
return json.loads(resp.read())
def enrich(client, appid):
"""(Candidate, None) or (None, reason). One appdetails call."""
_, data = client.fetch_app_details_full(appid)
if not data:
return None, "no_data"
if (data.get("type") or "") not in ALLOWED_TYPES:
return None, f"type_{data.get('type') or 'unknown'}"
if not data.get("is_free"):
return None, "not_free"
if (data.get("release_date") or {}).get("coming_soon"):
return None, "coming_soon"
return Candidate(appid, data), None
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--mode", choices=["daily"], default="daily")
ap.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
ap.add_argument("--stop-after-known-pages", type=int, default=DEFAULT_STOP_AFTER_KNOWN)
ap.add_argument("--limit", type=int, default=0,
help="stop after this many candidates (0 = no limit)")
ap.add_argument("--dry-run", action="store_true",
help="do everything except POST to the Worker")
args = ap.parse_args()
safe_stdout()
base = os.environ.get("WORKER_BASE_URL", "").strip()
tid = os.environ.get("CF_ACCESS_CLIENT_ID", "").strip()
tsec = os.environ.get("CF_ACCESS_CLIENT_SECRET", "").strip()
if not args.dry_run and not (base and tid and tsec):
print("ERROR: WORKER_BASE_URL, CF_ACCESS_CLIENT_ID and "
"CF_ACCESS_CLIENT_SECRET are required (or use --dry-run)")
return 2
known = catalogue_appids()
print(f"catalogue: {len(known)} appids")
# Talk to the Worker whenever credentials exist, INCLUDING under --dry-run.
# Both calls are read-only, so the dry run stays a dry run -- and a dry run
# that skipped them was verifying nothing: it exercised Steam and the
# parser, never the Access service token, which is the part that is
# actually new and actually breaks. docs/ADMIN.md offers `-f dry_run=true`
# as the post-setup check, so it has to be one.
have_creds = bool(base and tid and tsec)
if have_creds:
# Prove the credential BEFORE spending Steam requests.
try:
ping = worker_get(base, "/api/ingest/ping", tid, tsec)
print(f"worker: reachable as {ping.get('actor')}")
except Exception as e:
print(f"ERROR: worker ping failed: {e}")
return 2
try:
wk = worker_get(base, "/api/ingest/known", tid, tsec)
queued = set(wk.get("appids") or [])
print(f"worker: {len(queued)} appid(s) already queued or decided")
# Counting these as known is what lets the stop rule fire; without
# it yesterday's pending rows look new and every run walks to
# --max-pages.
known |= queued
except Exception as e:
print(f"ERROR: /api/ingest/known failed: {e}")
return 2
else:
print("worker: no credentials in env, skipping reachability check")
client = get_client()
candidates, rejected, seen = [], {}, set()
known_pages = 0
try:
for page in range(args.max_pages):
start = page * PAGE_SIZE
body = client.fetch_search_page(start, PAGE_SIZE)
if not body:
print(f"page {page}: search failed, stopping")
break
rows = parse_search_rows(body.get("results_html", ""))
if not rows:
print(f"page {page}: no rows, stopping")
break
unknown = []
for appid, name, released in rows:
if appid in seen:
continue # paging drift: a mid-sweep release shifts rows down
seen.add(appid)
if appid in known:
continue
if released.lower().startswith("coming"):
# Cheap local skip. The search text lags appdetails, so this
# is a this-run skip only, never a durable rejection.
rejected["coming_soon_search"] = rejected.get("coming_soon_search", 0) + 1
continue
unknown.append(appid)
print(f"page {page}: {len(rows)} rows, {len(unknown)} unknown")
if not unknown:
known_pages += 1
if known_pages >= args.stop_after_known_pages:
print(f"stopping: {known_pages} consecutive pages fully known")
break
continue
known_pages = 0
for appid in unknown:
cand, reason = enrich(client, appid)
if reason:
rejected[reason] = rejected.get(reason, 0) + 1
continue
candidates.append(cand)
if args.limit and len(candidates) >= args.limit:
print(f"stopping: hit --limit {args.limit}")
break
if args.limit and len(candidates) >= args.limit:
break
finally:
client.close()
print(f"\ncandidates: {len(candidates)}")
if candidates:
print(f" of which without English: {without_english(candidates)}")
if rejected:
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
if args.dry_run:
for c in candidates[:20]:
note = language_note(c.payload["languages"])
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
if len(candidates) > 20:
print(f" ... and {len(candidates) - 20} more")
return 0
if not candidates:
print("nothing to post")
return 0
posted = 0
for i in range(0, len(candidates), POST_BATCH):
chunk = candidates[i:i + POST_BATCH]
try:
res = worker_post(
base, "/api/ingest/candidates",
{"source": "discover", "candidates": [c.to_wire() for c in chunk]},
tid, tsec,
)
except urllib.error.HTTPError as e:
print(f"ERROR: POST failed {e.code}: {e.read()[:300]!r}")
return 1
posted += res.get("accepted", 0)
for r in (res.get("rejected") or [])[:10]:
print(f" worker rejected {r.get('appid')}: {r.get('reason')}")
print(f"posted: {posted} accepted")
return 0
if __name__ == "__main__":
sys.exit(main())