The store search pinned supportedlang=english, which left out every free
game that does not list English: 1,592 of 18,790 on 2026-10-30, most of
them Chinese, Japanese or Korean only (5118700 among them). Without the
parameter Steam filters on no language at all, with l=english or an
Accept-Language: ja header alike.
- fetch_search_page() sends no supportedlang, and sends l=english so the
row text ("Coming soon", month names) stays English by parameter.
- Candidates carry their languages (appdetails_languages()), a dry run
says which list no English, and every run counts them.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
476 lines
18 KiB
Python
476 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Discover newly-released free-to-play games and propose them for review.
|
|
|
|
This script PROPOSES ONLY. It never writes to data/, never commits, and cannot
|
|
add a game to the catalogue: it POSTs candidates to the admin Worker, which
|
|
stores them as `pending` for a human to approve. That separation is the point —
|
|
discovery is automated, publication is not.
|
|
|
|
Deliberately NOT imported: anything that writes the dataset. See the CI guard in
|
|
.github/workflows/discover-new.yml.
|
|
|
|
Usage:
|
|
python scripts/discover_new.py --mode daily
|
|
python scripts/discover_new.py --mode daily --dry-run
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import html
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from core.blocklist import blocked_appids
|
|
from core.data_store import load_main, extract_appid
|
|
from core.steam_client import get_client
|
|
|
|
# Steam caps the search page at 100 rows regardless of what we ask for.
|
|
PAGE_SIZE = 100
|
|
# A page spans about a week of releases (13 a day in October 2026, every
|
|
# language), so five pages is over a month of insurance. If the cron silently
|
|
# stops for a few weeks, the next successful run still catches everything with
|
|
# no intervention.
|
|
DEFAULT_MAX_PAGES = 5
|
|
DEFAULT_STOP_AFTER_KNOWN = 2
|
|
# The Worker's own cap; batches larger than this are rejected.
|
|
POST_BATCH = 100
|
|
# Steam knows about 30 languages; the cap only bounds what a publisher's
|
|
# string can put into the payload.
|
|
MAX_LANGUAGES = 40
|
|
|
|
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
|
|
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
|
|
# NOT `col search_released`: the live markup is
|
|
# `<div class="search_released responsive_secondrow">`. The `col ` prefix
|
|
# never matched, so every row came back with an empty date and the cheap
|
|
# "Coming soon" skip below silently never fired.
|
|
_RE_RELEASED = re.compile(r'<div class="search_released[^"]*">(.*?)</div>', re.S)
|
|
_RE_TAGS = re.compile(r"<[^>]+>")
|
|
_RE_BR = re.compile(r"<br\s*/?>", re.I)
|
|
|
|
# Types that are not games. `category1=998` already filters most of these out,
|
|
# but appdetails is the authoritative source and occasionally disagrees.
|
|
ALLOWED_TYPES = {"game"}
|
|
|
|
|
|
class Candidate:
|
|
__slots__ = ("appid", "link", "name", "header_image", "release_date",
|
|
"app_type", "is_free", "payload")
|
|
|
|
def __init__(self, appid, data):
|
|
self.appid = appid
|
|
self.link = f"https://store.steampowered.com/app/{appid}/"
|
|
self.name = (data.get("name") or "")[:200]
|
|
self.header_image = (data.get("header_image") or "")[:500]
|
|
rd = data.get("release_date") or {}
|
|
self.release_date = (rd.get("date") or "")[:60]
|
|
self.app_type = (data.get("type") or "")[:30]
|
|
self.is_free = bool(data.get("is_free"))
|
|
self.payload = {
|
|
"short_description": (data.get("short_description") or "")[:600],
|
|
"genres": [g.get("description") for g in (data.get("genres") or [])][:12],
|
|
"platforms": data.get("platforms") or {},
|
|
"developers": (data.get("developers") or [])[:6],
|
|
"publishers": (data.get("publishers") or [])[:6],
|
|
# Discovery proposes games in every language; this is what tells a
|
|
# reviewer which ones a reader of English cannot play.
|
|
"languages": appdetails_languages(data.get("supported_languages")),
|
|
}
|
|
|
|
def to_wire(self):
|
|
return {
|
|
"appid": self.appid,
|
|
"link": self.link,
|
|
"name": self.name,
|
|
"header_image": self.header_image,
|
|
"release_date": self.release_date,
|
|
"app_type": self.app_type,
|
|
"is_free": self.is_free,
|
|
"health_status": "ok",
|
|
"payload": self.payload,
|
|
}
|
|
|
|
|
|
def safe_stdout():
|
|
"""Stop a non-Latin game name from killing the run on a cp1252 console.
|
|
|
|
Game names are publisher-controlled and routinely contain CJK, Cyrillic or
|
|
emoji. On a Windows console that is a UnicodeEncodeError from print(), which
|
|
aborts the sweep AFTER every Steam request has been paid for. GitHub runners
|
|
are UTF-8 so this only bites locally -- which is exactly where the script is
|
|
developed.
|
|
"""
|
|
for stream in (sys.stdout, sys.stderr):
|
|
try:
|
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
except (AttributeError, OSError):
|
|
pass
|
|
|
|
|
|
def _row_anchors(results_html):
|
|
"""(appid, inner HTML) of each search row: an `<a ...>` whose tag carries
|
|
data-ds-appid (the last one, if twice), up to the next `</a>`.
|
|
|
|
One forward pass. It reads exactly what
|
|
`<a[^>]*data-ds-appid="([^"]+)"[^>]*>(.*?)</a>` did, without that regex's
|
|
backtracking (SonarCloud python:S8786): a tag ends at its first `>`, and
|
|
the scan goes on after a row's `</a>`, or after the tag of an anchor that
|
|
is not a row.
|
|
"""
|
|
pos = 0
|
|
while True:
|
|
start = results_html.find("<a", pos)
|
|
if start < 0:
|
|
return
|
|
close = results_html.find(">", start)
|
|
if close < 0:
|
|
return
|
|
appids = _RE_APPID.findall(results_html, start, close)
|
|
if not appids:
|
|
pos = close + 1
|
|
continue
|
|
end = results_html.find("</a>", close + 1)
|
|
if end < 0:
|
|
return
|
|
yield appids[-1], results_html[close + 1:end]
|
|
pos = end + len("</a>")
|
|
|
|
|
|
def parse_search_rows(results_html):
|
|
"""[(appid, name, released_text)] from one search page.
|
|
|
|
Kept local rather than in core/scraper.py: that module is imported by
|
|
fetcher.py, which drives every data-WRITING script, so a parsing bug there
|
|
would break refetch_all/update_data/ingest_new. Here the blast radius is
|
|
discovery only.
|
|
"""
|
|
rows = []
|
|
for appid, inner in _row_anchors(results_html):
|
|
if "," in appid or not appid.isdigit():
|
|
continue # bundle/package id
|
|
title_m = _RE_TITLE.search(inner)
|
|
rel_m = _RE_RELEASED.search(inner)
|
|
# Steam HTML-escapes titles: "Flip & Shop" must not reach a review
|
|
# screen (or a repo file) with the entity intact.
|
|
name = html.unescape(_RE_TAGS.sub("", title_m.group(1)).strip()) if title_m else ""
|
|
released = _RE_TAGS.sub("", rel_m.group(1)).strip() if rel_m else ""
|
|
rows.append((appid, name, released))
|
|
if not rows:
|
|
# Fallback for a markup change: at least recover the appids so the sweep
|
|
# degrades to "fetch details for everything" rather than silently
|
|
# finding nothing.
|
|
rows = [(a, "", "") for a in _RE_APPID.findall(results_html)
|
|
if a.isdigit()]
|
|
return rows
|
|
|
|
|
|
def appdetails_languages(text):
|
|
"""The language names in appdetails' `supported_languages`, in its order.
|
|
|
|
Steam sends one HTML string, named in the request's language (English
|
|
here): "English<strong>*</strong>, Japanese<br><strong>*</strong>languages
|
|
with full audio support". The asterisks mark full audio and the line break
|
|
starts that footnote. A game that declares no language has no key at all:
|
|
an empty list, which is not the same thing as "no English".
|
|
"""
|
|
if not isinstance(text, str):
|
|
return []
|
|
first = _RE_BR.split(text, maxsplit=1)[0]
|
|
plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "")
|
|
return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES]
|
|
|
|
|
|
def language_note(languages):
|
|
"""What a dry run says about a candidate's languages: nothing for a game
|
|
that lists English."""
|
|
if "English" in languages:
|
|
return ""
|
|
if not languages:
|
|
return "no languages declared"
|
|
more = ", ..." if len(languages) > 3 else ""
|
|
return "no English: " + ", ".join(languages[:3]) + more
|
|
|
|
|
|
def without_english(candidates):
|
|
"""How many candidates do not list English, for the run's summary."""
|
|
return sum(1 for c in candidates if "English" not in c.payload["languages"])
|
|
|
|
|
|
def catalogue_appids():
|
|
"""Appids already in data/, plus the ones the maintainer removed.
|
|
|
|
A game in data/blocklist.jsonl was taken out on purpose; proposing it again
|
|
would only put it back in the review queue. The Worker refuses it too (its
|
|
removal recorded a rejected decision), so this is the second fence, and the
|
|
one that holds even when D1 and Git disagree.
|
|
"""
|
|
out = set(blocked_appids())
|
|
for rec in load_main():
|
|
aid = extract_appid(rec.get("link", ""))
|
|
if aid:
|
|
out.add(aid)
|
|
return out
|
|
|
|
|
|
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
"""Surface Access redirects instead of silently following them.
|
|
|
|
urlopen follows the 302 to the Access login page and hands back HTTP 200
|
|
text/html, which reads as "the endpoint returned the wrong content type"
|
|
when the real fact is "Access refused the credential". Stopping at the
|
|
redirect keeps the actual failure visible.
|
|
"""
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
_OPENER = urllib.request.build_opener(_NoRedirect)
|
|
|
|
|
|
def _describe_access_redirect(location):
|
|
"""Explain an Access login redirect, using its own meta JWT."""
|
|
host = urllib.parse.urlparse(location).hostname or ""
|
|
# With the dot: "evilcloudflareaccess.com" also ends in "cloudflareaccess.com",
|
|
# and would have been explained as Access refusing the service token.
|
|
if not host.endswith(".cloudflareaccess.com"):
|
|
return f"unexpected redirect to {location[:120]}"
|
|
|
|
# The login URL carries a `meta` JWT whose claims say what Access decided.
|
|
# service_token_status is the one that matters: false means the service
|
|
# token was not accepted for this application.
|
|
detail = ""
|
|
try:
|
|
meta = urllib.parse.parse_qs(urllib.parse.urlparse(location).query).get("meta", [""])[0]
|
|
payload = meta.split(".")[1]
|
|
payload += "=" * (-len(payload) % 4)
|
|
claims = json.loads(base64.urlsafe_b64decode(payload))
|
|
detail = (f" [aud={str(claims.get('aud'))[:12]}... "
|
|
f"service_token_status={claims.get('service_token_status')} "
|
|
f"auth_status={claims.get('auth_status')}]")
|
|
except Exception:
|
|
pass
|
|
|
|
return chr(10).join([
|
|
"Cloudflare Access redirected to its LOGIN PAGE instead of accepting "
|
|
"the service token" + detail + ".",
|
|
" The application covering this path exists, but its policy does not "
|
|
"admit service tokens.",
|
|
" Fix: give that application a policy with Action = 'Service Auth' and "
|
|
"a 'Service Token' rule naming the token.",
|
|
" An 'Allow' policy carrying only an Emails rule serves the "
|
|
"interactive login page to a service token - exactly this symptom.",
|
|
])
|
|
|
|
|
|
def worker_get(base, path, token_id, token_secret, timeout=30):
|
|
req = urllib.request.Request(
|
|
base.rstrip("/") + path,
|
|
headers={
|
|
"CF-Access-Client-Id": token_id,
|
|
"CF-Access-Client-Secret": token_secret,
|
|
"Accept": "application/json",
|
|
"User-Agent": "f2p-discover",
|
|
},
|
|
)
|
|
try:
|
|
resp = _OPENER.open(req, timeout=timeout)
|
|
except urllib.error.HTTPError as e:
|
|
if e.code in (301, 302, 303, 307, 308):
|
|
raise RuntimeError(_describe_access_redirect(e.headers.get("Location", "")))
|
|
raise
|
|
with resp:
|
|
ctype = resp.headers.get("Content-Type", "")
|
|
raw = resp.read()
|
|
if "application/json" not in ctype:
|
|
raise RuntimeError(
|
|
f"expected JSON from {path}, got {ctype or 'no content-type'} "
|
|
f"(HTTP {resp.status}) - is the Access service token configured?"
|
|
)
|
|
return json.loads(raw)
|
|
|
|
|
|
def worker_post(base, path, payload, token_id, token_secret, timeout=60):
|
|
body = json.dumps(payload).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
base.rstrip("/") + path,
|
|
data=body,
|
|
headers={
|
|
"CF-Access-Client-Id": token_id,
|
|
"CF-Access-Client-Secret": token_secret,
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json",
|
|
"User-Agent": "f2p-discover",
|
|
},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return json.loads(resp.read())
|
|
|
|
|
|
def enrich(client, appid):
|
|
"""(Candidate, None) or (None, reason). One appdetails call."""
|
|
_, data = client.fetch_app_details_full(appid)
|
|
if not data:
|
|
return None, "no_data"
|
|
if (data.get("type") or "") not in ALLOWED_TYPES:
|
|
return None, f"type_{data.get('type') or 'unknown'}"
|
|
if not data.get("is_free"):
|
|
return None, "not_free"
|
|
if (data.get("release_date") or {}).get("coming_soon"):
|
|
return None, "coming_soon"
|
|
return Candidate(appid, data), None
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--mode", choices=["daily"], default="daily")
|
|
ap.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
|
|
ap.add_argument("--stop-after-known-pages", type=int, default=DEFAULT_STOP_AFTER_KNOWN)
|
|
ap.add_argument("--limit", type=int, default=0,
|
|
help="stop after this many candidates (0 = no limit)")
|
|
ap.add_argument("--dry-run", action="store_true",
|
|
help="do everything except POST to the Worker")
|
|
args = ap.parse_args()
|
|
safe_stdout()
|
|
|
|
base = os.environ.get("WORKER_BASE_URL", "").strip()
|
|
tid = os.environ.get("CF_ACCESS_CLIENT_ID", "").strip()
|
|
tsec = os.environ.get("CF_ACCESS_CLIENT_SECRET", "").strip()
|
|
if not args.dry_run and not (base and tid and tsec):
|
|
print("ERROR: WORKER_BASE_URL, CF_ACCESS_CLIENT_ID and "
|
|
"CF_ACCESS_CLIENT_SECRET are required (or use --dry-run)")
|
|
return 2
|
|
|
|
known = catalogue_appids()
|
|
print(f"catalogue: {len(known)} appids")
|
|
|
|
# Talk to the Worker whenever credentials exist, INCLUDING under --dry-run.
|
|
# Both calls are read-only, so the dry run stays a dry run -- and a dry run
|
|
# that skipped them was verifying nothing: it exercised Steam and the
|
|
# parser, never the Access service token, which is the part that is
|
|
# actually new and actually breaks. docs/ADMIN.md offers `-f dry_run=true`
|
|
# as the post-setup check, so it has to be one.
|
|
have_creds = bool(base and tid and tsec)
|
|
if have_creds:
|
|
# Prove the credential BEFORE spending Steam requests.
|
|
try:
|
|
ping = worker_get(base, "/api/ingest/ping", tid, tsec)
|
|
print(f"worker: reachable as {ping.get('actor')}")
|
|
except Exception as e:
|
|
print(f"ERROR: worker ping failed: {e}")
|
|
return 2
|
|
try:
|
|
wk = worker_get(base, "/api/ingest/known", tid, tsec)
|
|
queued = set(wk.get("appids") or [])
|
|
print(f"worker: {len(queued)} appid(s) already queued or decided")
|
|
# Counting these as known is what lets the stop rule fire; without
|
|
# it yesterday's pending rows look new and every run walks to
|
|
# --max-pages.
|
|
known |= queued
|
|
except Exception as e:
|
|
print(f"ERROR: /api/ingest/known failed: {e}")
|
|
return 2
|
|
else:
|
|
print("worker: no credentials in env, skipping reachability check")
|
|
|
|
client = get_client()
|
|
candidates, rejected, seen = [], {}, set()
|
|
known_pages = 0
|
|
|
|
try:
|
|
for page in range(args.max_pages):
|
|
start = page * PAGE_SIZE
|
|
body = client.fetch_search_page(start, PAGE_SIZE)
|
|
if not body:
|
|
print(f"page {page}: search failed, stopping")
|
|
break
|
|
rows = parse_search_rows(body.get("results_html", ""))
|
|
if not rows:
|
|
print(f"page {page}: no rows, stopping")
|
|
break
|
|
|
|
unknown = []
|
|
for appid, name, released in rows:
|
|
if appid in seen:
|
|
continue # paging drift: a mid-sweep release shifts rows down
|
|
seen.add(appid)
|
|
if appid in known:
|
|
continue
|
|
if released.lower().startswith("coming"):
|
|
# Cheap local skip. The search text lags appdetails, so this
|
|
# is a this-run skip only, never a durable rejection.
|
|
rejected["coming_soon_search"] = rejected.get("coming_soon_search", 0) + 1
|
|
continue
|
|
unknown.append(appid)
|
|
|
|
print(f"page {page}: {len(rows)} rows, {len(unknown)} unknown")
|
|
|
|
if not unknown:
|
|
known_pages += 1
|
|
if known_pages >= args.stop_after_known_pages:
|
|
print(f"stopping: {known_pages} consecutive pages fully known")
|
|
break
|
|
continue
|
|
known_pages = 0
|
|
|
|
for appid in unknown:
|
|
cand, reason = enrich(client, appid)
|
|
if reason:
|
|
rejected[reason] = rejected.get(reason, 0) + 1
|
|
continue
|
|
candidates.append(cand)
|
|
if args.limit and len(candidates) >= args.limit:
|
|
print(f"stopping: hit --limit {args.limit}")
|
|
break
|
|
if args.limit and len(candidates) >= args.limit:
|
|
break
|
|
finally:
|
|
client.close()
|
|
|
|
print(f"\ncandidates: {len(candidates)}")
|
|
if candidates:
|
|
print(f" of which without English: {without_english(candidates)}")
|
|
if rejected:
|
|
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
|
|
|
|
if args.dry_run:
|
|
for c in candidates[:20]:
|
|
note = language_note(c.payload["languages"])
|
|
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
|
|
if len(candidates) > 20:
|
|
print(f" ... and {len(candidates) - 20} more")
|
|
return 0
|
|
|
|
if not candidates:
|
|
print("nothing to post")
|
|
return 0
|
|
|
|
posted = 0
|
|
for i in range(0, len(candidates), POST_BATCH):
|
|
chunk = candidates[i:i + POST_BATCH]
|
|
try:
|
|
res = worker_post(
|
|
base, "/api/ingest/candidates",
|
|
{"source": "discover", "candidates": [c.to_wire() for c in chunk]},
|
|
tid, tsec,
|
|
)
|
|
except urllib.error.HTTPError as e:
|
|
print(f"ERROR: POST failed {e.code}: {e.read()[:300]!r}")
|
|
return 1
|
|
posted += res.get("accepted", 0)
|
|
for r in (res.get("rejected") or [])[:10]:
|
|
print(f" worker rejected {r.get('appid')}: {r.get('reason')}")
|
|
print(f"posted: {posted} accepted")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|