diff --git a/scripts/backfill_discover.py b/scripts/backfill_discover.py index fe14be72..24aca8a0 100644 --- a/scripts/backfill_discover.py +++ b/scripts/backfill_discover.py @@ -23,10 +23,12 @@ appid. That is the reason for windowing at all -- ~12,000 unknown appids at the store throttle is several hours, far past a job timeout. One quarter is a sitting most of the way through. -Language filter: fetch_search_page pins supportedlang=english, which is known to -exclude ~1,537 titles. That is deliberately left as-is so this sweep and the -daily one agree on what the catalogue is supposed to contain; widening it is a -separate decision, not a backfill detail. +Languages: since v4.6.0 fetch_search_page lists free games in every language, +for this sweep and the daily one alike. Until then it pinned +supportedlang=english, which left out 1,592 titles (2026-10-30), most of them +Chinese, Japanese or Korean only - so every window swept before v4.6.0 needs +one more pass to find them. Each candidate carries its languages, and a dry +run says which ones list no English. Usage: python scripts/backfill_discover.py --year 2025 --quarter 3 --dry-run @@ -41,8 +43,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from core.steam_client import get_client from discover_new import ( - PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, parse_search_rows, - safe_stdout, worker_get, worker_post, + PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, language_note, + parse_search_rows, safe_stdout, without_english, worker_get, worker_post, ) MONTHS = {m: i for i, m in enumerate( @@ -242,12 +244,15 @@ def main(): client.close() print(f"\ncandidates: {len(candidates)}") + if candidates: + print(f" of which without English: {without_english(candidates)}") if rejected: print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items()))) if args.dry_run: for c in candidates[:20]: - print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}") + note = language_note(c.payload["languages"]) + print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip()) if len(candidates) > 20: print(f" ... and {len(candidates) - 20} more") return 0 diff --git a/scripts/core/steam_client.py b/scripts/core/steam_client.py index f110903e..0c00bef0 100644 --- a/scripts/core/steam_client.py +++ b/scripts/core/steam_client.py @@ -314,8 +314,8 @@ class SteamClient: return self._tag_names def fetch_search_page(self, start: int, count: int = 100) -> Optional[dict]: - """Store search: free games, newest first. Returns the parsed JSON - ({'success', 'results_html', 'total_count', 'start'}) or None. + """Store search: free games in every language, newest first. Returns the + parsed JSON ({'success', 'results_html', 'total_count', 'start'}) or None. This is the ONLY discovery-capable method on this client - every other one needs an appid you already have. It deliberately shares @@ -325,6 +325,17 @@ class SteamClient: Measured: the server caps page size at 100 regardless of `count`, and a `start` past the end returns zero rows rather than an error, so a paging loop terminates naturally. + + No `supportedlang`. Until v4.6.0 it was pinned to english, which left out + every game that does not list English: 1,592 of 18,790 free games on + 2026-10-30, most of them Chinese, Japanese or Korean only. Without it + Steam filters on no language at all (the same 18,790 with `l=english` + and with an `Accept-Language: ja` header). + + `l=english` sets the language of the row TEXT, never which rows come + back: discovery skips a row whose release reads "Coming soon", and the + backfill reads English month names. Before, only the session's + Accept-Language header kept that text English. """ resp = self._get( "https://store.steampowered.com/search/results/", @@ -334,9 +345,9 @@ class SteamClient: "count": count, "maxprice": "free", "category1": 998, # "Games" - excludes DLC/soundtracks/videos/software - "supportedlang": "english", "sort_by": "Released_DESC", "infinite": 1, + "l": "english", }, throttle_fn=self._throttle_store, timeout=20, diff --git a/scripts/discover_new.py b/scripts/discover_new.py index 4a2a6143..5b399407 100644 --- a/scripts/discover_new.py +++ b/scripts/discover_new.py @@ -32,13 +32,17 @@ from core.steam_client import get_client # Steam caps the search page at 100 rows regardless of what we ask for. PAGE_SIZE = 100 -# A page spans roughly 11 days of releases at the current rate (~8-9/day), so -# five pages is about eight weeks of insurance. If the cron silently stops for a -# month, the next successful run still catches everything with no intervention. +# A page spans about a week of releases (13 a day in October 2026, every +# language), so five pages is over a month of insurance. If the cron silently +# stops for a few weeks, the next successful run still catches everything with +# no intervention. DEFAULT_MAX_PAGES = 5 DEFAULT_STOP_AFTER_KNOWN = 2 # The Worker's own cap; batches larger than this are rejected. POST_BATCH = 100 +# Steam knows about 30 languages; the cap only bounds what a publisher's +# string can put into the payload. +MAX_LANGUAGES = 40 _RE_APPID = re.compile(r'data-ds-appid="([^"]+)"') _RE_TITLE = re.compile(r'(.*?)', re.S) @@ -48,6 +52,7 @@ _RE_TITLE = re.compile(r'(.*?)', re.S) # "Coming soon" skip below silently never fired. _RE_RELEASED = re.compile(r'
(.*?)
', re.S) _RE_TAGS = re.compile(r"<[^>]+>") +_RE_BR = re.compile(r"", re.I) # Types that are not games. `category1=998` already filters most of these out, # but appdetails is the authoritative source and occasionally disagrees. @@ -73,6 +78,9 @@ class Candidate: "platforms": data.get("platforms") or {}, "developers": (data.get("developers") or [])[:6], "publishers": (data.get("publishers") or [])[:6], + # Discovery proposes games in every language; this is what tells a + # reviewer which ones a reader of English cannot play. + "languages": appdetails_languages(data.get("supported_languages")), } def to_wire(self): @@ -162,6 +170,38 @@ def parse_search_rows(results_html): return rows +def appdetails_languages(text): + """The language names in appdetails' `supported_languages`, in its order. + + Steam sends one HTML string, named in the request's language (English + here): "English*, Japanese
*languages + with full audio support". The asterisks mark full audio and the line break + starts that footnote. A game that declares no language has no key at all: + an empty list, which is not the same thing as "no English". + """ + if not isinstance(text, str): + return [] + first = _RE_BR.split(text, maxsplit=1)[0] + plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "") + return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES] + + +def language_note(languages): + """What a dry run says about a candidate's languages: nothing for a game + that lists English.""" + if "English" in languages: + return "" + if not languages: + return "no languages declared" + more = ", ..." if len(languages) > 3 else "" + return "no English: " + ", ".join(languages[:3]) + more + + +def without_english(candidates): + """How many candidates do not list English, for the run's summary.""" + return sum(1 for c in candidates if "English" not in c.payload["languages"]) + + def catalogue_appids(): """Appids already in data/, plus the ones the maintainer removed. @@ -395,12 +435,15 @@ def main(): client.close() print(f"\ncandidates: {len(candidates)}") + if candidates: + print(f" of which without English: {without_english(candidates)}") if rejected: print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items()))) if args.dry_run: for c in candidates[:20]: - print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}") + note = language_note(c.payload["languages"]) + print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip()) if len(candidates) > 20: print(f" ... and {len(candidates) - 20} more") return 0 diff --git a/scripts/tests/test_discover_new.py b/scripts/tests/test_discover_new.py index f89af71f..b18f3660 100644 --- a/scripts/tests/test_discover_new.py +++ b/scripts/tests/test_discover_new.py @@ -1,4 +1,7 @@ -"""How discover_new reads the redirect Cloudflare Access answers with.""" +"""How discover_new reads the redirect Cloudflare Access answers with, the +search rows, and the languages a candidate carries to the review queue.""" +import pytest + import discover_new @@ -72,3 +75,50 @@ def test_appids_outside_anchors_are_still_recovered(): def test_a_page_cut_off_inside_a_tag_keeps_the_rows_before_it(): html = 'Dota 2*, Simplified Chinese*, Japanese" + "
*languages with full audio support", + ["English", "Simplified Chinese", "Japanese"]), + ("Korean
*languages with full audio support", ["Korean"]), + ("Spanish - Spain, Portuguese - Brazil
footnote", ["Spanish - Spain", "Portuguese - Brazil"]), + ("Français , , Deutsch", ["Français", "Deutsch"]), + ("", []), + (None, []), + (["English"], []), # not the shape Steam sends +]) +def test_appdetails_languages(text, expected): + assert discover_new.appdetails_languages(text) == expected + + +def test_appdetails_languages_are_capped(): + text = ", ".join(f"Lang{i}" for i in range(60)) + assert len(discover_new.appdetails_languages(text)) == discover_new.MAX_LANGUAGES + + +def test_a_candidate_carries_its_languages(): + data = {"name": "//トロイメライ・ノクターン", "type": "game", "is_free": True, + "supported_languages": "Japanese", "release_date": {"date": "Oct 16, 2026"}} + cand = discover_new.Candidate("5118700", data) + assert cand.link == "https://store.steampowered.com/app/5118700/" + assert cand.to_wire()["payload"]["languages"] == ["Japanese"] + assert discover_new.Candidate("1", {}).payload["languages"] == [] + + +@pytest.mark.parametrize("languages, note", [ + (["English", "Japanese"], ""), + (["Japanese"], "no English: Japanese"), + (["Korean", "Japanese", "Thai", "Russian"], "no English: Korean, Japanese, Thai, ..."), + ([], "no languages declared"), +]) +def test_language_note(languages, note): + assert discover_new.language_note(languages) == note + + +def test_candidates_without_english_are_counted(): + cands = [discover_new.Candidate(str(i), {"supported_languages": s}) + for i, s in enumerate(["English", "Japanese", "", "English, Korean"])] + assert discover_new.without_english(cands) == 2 diff --git a/scripts/tests/test_steam_client.py b/scripts/tests/test_steam_client.py index 0f0da14e..2a39802c 100644 --- a/scripts/tests/test_steam_client.py +++ b/scripts/tests/test_steam_client.py @@ -92,6 +92,22 @@ def test_appdetails_asks_for_english(): assert session.requests[0]["cookies"] is None # age cookies: store pages only +def test_search_asks_for_every_language_and_english_text(): + body = {"success": 1, "results_html": "", "total_count": 0, "start": 0} + client, session = _client(_Resp("https://store.steampowered.com/search/results/", body=body)) + assert client.fetch_search_page(0) == body + [req] = session.requests + assert "supportedlang" not in req["params"] # games in every language + assert req["params"]["l"] == "english" # "Coming soon" and month names + assert req["params"]["maxprice"] == "free" and req["params"]["category1"] == 998 + assert req["cookies"] is None # age cookies: store pages only + + +def test_a_failed_search_page_is_none(): + client, _ = _client(_Resp("https://store.steampowered.com/search/results/", body={"success": 0})) + assert client.fetch_search_page(0) is None + + def test_store_items_are_keyed_by_the_id_asked_for(): body = {"response": {"store_items": [ {"id": 2279510, "appid": 2279510, "success": 1, "tags": []},