diff --git a/scripts/backfill_discover.py b/scripts/backfill_discover.py
index fe14be72..24aca8a0 100644
--- a/scripts/backfill_discover.py
+++ b/scripts/backfill_discover.py
@@ -23,10 +23,12 @@ appid. That is the reason for windowing at all -- ~12,000 unknown appids at the
store throttle is several hours, far past a job timeout. One quarter is a
sitting most of the way through.
-Language filter: fetch_search_page pins supportedlang=english, which is known to
-exclude ~1,537 titles. That is deliberately left as-is so this sweep and the
-daily one agree on what the catalogue is supposed to contain; widening it is a
-separate decision, not a backfill detail.
+Languages: since v4.6.0 fetch_search_page lists free games in every language,
+for this sweep and the daily one alike. Until then it pinned
+supportedlang=english, which left out 1,592 titles (2026-10-30), most of them
+Chinese, Japanese or Korean only - so every window swept before v4.6.0 needs
+one more pass to find them. Each candidate carries its languages, and a dry
+run says which ones list no English.
Usage:
python scripts/backfill_discover.py --year 2025 --quarter 3 --dry-run
@@ -41,8 +43,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.steam_client import get_client
from discover_new import (
- PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, parse_search_rows,
- safe_stdout, worker_get, worker_post,
+ PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, language_note,
+ parse_search_rows, safe_stdout, without_english, worker_get, worker_post,
)
MONTHS = {m: i for i, m in enumerate(
@@ -242,12 +244,15 @@ def main():
client.close()
print(f"\ncandidates: {len(candidates)}")
+ if candidates:
+ print(f" of which without English: {without_english(candidates)}")
if rejected:
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
if args.dry_run:
for c in candidates[:20]:
- print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
+ note = language_note(c.payload["languages"])
+ print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
if len(candidates) > 20:
print(f" ... and {len(candidates) - 20} more")
return 0
diff --git a/scripts/core/steam_client.py b/scripts/core/steam_client.py
index f110903e..0c00bef0 100644
--- a/scripts/core/steam_client.py
+++ b/scripts/core/steam_client.py
@@ -314,8 +314,8 @@ class SteamClient:
return self._tag_names
def fetch_search_page(self, start: int, count: int = 100) -> Optional[dict]:
- """Store search: free games, newest first. Returns the parsed JSON
- ({'success', 'results_html', 'total_count', 'start'}) or None.
+ """Store search: free games in every language, newest first. Returns the
+ parsed JSON ({'success', 'results_html', 'total_count', 'start'}) or None.
This is the ONLY discovery-capable method on this client - every other
one needs an appid you already have. It deliberately shares
@@ -325,6 +325,17 @@ class SteamClient:
Measured: the server caps page size at 100 regardless of `count`, and a
`start` past the end returns zero rows rather than an error, so a paging
loop terminates naturally.
+
+ No `supportedlang`. Until v4.6.0 it was pinned to english, which left out
+ every game that does not list English: 1,592 of 18,790 free games on
+ 2026-10-30, most of them Chinese, Japanese or Korean only. Without it
+ Steam filters on no language at all (the same 18,790 with `l=english`
+ and with an `Accept-Language: ja` header).
+
+ `l=english` sets the language of the row TEXT, never which rows come
+ back: discovery skips a row whose release reads "Coming soon", and the
+ backfill reads English month names. Before, only the session's
+ Accept-Language header kept that text English.
"""
resp = self._get(
"https://store.steampowered.com/search/results/",
@@ -334,9 +345,9 @@ class SteamClient:
"count": count,
"maxprice": "free",
"category1": 998, # "Games" - excludes DLC/soundtracks/videos/software
- "supportedlang": "english",
"sort_by": "Released_DESC",
"infinite": 1,
+ "l": "english",
},
throttle_fn=self._throttle_store,
timeout=20,
diff --git a/scripts/discover_new.py b/scripts/discover_new.py
index 4a2a6143..5b399407 100644
--- a/scripts/discover_new.py
+++ b/scripts/discover_new.py
@@ -32,13 +32,17 @@ from core.steam_client import get_client
# Steam caps the search page at 100 rows regardless of what we ask for.
PAGE_SIZE = 100
-# A page spans roughly 11 days of releases at the current rate (~8-9/day), so
-# five pages is about eight weeks of insurance. If the cron silently stops for a
-# month, the next successful run still catches everything with no intervention.
+# A page spans about a week of releases (13 a day in October 2026, every
+# language), so five pages is over a month of insurance. If the cron silently
+# stops for a few weeks, the next successful run still catches everything with
+# no intervention.
DEFAULT_MAX_PAGES = 5
DEFAULT_STOP_AFTER_KNOWN = 2
# The Worker's own cap; batches larger than this are rejected.
POST_BATCH = 100
+# Steam knows about 30 languages; the cap only bounds what a publisher's
+# string can put into the payload.
+MAX_LANGUAGES = 40
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
_RE_TITLE = re.compile(r'(.*?)', re.S)
@@ -48,6 +52,7 @@ _RE_TITLE = re.compile(r'(.*?)', re.S)
# "Coming soon" skip below silently never fired.
_RE_RELEASED = re.compile(r'
(.*?)
', re.S)
_RE_TAGS = re.compile(r"<[^>]+>")
+_RE_BR = re.compile(r"
", re.I)
# Types that are not games. `category1=998` already filters most of these out,
# but appdetails is the authoritative source and occasionally disagrees.
@@ -73,6 +78,9 @@ class Candidate:
"platforms": data.get("platforms") or {},
"developers": (data.get("developers") or [])[:6],
"publishers": (data.get("publishers") or [])[:6],
+ # Discovery proposes games in every language; this is what tells a
+ # reviewer which ones a reader of English cannot play.
+ "languages": appdetails_languages(data.get("supported_languages")),
}
def to_wire(self):
@@ -162,6 +170,38 @@ def parse_search_rows(results_html):
return rows
+def appdetails_languages(text):
+ """The language names in appdetails' `supported_languages`, in its order.
+
+ Steam sends one HTML string, named in the request's language (English
+ here): "English*, Japanese
*languages
+ with full audio support". The asterisks mark full audio and the line break
+ starts that footnote. A game that declares no language has no key at all:
+ an empty list, which is not the same thing as "no English".
+ """
+ if not isinstance(text, str):
+ return []
+ first = _RE_BR.split(text, maxsplit=1)[0]
+ plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "")
+ return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES]
+
+
+def language_note(languages):
+ """What a dry run says about a candidate's languages: nothing for a game
+ that lists English."""
+ if "English" in languages:
+ return ""
+ if not languages:
+ return "no languages declared"
+ more = ", ..." if len(languages) > 3 else ""
+ return "no English: " + ", ".join(languages[:3]) + more
+
+
+def without_english(candidates):
+ """How many candidates do not list English, for the run's summary."""
+ return sum(1 for c in candidates if "English" not in c.payload["languages"])
+
+
def catalogue_appids():
"""Appids already in data/, plus the ones the maintainer removed.
@@ -395,12 +435,15 @@ def main():
client.close()
print(f"\ncandidates: {len(candidates)}")
+ if candidates:
+ print(f" of which without English: {without_english(candidates)}")
if rejected:
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
if args.dry_run:
for c in candidates[:20]:
- print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
+ note = language_note(c.payload["languages"])
+ print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
if len(candidates) > 20:
print(f" ... and {len(candidates) - 20} more")
return 0
diff --git a/scripts/tests/test_discover_new.py b/scripts/tests/test_discover_new.py
index f89af71f..b18f3660 100644
--- a/scripts/tests/test_discover_new.py
+++ b/scripts/tests/test_discover_new.py
@@ -1,4 +1,7 @@
-"""How discover_new reads the redirect Cloudflare Access answers with."""
+"""How discover_new reads the redirect Cloudflare Access answers with, the
+search rows, and the languages a candidate carries to the review queue."""
+import pytest
+
import discover_new
@@ -72,3 +75,50 @@ def test_appids_outside_anchors_are_still_recovered():
def test_a_page_cut_off_inside_a_tag_keeps_the_rows_before_it():
html = 'Dota 2*, Simplified Chinese*, Japanese"
+ "
*languages with full audio support",
+ ["English", "Simplified Chinese", "Japanese"]),
+ ("Korean
*languages with full audio support", ["Korean"]),
+ ("Spanish - Spain, Portuguese - Brazil
footnote", ["Spanish - Spain", "Portuguese - Brazil"]),
+ ("Français , , Deutsch", ["Français", "Deutsch"]),
+ ("", []),
+ (None, []),
+ (["English"], []), # not the shape Steam sends
+])
+def test_appdetails_languages(text, expected):
+ assert discover_new.appdetails_languages(text) == expected
+
+
+def test_appdetails_languages_are_capped():
+ text = ", ".join(f"Lang{i}" for i in range(60))
+ assert len(discover_new.appdetails_languages(text)) == discover_new.MAX_LANGUAGES
+
+
+def test_a_candidate_carries_its_languages():
+ data = {"name": "//トロイメライ・ノクターン", "type": "game", "is_free": True,
+ "supported_languages": "Japanese", "release_date": {"date": "Oct 16, 2026"}}
+ cand = discover_new.Candidate("5118700", data)
+ assert cand.link == "https://store.steampowered.com/app/5118700/"
+ assert cand.to_wire()["payload"]["languages"] == ["Japanese"]
+ assert discover_new.Candidate("1", {}).payload["languages"] == []
+
+
+@pytest.mark.parametrize("languages, note", [
+ (["English", "Japanese"], ""),
+ (["Japanese"], "no English: Japanese"),
+ (["Korean", "Japanese", "Thai", "Russian"], "no English: Korean, Japanese, Thai, ..."),
+ ([], "no languages declared"),
+])
+def test_language_note(languages, note):
+ assert discover_new.language_note(languages) == note
+
+
+def test_candidates_without_english_are_counted():
+ cands = [discover_new.Candidate(str(i), {"supported_languages": s})
+ for i, s in enumerate(["English", "Japanese", "", "English, Korean"])]
+ assert discover_new.without_english(cands) == 2
diff --git a/scripts/tests/test_steam_client.py b/scripts/tests/test_steam_client.py
index 0f0da14e..2a39802c 100644
--- a/scripts/tests/test_steam_client.py
+++ b/scripts/tests/test_steam_client.py
@@ -92,6 +92,22 @@ def test_appdetails_asks_for_english():
assert session.requests[0]["cookies"] is None # age cookies: store pages only
+def test_search_asks_for_every_language_and_english_text():
+ body = {"success": 1, "results_html": "", "total_count": 0, "start": 0}
+ client, session = _client(_Resp("https://store.steampowered.com/search/results/", body=body))
+ assert client.fetch_search_page(0) == body
+ [req] = session.requests
+ assert "supportedlang" not in req["params"] # games in every language
+ assert req["params"]["l"] == "english" # "Coming soon" and month names
+ assert req["params"]["maxprice"] == "free" and req["params"]["category1"] == 998
+ assert req["cookies"] is None # age cookies: store pages only
+
+
+def test_a_failed_search_page_is_none():
+ client, _ = _client(_Resp("https://store.steampowered.com/search/results/", body={"success": 0}))
+ assert client.fetch_search_page(0) is None
+
+
def test_store_items_are_keyed_by_the_id_asked_for():
body = {"response": {"store_items": [
{"id": 2279510, "appid": 2279510, "success": 1, "tags": []},