feat(discovery): propose free games in every language

The store search pinned supportedlang=english, which left out every free
game that does not list English: 1,592 of 18,790 on 2026-10-30, most of
them Chinese, Japanese or Korean only (5118700 among them). Without the
parameter Steam filters on no language at all, with l=english or an
Accept-Language: ja header alike.

- fetch_search_page() sends no supportedlang, and sends l=english so the
  row text ("Coming soon", month names) stays English by parameter.
- Candidates carry their languages (appdetails_languages()), a dry run
  says which list no English, and every run counts them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-30 17:28:10 +07:00
1 parent 60b710a646
commit 6f6eb3a71f
5 files changed
+140 -15

No files matched your search

+12 -7
View File
@@ -23,10 +23,12 @@ appid. That is the reason for windowing at all -- ~12,000 unknown appids at the
store throttle is several hours, far past a job timeout. One quarter is a
sitting most of the way through.
Language filter: fetch_search_page pins supportedlang=english, which is known to
exclude ~1,537 titles. That is deliberately left as-is so this sweep and the
daily one agree on what the catalogue is supposed to contain; widening it is a
separate decision, not a backfill detail.
Languages: since v4.6.0 fetch_search_page lists free games in every language,
for this sweep and the daily one alike. Until then it pinned
supportedlang=english, which left out 1,592 titles (2026-10-30), most of them
Chinese, Japanese or Korean only - so every window swept before v4.6.0 needs
one more pass to find them. Each candidate carries its languages, and a dry
run says which ones list no English.
Usage:
python scripts/backfill_discover.py --year 2025 --quarter 3 --dry-run
@@ -41,8 +43,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.steam_client import get_client
from discover_new import (
PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, parse_search_rows,
safe_stdout, worker_get, worker_post,
PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, language_note,
parse_search_rows, safe_stdout, without_english, worker_get, worker_post,
)
MONTHS = {m: i for i, m in enumerate(
@@ -242,12 +244,15 @@ def main():
client.close()
print(f"\ncandidates: {len(candidates)}")
if candidates:
print(f" of which without English: {without_english(candidates)}")
if rejected:
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
if args.dry_run:
for c in candidates[:20]:
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
note = language_note(c.payload["languages"])
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
if len(candidates) > 20:
print(f" ... and {len(candidates) - 20} more")
return 0
+14 -3
View File
@@ -314,8 +314,8 @@ class SteamClient:
return self._tag_names
def fetch_search_page(self, start: int, count: int = 100) -> Optional[dict]:
"""Store search: free games, newest first. Returns the parsed JSON
({'success', 'results_html', 'total_count', 'start'}) or None.
"""Store search: free games in every language, newest first. Returns the
parsed JSON ({'success', 'results_html', 'total_count', 'start'}) or None.
This is the ONLY discovery-capable method on this client - every other
one needs an appid you already have. It deliberately shares
@@ -325,6 +325,17 @@ class SteamClient:
Measured: the server caps page size at 100 regardless of `count`, and a
`start` past the end returns zero rows rather than an error, so a paging
loop terminates naturally.
No `supportedlang`. Until v4.6.0 it was pinned to english, which left out
every game that does not list English: 1,592 of 18,790 free games on
2026-10-30, most of them Chinese, Japanese or Korean only. Without it
Steam filters on no language at all (the same 18,790 with `l=english`
and with an `Accept-Language: ja` header).
`l=english` sets the language of the row TEXT, never which rows come
back: discovery skips a row whose release reads "Coming soon", and the
backfill reads English month names. Before, only the session's
Accept-Language header kept that text English.
"""
resp = self._get(
"https://store.steampowered.com/search/results/",
@@ -334,9 +345,9 @@ class SteamClient:
"count": count,
"maxprice": "free",
"category1": 998, # "Games" - excludes DLC/soundtracks/videos/software
"supportedlang": "english",
"sort_by": "Released_DESC",
"infinite": 1,
"l": "english",
},
throttle_fn=self._throttle_store,
timeout=20,
+47 -4
View File
@@ -32,13 +32,17 @@ from core.steam_client import get_client
# Steam caps the search page at 100 rows regardless of what we ask for.
PAGE_SIZE = 100
# A page spans roughly 11 days of releases at the current rate (~8-9/day), so
# five pages is about eight weeks of insurance. If the cron silently stops for a
# month, the next successful run still catches everything with no intervention.
# A page spans about a week of releases (13 a day in October 2026, every
# language), so five pages is over a month of insurance. If the cron silently
# stops for a few weeks, the next successful run still catches everything with
# no intervention.
DEFAULT_MAX_PAGES = 5
DEFAULT_STOP_AFTER_KNOWN = 2
# The Worker's own cap; batches larger than this are rejected.
POST_BATCH = 100
# Steam knows about 30 languages; the cap only bounds what a publisher's
# string can put into the payload.
MAX_LANGUAGES = 40
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
@@ -48,6 +52,7 @@ _RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
# "Coming soon" skip below silently never fired.
_RE_RELEASED = re.compile(r'<div class="search_released[^"]*">(.*?)</div>', re.S)
_RE_TAGS = re.compile(r"<[^>]+>")
_RE_BR = re.compile(r"<br\s*/?>", re.I)
# Types that are not games. `category1=998` already filters most of these out,
# but appdetails is the authoritative source and occasionally disagrees.
@@ -73,6 +78,9 @@ class Candidate:
"platforms": data.get("platforms") or {},
"developers": (data.get("developers") or [])[:6],
"publishers": (data.get("publishers") or [])[:6],
# Discovery proposes games in every language; this is what tells a
# reviewer which ones a reader of English cannot play.
"languages": appdetails_languages(data.get("supported_languages")),
}
def to_wire(self):
@@ -162,6 +170,38 @@ def parse_search_rows(results_html):
return rows
def appdetails_languages(text):
"""The language names in appdetails' `supported_languages`, in its order.
Steam sends one HTML string, named in the request's language (English
here): "English<strong>*</strong>, Japanese<br><strong>*</strong>languages
with full audio support". The asterisks mark full audio and the line break
starts that footnote. A game that declares no language has no key at all:
an empty list, which is not the same thing as "no English".
"""
if not isinstance(text, str):
return []
first = _RE_BR.split(text, maxsplit=1)[0]
plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "")
return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES]
def language_note(languages):
"""What a dry run says about a candidate's languages: nothing for a game
that lists English."""
if "English" in languages:
return ""
if not languages:
return "no languages declared"
more = ", ..." if len(languages) > 3 else ""
return "no English: " + ", ".join(languages[:3]) + more
def without_english(candidates):
"""How many candidates do not list English, for the run's summary."""
return sum(1 for c in candidates if "English" not in c.payload["languages"])
def catalogue_appids():
"""Appids already in data/, plus the ones the maintainer removed.
@@ -395,12 +435,15 @@ def main():
client.close()
print(f"\ncandidates: {len(candidates)}")
if candidates:
print(f" of which without English: {without_english(candidates)}")
if rejected:
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
if args.dry_run:
for c in candidates[:20]:
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
note = language_note(c.payload["languages"])
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
if len(candidates) > 20:
print(f" ... and {len(candidates) - 20} more")
return 0
+51 -1
View File
@@ -1,4 +1,7 @@
"""How discover_new reads the redirect Cloudflare Access answers with."""
"""How discover_new reads the redirect Cloudflare Access answers with, the
search rows, and the languages a candidate carries to the review queue."""
import pytest
import discover_new
@@ -72,3 +75,50 @@ def test_appids_outside_anchors_are_still_recovered():
def test_a_page_cut_off_inside_a_tag_keeps_the_rows_before_it():
html = '<a data-ds-appid="570"><span class="title">Dota 2</span></a><a data-ds-appid="730"'
assert discover_new.parse_search_rows(html) == [("570", "Dota 2", "")]
@pytest.mark.parametrize("text, expected", [
# 5118700 as appdetails answered it on 2026-10-30.
("Japanese", ["Japanese"]),
("English<strong>*</strong>, Simplified Chinese<strong>*</strong>, Japanese"
"<br><strong>*</strong>languages with full audio support",
["English", "Simplified Chinese", "Japanese"]),
("Korean<br/><strong>*</strong>languages with full audio support", ["Korean"]),
("Spanish - Spain, Portuguese - Brazil<BR />footnote", ["Spanish - Spain", "Portuguese - Brazil"]),
("Fran&ccedil;ais , , Deutsch", ["Français", "Deutsch"]),
("", []),
(None, []),
(["English"], []), # not the shape Steam sends
])
def test_appdetails_languages(text, expected):
assert discover_new.appdetails_languages(text) == expected
def test_appdetails_languages_are_capped():
text = ", ".join(f"Lang{i}" for i in range(60))
assert len(discover_new.appdetails_languages(text)) == discover_new.MAX_LANGUAGES
def test_a_candidate_carries_its_languages():
data = {"name": "//トロイメライ・ノクターン", "type": "game", "is_free": True,
"supported_languages": "Japanese", "release_date": {"date": "Oct 16, 2026"}}
cand = discover_new.Candidate("5118700", data)
assert cand.link == "https://store.steampowered.com/app/5118700/"
assert cand.to_wire()["payload"]["languages"] == ["Japanese"]
assert discover_new.Candidate("1", {}).payload["languages"] == []
@pytest.mark.parametrize("languages, note", [
(["English", "Japanese"], ""),
(["Japanese"], "no English: Japanese"),
(["Korean", "Japanese", "Thai", "Russian"], "no English: Korean, Japanese, Thai, ..."),
([], "no languages declared"),
])
def test_language_note(languages, note):
assert discover_new.language_note(languages) == note
def test_candidates_without_english_are_counted():
cands = [discover_new.Candidate(str(i), {"supported_languages": s})
for i, s in enumerate(["English", "Japanese", "", "English, Korean"])]
assert discover_new.without_english(cands) == 2
+16
View File
@@ -92,6 +92,22 @@ def test_appdetails_asks_for_english():
assert session.requests[0]["cookies"] is None # age cookies: store pages only
def test_search_asks_for_every_language_and_english_text():
body = {"success": 1, "results_html": "", "total_count": 0, "start": 0}
client, session = _client(_Resp("https://store.steampowered.com/search/results/", body=body))
assert client.fetch_search_page(0) == body
[req] = session.requests
assert "supportedlang" not in req["params"] # games in every language
assert req["params"]["l"] == "english" # "Coming soon" and month names
assert req["params"]["maxprice"] == "free" and req["params"]["category1"] == 998
assert req["cookies"] is None # age cookies: store pages only
def test_a_failed_search_page_is_none():
client, _ = _client(_Resp("https://store.steampowered.com/search/results/", body={"success": 0}))
assert client.fetch_search_page(0) is None
def test_store_items_are_keyed_by_the_id_asked_for():
body = {"response": {"store_items": [
{"id": 2279510, "appid": 2279510, "success": 1, "tags": []},