feat(discovery): propose free games in every language
The store search pinned supportedlang=english, which left out every free
game that does not list English: 1,592 of 18,790 on 2026-10-30, most of
them Chinese, Japanese or Korean only (5118700 among them). Without the
parameter Steam filters on no language at all, with l=english or an
Accept-Language: ja header alike.
- fetch_search_page() sends no supportedlang, and sends l=english so the
row text ("Coming soon", month names) stays English by parameter.
- Candidates carry their languages (appdetails_languages()), a dry run
says which list no English, and every run counts them.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
60b710a646
commit
6f6eb3a71f
5 files changed
+140
-15
No files matched your search
@@ -23,10 +23,12 @@ appid. That is the reason for windowing at all -- ~12,000 unknown appids at the
|
||||
store throttle is several hours, far past a job timeout. One quarter is a
|
||||
sitting most of the way through.
|
||||
|
||||
Language filter: fetch_search_page pins supportedlang=english, which is known to
|
||||
exclude ~1,537 titles. That is deliberately left as-is so this sweep and the
|
||||
daily one agree on what the catalogue is supposed to contain; widening it is a
|
||||
separate decision, not a backfill detail.
|
||||
Languages: since v4.6.0 fetch_search_page lists free games in every language,
|
||||
for this sweep and the daily one alike. Until then it pinned
|
||||
supportedlang=english, which left out 1,592 titles (2026-10-30), most of them
|
||||
Chinese, Japanese or Korean only - so every window swept before v4.6.0 needs
|
||||
one more pass to find them. Each candidate carries its languages, and a dry
|
||||
run says which ones list no English.
|
||||
|
||||
Usage:
|
||||
python scripts/backfill_discover.py --year 2025 --quarter 3 --dry-run
|
||||
@@ -41,8 +43,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.steam_client import get_client
|
||||
from discover_new import (
|
||||
PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, parse_search_rows,
|
||||
safe_stdout, worker_get, worker_post,
|
||||
PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, language_note,
|
||||
parse_search_rows, safe_stdout, without_english, worker_get, worker_post,
|
||||
)
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(
|
||||
@@ -242,12 +244,15 @@ def main():
|
||||
client.close()
|
||||
|
||||
print(f"\ncandidates: {len(candidates)}")
|
||||
if candidates:
|
||||
print(f" of which without English: {without_english(candidates)}")
|
||||
if rejected:
|
||||
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
|
||||
|
||||
if args.dry_run:
|
||||
for c in candidates[:20]:
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
|
||||
note = language_note(c.payload["languages"])
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
|
||||
if len(candidates) > 20:
|
||||
print(f" ... and {len(candidates) - 20} more")
|
||||
return 0
|
||||
|
||||
@@ -314,8 +314,8 @@ class SteamClient:
|
||||
return self._tag_names
|
||||
|
||||
def fetch_search_page(self, start: int, count: int = 100) -> Optional[dict]:
|
||||
"""Store search: free games, newest first. Returns the parsed JSON
|
||||
({'success', 'results_html', 'total_count', 'start'}) or None.
|
||||
"""Store search: free games in every language, newest first. Returns the
|
||||
parsed JSON ({'success', 'results_html', 'total_count', 'start'}) or None.
|
||||
|
||||
This is the ONLY discovery-capable method on this client - every other
|
||||
one needs an appid you already have. It deliberately shares
|
||||
@@ -325,6 +325,17 @@ class SteamClient:
|
||||
Measured: the server caps page size at 100 regardless of `count`, and a
|
||||
`start` past the end returns zero rows rather than an error, so a paging
|
||||
loop terminates naturally.
|
||||
|
||||
No `supportedlang`. Until v4.6.0 it was pinned to english, which left out
|
||||
every game that does not list English: 1,592 of 18,790 free games on
|
||||
2026-10-30, most of them Chinese, Japanese or Korean only. Without it
|
||||
Steam filters on no language at all (the same 18,790 with `l=english`
|
||||
and with an `Accept-Language: ja` header).
|
||||
|
||||
`l=english` sets the language of the row TEXT, never which rows come
|
||||
back: discovery skips a row whose release reads "Coming soon", and the
|
||||
backfill reads English month names. Before, only the session's
|
||||
Accept-Language header kept that text English.
|
||||
"""
|
||||
resp = self._get(
|
||||
"https://store.steampowered.com/search/results/",
|
||||
@@ -334,9 +345,9 @@ class SteamClient:
|
||||
"count": count,
|
||||
"maxprice": "free",
|
||||
"category1": 998, # "Games" - excludes DLC/soundtracks/videos/software
|
||||
"supportedlang": "english",
|
||||
"sort_by": "Released_DESC",
|
||||
"infinite": 1,
|
||||
"l": "english",
|
||||
},
|
||||
throttle_fn=self._throttle_store,
|
||||
timeout=20,
|
||||
|
||||
+47
-4
@@ -32,13 +32,17 @@ from core.steam_client import get_client
|
||||
|
||||
# Steam caps the search page at 100 rows regardless of what we ask for.
|
||||
PAGE_SIZE = 100
|
||||
# A page spans roughly 11 days of releases at the current rate (~8-9/day), so
|
||||
# five pages is about eight weeks of insurance. If the cron silently stops for a
|
||||
# month, the next successful run still catches everything with no intervention.
|
||||
# A page spans about a week of releases (13 a day in October 2026, every
|
||||
# language), so five pages is over a month of insurance. If the cron silently
|
||||
# stops for a few weeks, the next successful run still catches everything with
|
||||
# no intervention.
|
||||
DEFAULT_MAX_PAGES = 5
|
||||
DEFAULT_STOP_AFTER_KNOWN = 2
|
||||
# The Worker's own cap; batches larger than this are rejected.
|
||||
POST_BATCH = 100
|
||||
# Steam knows about 30 languages; the cap only bounds what a publisher's
|
||||
# string can put into the payload.
|
||||
MAX_LANGUAGES = 40
|
||||
|
||||
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
|
||||
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
|
||||
@@ -48,6 +52,7 @@ _RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
|
||||
# "Coming soon" skip below silently never fired.
|
||||
_RE_RELEASED = re.compile(r'<div class="search_released[^"]*">(.*?)</div>', re.S)
|
||||
_RE_TAGS = re.compile(r"<[^>]+>")
|
||||
_RE_BR = re.compile(r"<br\s*/?>", re.I)
|
||||
|
||||
# Types that are not games. `category1=998` already filters most of these out,
|
||||
# but appdetails is the authoritative source and occasionally disagrees.
|
||||
@@ -73,6 +78,9 @@ class Candidate:
|
||||
"platforms": data.get("platforms") or {},
|
||||
"developers": (data.get("developers") or [])[:6],
|
||||
"publishers": (data.get("publishers") or [])[:6],
|
||||
# Discovery proposes games in every language; this is what tells a
|
||||
# reviewer which ones a reader of English cannot play.
|
||||
"languages": appdetails_languages(data.get("supported_languages")),
|
||||
}
|
||||
|
||||
def to_wire(self):
|
||||
@@ -162,6 +170,38 @@ def parse_search_rows(results_html):
|
||||
return rows
|
||||
|
||||
|
||||
def appdetails_languages(text):
|
||||
"""The language names in appdetails' `supported_languages`, in its order.
|
||||
|
||||
Steam sends one HTML string, named in the request's language (English
|
||||
here): "English<strong>*</strong>, Japanese<br><strong>*</strong>languages
|
||||
with full audio support". The asterisks mark full audio and the line break
|
||||
starts that footnote. A game that declares no language has no key at all:
|
||||
an empty list, which is not the same thing as "no English".
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
return []
|
||||
first = _RE_BR.split(text, maxsplit=1)[0]
|
||||
plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "")
|
||||
return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES]
|
||||
|
||||
|
||||
def language_note(languages):
|
||||
"""What a dry run says about a candidate's languages: nothing for a game
|
||||
that lists English."""
|
||||
if "English" in languages:
|
||||
return ""
|
||||
if not languages:
|
||||
return "no languages declared"
|
||||
more = ", ..." if len(languages) > 3 else ""
|
||||
return "no English: " + ", ".join(languages[:3]) + more
|
||||
|
||||
|
||||
def without_english(candidates):
|
||||
"""How many candidates do not list English, for the run's summary."""
|
||||
return sum(1 for c in candidates if "English" not in c.payload["languages"])
|
||||
|
||||
|
||||
def catalogue_appids():
|
||||
"""Appids already in data/, plus the ones the maintainer removed.
|
||||
|
||||
@@ -395,12 +435,15 @@ def main():
|
||||
client.close()
|
||||
|
||||
print(f"\ncandidates: {len(candidates)}")
|
||||
if candidates:
|
||||
print(f" of which without English: {without_english(candidates)}")
|
||||
if rejected:
|
||||
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
|
||||
|
||||
if args.dry_run:
|
||||
for c in candidates[:20]:
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
|
||||
note = language_note(c.payload["languages"])
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
|
||||
if len(candidates) > 20:
|
||||
print(f" ... and {len(candidates) - 20} more")
|
||||
return 0
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
"""How discover_new reads the redirect Cloudflare Access answers with."""
|
||||
"""How discover_new reads the redirect Cloudflare Access answers with, the
|
||||
search rows, and the languages a candidate carries to the review queue."""
|
||||
import pytest
|
||||
|
||||
import discover_new
|
||||
|
||||
|
||||
@@ -72,3 +75,50 @@ def test_appids_outside_anchors_are_still_recovered():
|
||||
def test_a_page_cut_off_inside_a_tag_keeps_the_rows_before_it():
|
||||
html = '<a data-ds-appid="570"><span class="title">Dota 2</span></a><a data-ds-appid="730"'
|
||||
assert discover_new.parse_search_rows(html) == [("570", "Dota 2", "")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
# 5118700 as appdetails answered it on 2026-10-30.
|
||||
("Japanese", ["Japanese"]),
|
||||
("English<strong>*</strong>, Simplified Chinese<strong>*</strong>, Japanese"
|
||||
"<br><strong>*</strong>languages with full audio support",
|
||||
["English", "Simplified Chinese", "Japanese"]),
|
||||
("Korean<br/><strong>*</strong>languages with full audio support", ["Korean"]),
|
||||
("Spanish - Spain, Portuguese - Brazil<BR />footnote", ["Spanish - Spain", "Portuguese - Brazil"]),
|
||||
("Français , , Deutsch", ["Français", "Deutsch"]),
|
||||
("", []),
|
||||
(None, []),
|
||||
(["English"], []), # not the shape Steam sends
|
||||
])
|
||||
def test_appdetails_languages(text, expected):
|
||||
assert discover_new.appdetails_languages(text) == expected
|
||||
|
||||
|
||||
def test_appdetails_languages_are_capped():
|
||||
text = ", ".join(f"Lang{i}" for i in range(60))
|
||||
assert len(discover_new.appdetails_languages(text)) == discover_new.MAX_LANGUAGES
|
||||
|
||||
|
||||
def test_a_candidate_carries_its_languages():
|
||||
data = {"name": "//トロイメライ・ノクターン", "type": "game", "is_free": True,
|
||||
"supported_languages": "Japanese", "release_date": {"date": "Oct 16, 2026"}}
|
||||
cand = discover_new.Candidate("5118700", data)
|
||||
assert cand.link == "https://store.steampowered.com/app/5118700/"
|
||||
assert cand.to_wire()["payload"]["languages"] == ["Japanese"]
|
||||
assert discover_new.Candidate("1", {}).payload["languages"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("languages, note", [
|
||||
(["English", "Japanese"], ""),
|
||||
(["Japanese"], "no English: Japanese"),
|
||||
(["Korean", "Japanese", "Thai", "Russian"], "no English: Korean, Japanese, Thai, ..."),
|
||||
([], "no languages declared"),
|
||||
])
|
||||
def test_language_note(languages, note):
|
||||
assert discover_new.language_note(languages) == note
|
||||
|
||||
|
||||
def test_candidates_without_english_are_counted():
|
||||
cands = [discover_new.Candidate(str(i), {"supported_languages": s})
|
||||
for i, s in enumerate(["English", "Japanese", "", "English, Korean"])]
|
||||
assert discover_new.without_english(cands) == 2
|
||||
@@ -92,6 +92,22 @@ def test_appdetails_asks_for_english():
|
||||
assert session.requests[0]["cookies"] is None # age cookies: store pages only
|
||||
|
||||
|
||||
def test_search_asks_for_every_language_and_english_text():
|
||||
body = {"success": 1, "results_html": "", "total_count": 0, "start": 0}
|
||||
client, session = _client(_Resp("https://store.steampowered.com/search/results/", body=body))
|
||||
assert client.fetch_search_page(0) == body
|
||||
[req] = session.requests
|
||||
assert "supportedlang" not in req["params"] # games in every language
|
||||
assert req["params"]["l"] == "english" # "Coming soon" and month names
|
||||
assert req["params"]["maxprice"] == "free" and req["params"]["category1"] == 998
|
||||
assert req["cookies"] is None # age cookies: store pages only
|
||||
|
||||
|
||||
def test_a_failed_search_page_is_none():
|
||||
client, _ = _client(_Resp("https://store.steampowered.com/search/results/", body={"success": 0}))
|
||||
assert client.fetch_search_page(0) is None
|
||||
|
||||
|
||||
def test_store_items_are_keyed_by_the_id_asked_for():
|
||||
body = {"response": {"store_items": [
|
||||
{"id": 2279510, "appid": 2279510, "success": 1, "tags": []},
|
||||
|
||||
Reference in new issue
Block a user