fix(pipeline): act on SonarCloud's findings in scripts/

- Linear parsers (python:S8786): the store page's "unsupported" language
  rows and its tags, and the discovery sweep's search rows, no longer use
  regexes that backtrack quadratically on long input. New tests, written
  against the old regexes first, pin what they read, and a differential
  fuzz (200,000 random inputs) found no case where old and new disagree.
- Raw invisible and bidi characters in normalize.py and two test files are
  written as escapes (text:S6389); every string value is unchanged.
- scrape_store_fields() states when there is no usable page (scraped = None)
  instead of deriving it into a separate flag (pythonbugs:S2259).
- cmd_list() in delete_game.py and edit_game.py returns nothing; the
  dispatch returns 0 (python:S3516). remove_games.main() keeps its int:
  delete_game.py and its tests use the exit code.
- The Steam client mounts its retrying adapter for https only
  (python:S5332); nothing is fetched over http.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-19 09:31:48 +07:00
1 parent 988e4f4ce6
commit d8991dada4
11 files changed
+167 -35

No files matched your search

+5 -4
View File
@@ -219,15 +219,16 @@ def scrape_store_fields(client, appid: str, details: Optional[dict] = None) -> O
if status in ("network_error", "not_found"):
return None
scraped = scrape_store_page(page) if status == "ok" else None
real_page = bool(scraped and (scraped["languages"] or scraped["tags"]))
if real_page and scraped["languages"] and scraped["tags"]:
if scraped is not None and not (scraped["languages"] or scraped["tags"]):
scraped = None # a page with neither tells nothing: all of it comes from the API
if scraped is not None and scraped["languages"] and scraped["tags"]:
return scraped
item = (client.fetch_store_items([appid]) or {}).get(str(appid))
if item is None:
return scraped if real_page else None
return scraped
api = parse_store_item(item, client.fetch_tag_names())
if not real_page:
if scraped is None:
api["has_paid_dlc"] = _paid_dlc(client, appid, details)
return api
if not scraped["languages"] and api["languages"]:
+6 -6
View File
@@ -133,18 +133,18 @@ def normalize_release_date(value):
# controls. "Gameforge 4D GmbH" was stored with a stray U+202C after it, and
# "Triple Fantasy" with a ZWSP in front.
_INVISIBLE = dict.fromkeys(map(ord, (
"​⁠"
"‪‫‬‭‮"
"⁦⁧⁨⁩"
"\u200b\ufeff\u2060"
"\u202a\u202b\u202c\u202d\u202e"
"\u2066\u2067\u2068\u2069"
)))
# Two or more zero-width characters in a row are a watermark, not writing: one
# developer name carried 43 of them between "YAMAYURI " and "GAMES". A SINGLE
# zero-width joiner is how emoji sequences are built, and a single zero-width
# non-joiner is spelling in Persian and the Indic scripts, so those stay.
_ZERO_WIDTH_RUN = re.compile("[​‌‍]{2,}")
_ZERO_WIDTH_RUN = re.compile("[\u200b\u200c\u200d]{2,}")
# Plain whitespace, tabs and NBSP ("Cagliari 1943\xa0VR", "ARTFX - GAME\t"), but
# not the ideographic space: "鳥獣妖怪戯画 (Choju Yokai Giga)" spells it that way.
_WHITESPACE = re.compile("[ \t\r\n\f\v ]+")
# not the ideographic space: "鳥獣妖怪戯画\u3000(Choju Yokai Giga)" spells it that way.
_WHITESPACE = re.compile("[ \t\r\n\f\v\xa0]+")
def clean_text(value, single_line: bool = True):
+13 -3
View File
@@ -22,7 +22,10 @@ _RE_TABLE_ROW = re.compile(r'(<tr[^>]*>)(.*?)</tr>', re.DOTALL | re.IGNORECASE)
_RE_TABLE_CELL = re.compile(r'<td[^>]*>(.*?)</td>', re.DOTALL | re.IGNORECASE)
_RE_HAS_CHECK = re.compile(r'<span[^>]*>[^<]*</span>')
_RE_STRIP_HTML = re.compile(r'<[^>]+>')
_RE_UNSUPPORTED = re.compile(r'class\s*=\s*"[^"]*unsupported[^"]*"', re.IGNORECASE)
# Each class="..." value of a tag; _marked_unsupported() looks inside them.
# One regex with "unsupported" between two [^"]* runs backtracked
# quadratically on a long attribute (SonarCloud python:S8786).
_RE_CLASS_VALUE = re.compile(r'class\s*=\s*"([^"]*)"', re.IGNORECASE)
_RE_DLC_SECTION_START = re.compile(
r'(?:id\s*=\s*"gameAreaDLCSection"|class\s*=\s*"game_area_dlc_section")',
@@ -39,7 +42,9 @@ _RE_DLC_PRICE_TEXT = re.compile(
)
_RE_HAS_DIGIT = re.compile(r'\d')
_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>\s*([^<]+?)\s*</a>')
# The text is taken whole and trimmed by parse_tags(), which also drops an
# empty one: `\s*([^<]+?)\s*` around it backtracked on long whitespace.
_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>([^<]*)</a>')
# Steam's own page sometimes prints a language's localisation token where its
# name belongs: TrackMania Nations Forever lists "#lang_slovakian". A token
@@ -49,6 +54,11 @@ _LANGUAGE_TOKENS = {"#lang_slovakian": "Slovak"}
# ──── Language table ────
def _marked_unsupported(tag: str) -> bool:
"""Whether any class="..." of a tag names "unsupported", in any case."""
return any("unsupported" in m.group(1).lower() for m in _RE_CLASS_VALUE.finditer(tag))
def parse_language_table(html: str) -> dict:
"""
Parse #languageTable → {languages: [...], language_details: [...]}.
@@ -76,7 +86,7 @@ def parse_language_table(html: str) -> dict:
continue
# Skip unsupported rows (class on <tr>)
if _RE_UNSUPPORTED.search(tr_tag):
if _marked_unsupported(tr_tag):
continue
# Skip rows with "not supported" text
+1 -1
View File
@@ -82,8 +82,8 @@ class SteamClient:
status_forcelist=[], allowed_methods=["GET", "HEAD"]),
pool_connections=10, pool_maxsize=10,
)
# https only: every endpoint this client calls is.
s.mount("https://", adapter)
s.mount("http://", adapter)
return s
def _throttle_store(self):
+4 -4
View File
@@ -92,16 +92,15 @@ def cmd_check(path: str = BLOCKLIST_JSONL) -> int:
return 1 if problems else 0
def cmd_list() -> int:
def cmd_list() -> None:
entries = load_blocklist()
if not entries:
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
return 0
return
for e in entries.values():
note = f" - {e['note']}" if e.get("note") else ""
print(f"{e['appid']:>10} {e['blocked_at']} {e['reason']:<13} {e.get('name', '')}{note}")
print(f"{len(entries)} blocked game(s).")
return 0
def cmd_add(targets: list[str], reason: str, note: str, apply: bool) -> int:
@@ -179,7 +178,8 @@ def main(argv=None) -> int:
if args.check:
return cmd_check()
if args.list:
return cmd_list()
cmd_list()
return 0
if not args.targets:
ap.print_help()
return 2
+30 -2
View File
@@ -41,7 +41,6 @@ DEFAULT_STOP_AFTER_KNOWN = 2
POST_BATCH = 100
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
_RE_ROW = re.compile(r'<a[^>]*data-ds-appid="([^"]+)"[^>]*>(.*?)</a>', re.S)
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
# NOT `col search_released`: the live markup is
# `<div class="search_released responsive_secondrow">`. The `col ` prefix
@@ -106,6 +105,35 @@ def safe_stdout():
pass
def _row_anchors(results_html):
"""(appid, inner HTML) of each search row: an `<a ...>` whose tag carries
data-ds-appid (the last one, if twice), up to the next `</a>`.
One forward pass. It reads exactly what
`<a[^>]*data-ds-appid="([^"]+)"[^>]*>(.*?)</a>` did, without that regex's
backtracking (SonarCloud python:S8786): a tag ends at its first `>`, and
the scan goes on after a row's `</a>`, or after the tag of an anchor that
is not a row.
"""
pos = 0
while True:
start = results_html.find("<a", pos)
if start < 0:
return
close = results_html.find(">", start)
if close < 0:
return
appids = _RE_APPID.findall(results_html, start, close)
if not appids:
pos = close + 1
continue
end = results_html.find("</a>", close + 1)
if end < 0:
return
yield appids[-1], results_html[close + 1:end]
pos = end + len("</a>")
def parse_search_rows(results_html):
"""[(appid, name, released_text)] from one search page.
@@ -115,7 +143,7 @@ def parse_search_rows(results_html):
discovery only.
"""
rows = []
for appid, inner in _RE_ROW.findall(results_html):
for appid, inner in _row_anchors(results_html):
if "," in appid or not appid.isdigit():
continue # bundle/package id
title_m = _RE_TITLE.search(inner)
+3 -3
View File
@@ -76,7 +76,7 @@ def cmd_list():
docs = load_overrides()
if not docs:
print("no overrides")
return 0
return
print(f"{len(docs)} override(s):")
for appid, doc in sorted(docs.items()):
live = doc.get("fields") or {}
@@ -85,7 +85,6 @@ def cmd_list():
print(f" {appid:>8} {name[:44]:<44} {len(live)} active, {len(gone)} retired")
for f, e in sorted(live.items()):
print(f" {f} = {e.get('value')!r} (was {e.get('was')!r})")
return 0
def cmd_check():
@@ -155,7 +154,8 @@ def main():
safe_stdout()
if args.list:
return cmd_list()
cmd_list()
return 0
if args.check:
return cmd_check()
if not args.appid:
+37
View File
@@ -30,3 +30,40 @@ def test_removed_games_count_as_known(tmp_path, monkeypatch):
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
f.write(serialize_entry(make_entry("2", "Two", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
assert discover_new.catalogue_appids() == {"1", "2"}
# Written against the row regex parse_search_rows used before it became a
# linear scan (SonarCloud python:S8786); every case is behaviour that existed.
SEARCH_HTML = """
<a href="https://store.steampowered.com/app/570/Dota_2/?snr=1_7_7_230_150_1" data-ds-appid="570" data-ds-itemkey="App_570" class="search_result_row ds_collapse_flag " data-search-page="1">
<div class="responsive_search_name_combined">
<div class="col search_name ellipsis"><span class="title">Dota 2</span></div>
<div class="search_released responsive_secondrow">Jul 9, 2013</div>
</div>
</a>
<a href="https://store.steampowered.com/sub/123/" data-ds-packageid="123" data-ds-appid="10,20" class="search_result_row"><span class="title">A Bundle</span></a>
<a href="https://store.steampowered.com/app/1/" data-ds-appid="1" class="search_result_row"><span class="title">Flip &amp; Shop</span><div class="search_released responsive_secondrow"> Coming soon </div></a>
<a href="#" class="pagination">2</a>
<a href="https://store.steampowered.com/app/2/" data-ds-appid="2"><span class="title"><b>Bold</b> Name</span></a>
<a data-ds-appid="4"><span class="title">Four</span><a href="y">inner</a> tail</a>
<a data-ds-appid="7" data-ds-appid="8"><span class="title">Twice</span></a>
<a data-ds-appid="99"><span class="title">Never closed</span>
"""
def test_search_rows_are_read_from_each_anchor():
assert discover_new.parse_search_rows(SEARCH_HTML) == [
("570", "Dota 2", "Jul 9, 2013"),
("1", "Flip & Shop", "Coming soon"),
("2", "Bold Name", ""),
("4", "Four", ""),
("8", "Twice", ""),
]
def test_appids_outside_anchors_are_still_recovered():
# A markup change that drops the anchors degrades to "fetch details for
# everything" instead of finding nothing.
assert discover_new.parse_search_rows('<div data-ds-appid="5"></div><div data-ds-appid="x"></div>') == [
("5", "", ""),
]
+10 -10
View File
@@ -82,17 +82,17 @@ def test_the_shape_is_a_fixed_point():
("Raffiné Studio\t", "Raffiné Studio"),
("Fire from the sky: Cagliari 1943\xa0VR", "Fire from the sky: Cagliari 1943 VR"),
("Stuffed Wombat", "Stuffed Wombat"),
("Gameforge 4D GmbH‬", "Gameforge 4D GmbH"),
("​Triple Fantasy", "Triple Fantasy"),
("Wilma Wiiand​​", "Wilma Wiiand"),
("YAMAYURI ‌‌‍‍‍​‍‌‌GAMES", "YAMAYURI GAMES"),
("Gameforge 4D GmbH\u202c", "Gameforge 4D GmbH"),
("\u200bTriple Fantasy", "Triple Fantasy"),
("Wilma Wiiand\u200b\u200b", "Wilma Wiiand"),
("YAMAYURI \u200c\u200c\u200d\u200d\u200d\u200b\u200d\u200c\u200cGAMES", "YAMAYURI GAMES"),
# Kept: real names in any script, an ideographic space, a single ZWNJ
# (Persian spelling) and a single ZWJ (an emoji sequence).
("千鳥豚工作室", "千鳥豚工作室"),
("鳥獣妖怪戯画 (Choju Yokai Giga)", "鳥獣妖怪戯画 (Choju Yokai Giga)"),
("می‌خواهم", "می‌خواهم"),
("鳥獣妖怪戯画\u3000(Choju Yokai Giga)", "鳥獣妖怪戯画\u3000(Choju Yokai Giga)"),
("می\u200cخواهم", "می\u200cخواهم"),
("Pocket Rocket Studio 🚀", "Pocket Rocket Studio 🚀"),
("👨‍🚀 Studio", "👨‍🚀 Studio"),
("👨\u200d🚀 Studio", "👨\u200d🚀 Studio"),
])
def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected):
assert clean_text(raw) == expected
@@ -100,7 +100,7 @@ def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected
def test_a_blurb_keeps_its_line_breaks():
assert clean_text("Let's start typing!\r\nDiscover​ more ", single_line=False) == \
assert clean_text("Let's start typing!\r\nDiscover\u200b more ", single_line=False) == \
"Let's start typing!\r\nDiscover more"
@@ -166,8 +166,8 @@ def test_a_record_only_has_its_own_fields_touched():
def _messy(appid: int) -> dict:
return {
"link": f"https://store.steampowered.com/app/{appid}/",
"name": "​Triple Fantasy ",
"developer": ["Wilma Wiiand​​"],
"name": "\u200bTriple Fantasy ",
"developer": ["Wilma Wiiand\u200b\u200b"],
"publisher": [" "],
"release_date": "28 апр. 2025 г.",
"description": "an art &quot;game&quot;",
+2 -2
View File
@@ -366,8 +366,8 @@ def test_a_release_date_is_stored_in_the_one_shape():
def test_names_are_compared_cleaned():
game = _game(developer=["Wilma Wiiand"], publisher=["Gameforge 4D GmbH"])
changed = rsd.apply_app_details(game, {
"developers": ["Wilma Wiiand​​", " "],
"publishers": ["Gameforge 4D GmbH‬"],
"developers": ["Wilma Wiiand\u200b\u200b", " "],
"publishers": ["Gameforge 4D GmbH\u202c"],
})
assert "developer" not in changed and "publisher" not in changed
+56
View File
@@ -0,0 +1,56 @@
"""How core/scraper.py reads a store page's tags and language table.
Written against the regexes these parsers used before they were made linear
(SonarCloud python:S8786), so each case pins behaviour that already existed.
"""
from core.scraper import parse_language_table, parse_tags
TAGS_HTML = """
<div class="glance_tags popular_tags" data-appid="1">
<a href="https://store.steampowered.com/tags/en/Free%20to%20Play/" class="app_tag" style="display: none;">
\t\t\t\t\t\tFree to Play\t\t\t\t\t</a><a href="x" class="app_tag" style="display: none;">
\t\t\t\t\t\tPoint &amp; Click\t\t\t\t\t</a><a class = "app_tag">action</a><a class="app_tag">Action</a>
<a class="app_tag"> </a><a class="app_tag"></a><a class="app_tag"><b>Nested</b></a>
<a class="app_tag">+</a><div class="app_tag add_button">+</div><a class="app_tag">RPG &gt; JRPG</a>
</div>
"""
def test_tags_are_unescaped_trimmed_and_deduplicated():
assert parse_tags(TAGS_HTML) == ["Free to Play", "Point & Click", "action", "RPG > JRPG"]
def test_a_page_without_tags_has_none():
assert parse_tags('<a class="app_tagger">Nope</a><span class="app_tag">Nope</span>') == []
CHECK = '<td class="checkcol"><span>&#10004;</span></td>'
EMPTY = '<td class="checkcol"></td>'
LANGUAGE_HTML = f"""
<table class="game_language_options" cellpadding="0" cellspacing="0">
<tr><th style="width: 94px;"></th><th class="checkcol">Interface</th><th class="checkcol">Full Audio</th><th class="checkcol">Subtitles</th></tr>
<tr style="" class="">
<td style="width: 94px; text-align: left" class="ellipsis">
English</td>
{CHECK}
{CHECK}
{CHECK}
</tr>
<tr class="unsupported"><td>French</td>{CHECK}{CHECK}{CHECK}</tr>
<tr CLASS="Lang Unsupported"><td>German</td>{CHECK}{EMPTY}{EMPTY}</tr>
<tr data-class="unsupported"><td>Italian</td>{CHECK}{EMPTY}{EMPTY}</tr>
<tr class="supported"><td>Spanish - Spain</td>{EMPTY}{EMPTY}{CHECK}</tr>
<tr class="unsupported"><td style="width: 94px; text-align: left" class="ellipsis">Japanese</td><td colspan="3">Not supported</td></tr>
</table>
"""
def test_a_row_marked_unsupported_is_skipped_whatever_its_cells_say():
out = parse_language_table(LANGUAGE_HTML)
assert out["languages"] == ["English", "Spanish - Spain"]
assert out["language_details"] == [
{"name": "English", "interface": True, "audio": True, "subtitles": True},
{"name": "Spanish - Spain", "interface": False, "audio": False, "subtitles": True},
]