fix(pipeline): act on SonarCloud's findings in scripts/
- Linear parsers (python:S8786): the store page's "unsupported" language rows and its tags, and the discovery sweep's search rows, no longer use regexes that backtrack quadratically on long input. New tests, written against the old regexes first, pin what they read, and a differential fuzz (200,000 random inputs) found no case where old and new disagree. - Raw invisible and bidi characters in normalize.py and two test files are written as escapes (text:S6389); every string value is unchanged. - scrape_store_fields() states when there is no usable page (scraped = None) instead of deriving it into a separate flag (pythonbugs:S2259). - cmd_list() in delete_game.py and edit_game.py returns nothing; the dispatch returns 0 (python:S3516). remove_games.main() keeps its int: delete_game.py and its tests use the exit code. - The Steam client mounts its retrying adapter for https only (python:S5332); nothing is fetched over http. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
988e4f4ce6
commit
d8991dada4
11 files changed
+167
-35
No files matched your search
@@ -219,15 +219,16 @@ def scrape_store_fields(client, appid: str, details: Optional[dict] = None) -> O
|
||||
if status in ("network_error", "not_found"):
|
||||
return None
|
||||
scraped = scrape_store_page(page) if status == "ok" else None
|
||||
real_page = bool(scraped and (scraped["languages"] or scraped["tags"]))
|
||||
if real_page and scraped["languages"] and scraped["tags"]:
|
||||
if scraped is not None and not (scraped["languages"] or scraped["tags"]):
|
||||
scraped = None # a page with neither tells nothing: all of it comes from the API
|
||||
if scraped is not None and scraped["languages"] and scraped["tags"]:
|
||||
return scraped
|
||||
|
||||
item = (client.fetch_store_items([appid]) or {}).get(str(appid))
|
||||
if item is None:
|
||||
return scraped if real_page else None
|
||||
return scraped
|
||||
api = parse_store_item(item, client.fetch_tag_names())
|
||||
if not real_page:
|
||||
if scraped is None:
|
||||
api["has_paid_dlc"] = _paid_dlc(client, appid, details)
|
||||
return api
|
||||
if not scraped["languages"] and api["languages"]:
|
||||
|
||||
@@ -133,18 +133,18 @@ def normalize_release_date(value):
|
||||
# controls. "Gameforge 4D GmbH" was stored with a stray U+202C after it, and
|
||||
# "Triple Fantasy" with a ZWSP in front.
|
||||
_INVISIBLE = dict.fromkeys(map(ord, (
|
||||
""
|
||||
""
|
||||
""
|
||||
"\u200b\ufeff\u2060"
|
||||
"\u202a\u202b\u202c\u202d\u202e"
|
||||
"\u2066\u2067\u2068\u2069"
|
||||
)))
|
||||
# Two or more zero-width characters in a row are a watermark, not writing: one
|
||||
# developer name carried 43 of them between "YAMAYURI " and "GAMES". A SINGLE
|
||||
# zero-width joiner is how emoji sequences are built, and a single zero-width
|
||||
# non-joiner is spelling in Persian and the Indic scripts, so those stay.
|
||||
_ZERO_WIDTH_RUN = re.compile("[]{2,}")
|
||||
_ZERO_WIDTH_RUN = re.compile("[\u200b\u200c\u200d]{2,}")
|
||||
# Plain whitespace, tabs and NBSP ("Cagliari 1943\xa0VR", "ARTFX - GAME\t"), but
|
||||
# not the ideographic space: "鳥獣妖怪戯画 (Choju Yokai Giga)" spells it that way.
|
||||
_WHITESPACE = re.compile("[ \t\r\n\f\v ]+")
|
||||
# not the ideographic space: "鳥獣妖怪戯画\u3000(Choju Yokai Giga)" spells it that way.
|
||||
_WHITESPACE = re.compile("[ \t\r\n\f\v\xa0]+")
|
||||
|
||||
|
||||
def clean_text(value, single_line: bool = True):
|
||||
|
||||
+13
-3
@@ -22,7 +22,10 @@ _RE_TABLE_ROW = re.compile(r'(<tr[^>]*>)(.*?)</tr>', re.DOTALL | re.IGNORECASE)
|
||||
_RE_TABLE_CELL = re.compile(r'<td[^>]*>(.*?)</td>', re.DOTALL | re.IGNORECASE)
|
||||
_RE_HAS_CHECK = re.compile(r'<span[^>]*>[^<]*</span>')
|
||||
_RE_STRIP_HTML = re.compile(r'<[^>]+>')
|
||||
_RE_UNSUPPORTED = re.compile(r'class\s*=\s*"[^"]*unsupported[^"]*"', re.IGNORECASE)
|
||||
# Each class="..." value of a tag; _marked_unsupported() looks inside them.
|
||||
# One regex with "unsupported" between two [^"]* runs backtracked
|
||||
# quadratically on a long attribute (SonarCloud python:S8786).
|
||||
_RE_CLASS_VALUE = re.compile(r'class\s*=\s*"([^"]*)"', re.IGNORECASE)
|
||||
|
||||
_RE_DLC_SECTION_START = re.compile(
|
||||
r'(?:id\s*=\s*"gameAreaDLCSection"|class\s*=\s*"game_area_dlc_section")',
|
||||
@@ -39,7 +42,9 @@ _RE_DLC_PRICE_TEXT = re.compile(
|
||||
)
|
||||
_RE_HAS_DIGIT = re.compile(r'\d')
|
||||
|
||||
_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>\s*([^<]+?)\s*</a>')
|
||||
# The text is taken whole and trimmed by parse_tags(), which also drops an
|
||||
# empty one: `\s*([^<]+?)\s*` around it backtracked on long whitespace.
|
||||
_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>([^<]*)</a>')
|
||||
|
||||
# Steam's own page sometimes prints a language's localisation token where its
|
||||
# name belongs: TrackMania Nations Forever lists "#lang_slovakian". A token
|
||||
@@ -49,6 +54,11 @@ _LANGUAGE_TOKENS = {"#lang_slovakian": "Slovak"}
|
||||
|
||||
# ──── Language table ────
|
||||
|
||||
def _marked_unsupported(tag: str) -> bool:
|
||||
"""Whether any class="..." of a tag names "unsupported", in any case."""
|
||||
return any("unsupported" in m.group(1).lower() for m in _RE_CLASS_VALUE.finditer(tag))
|
||||
|
||||
|
||||
def parse_language_table(html: str) -> dict:
|
||||
"""
|
||||
Parse #languageTable → {languages: [...], language_details: [...]}.
|
||||
@@ -76,7 +86,7 @@ def parse_language_table(html: str) -> dict:
|
||||
continue
|
||||
|
||||
# Skip unsupported rows (class on <tr>)
|
||||
if _RE_UNSUPPORTED.search(tr_tag):
|
||||
if _marked_unsupported(tr_tag):
|
||||
continue
|
||||
|
||||
# Skip rows with "not supported" text
|
||||
|
||||
@@ -82,8 +82,8 @@ class SteamClient:
|
||||
status_forcelist=[], allowed_methods=["GET", "HEAD"]),
|
||||
pool_connections=10, pool_maxsize=10,
|
||||
)
|
||||
# https only: every endpoint this client calls is.
|
||||
s.mount("https://", adapter)
|
||||
s.mount("http://", adapter)
|
||||
return s
|
||||
|
||||
def _throttle_store(self):
|
||||
|
||||
@@ -92,16 +92,15 @@ def cmd_check(path: str = BLOCKLIST_JSONL) -> int:
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
def cmd_list() -> int:
|
||||
def cmd_list() -> None:
|
||||
entries = load_blocklist()
|
||||
if not entries:
|
||||
print(f"{BLOCKLIST_JSONL}: nothing blocked.")
|
||||
return 0
|
||||
return
|
||||
for e in entries.values():
|
||||
note = f" - {e['note']}" if e.get("note") else ""
|
||||
print(f"{e['appid']:>10} {e['blocked_at']} {e['reason']:<13} {e.get('name', '')}{note}")
|
||||
print(f"{len(entries)} blocked game(s).")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_add(targets: list[str], reason: str, note: str, apply: bool) -> int:
|
||||
@@ -179,7 +178,8 @@ def main(argv=None) -> int:
|
||||
if args.check:
|
||||
return cmd_check()
|
||||
if args.list:
|
||||
return cmd_list()
|
||||
cmd_list()
|
||||
return 0
|
||||
if not args.targets:
|
||||
ap.print_help()
|
||||
return 2
|
||||
|
||||
+30
-2
@@ -41,7 +41,6 @@ DEFAULT_STOP_AFTER_KNOWN = 2
|
||||
POST_BATCH = 100
|
||||
|
||||
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
|
||||
_RE_ROW = re.compile(r'<a[^>]*data-ds-appid="([^"]+)"[^>]*>(.*?)</a>', re.S)
|
||||
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
|
||||
# NOT `col search_released`: the live markup is
|
||||
# `<div class="search_released responsive_secondrow">`. The `col ` prefix
|
||||
@@ -106,6 +105,35 @@ def safe_stdout():
|
||||
pass
|
||||
|
||||
|
||||
def _row_anchors(results_html):
|
||||
"""(appid, inner HTML) of each search row: an `<a ...>` whose tag carries
|
||||
data-ds-appid (the last one, if twice), up to the next `</a>`.
|
||||
|
||||
One forward pass. It reads exactly what
|
||||
`<a[^>]*data-ds-appid="([^"]+)"[^>]*>(.*?)</a>` did, without that regex's
|
||||
backtracking (SonarCloud python:S8786): a tag ends at its first `>`, and
|
||||
the scan goes on after a row's `</a>`, or after the tag of an anchor that
|
||||
is not a row.
|
||||
"""
|
||||
pos = 0
|
||||
while True:
|
||||
start = results_html.find("<a", pos)
|
||||
if start < 0:
|
||||
return
|
||||
close = results_html.find(">", start)
|
||||
if close < 0:
|
||||
return
|
||||
appids = _RE_APPID.findall(results_html, start, close)
|
||||
if not appids:
|
||||
pos = close + 1
|
||||
continue
|
||||
end = results_html.find("</a>", close + 1)
|
||||
if end < 0:
|
||||
return
|
||||
yield appids[-1], results_html[close + 1:end]
|
||||
pos = end + len("</a>")
|
||||
|
||||
|
||||
def parse_search_rows(results_html):
|
||||
"""[(appid, name, released_text)] from one search page.
|
||||
|
||||
@@ -115,7 +143,7 @@ def parse_search_rows(results_html):
|
||||
discovery only.
|
||||
"""
|
||||
rows = []
|
||||
for appid, inner in _RE_ROW.findall(results_html):
|
||||
for appid, inner in _row_anchors(results_html):
|
||||
if "," in appid or not appid.isdigit():
|
||||
continue # bundle/package id
|
||||
title_m = _RE_TITLE.search(inner)
|
||||
|
||||
@@ -76,7 +76,7 @@ def cmd_list():
|
||||
docs = load_overrides()
|
||||
if not docs:
|
||||
print("no overrides")
|
||||
return 0
|
||||
return
|
||||
print(f"{len(docs)} override(s):")
|
||||
for appid, doc in sorted(docs.items()):
|
||||
live = doc.get("fields") or {}
|
||||
@@ -85,7 +85,6 @@ def cmd_list():
|
||||
print(f" {appid:>8} {name[:44]:<44} {len(live)} active, {len(gone)} retired")
|
||||
for f, e in sorted(live.items()):
|
||||
print(f" {f} = {e.get('value')!r} (was {e.get('was')!r})")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_check():
|
||||
@@ -155,7 +154,8 @@ def main():
|
||||
safe_stdout()
|
||||
|
||||
if args.list:
|
||||
return cmd_list()
|
||||
cmd_list()
|
||||
return 0
|
||||
if args.check:
|
||||
return cmd_check()
|
||||
if not args.appid:
|
||||
|
||||
@@ -30,3 +30,40 @@ def test_removed_games_count_as_known(tmp_path, monkeypatch):
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(serialize_entry(make_entry("2", "Two", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
|
||||
assert discover_new.catalogue_appids() == {"1", "2"}
|
||||
|
||||
|
||||
# Written against the row regex parse_search_rows used before it became a
|
||||
# linear scan (SonarCloud python:S8786); every case is behaviour that existed.
|
||||
SEARCH_HTML = """
|
||||
<a href="https://store.steampowered.com/app/570/Dota_2/?snr=1_7_7_230_150_1" data-ds-appid="570" data-ds-itemkey="App_570" class="search_result_row ds_collapse_flag " data-search-page="1">
|
||||
<div class="responsive_search_name_combined">
|
||||
<div class="col search_name ellipsis"><span class="title">Dota 2</span></div>
|
||||
<div class="search_released responsive_secondrow">Jul 9, 2013</div>
|
||||
</div>
|
||||
</a>
|
||||
<a href="https://store.steampowered.com/sub/123/" data-ds-packageid="123" data-ds-appid="10,20" class="search_result_row"><span class="title">A Bundle</span></a>
|
||||
<a href="https://store.steampowered.com/app/1/" data-ds-appid="1" class="search_result_row"><span class="title">Flip & Shop</span><div class="search_released responsive_secondrow"> Coming soon </div></a>
|
||||
<a href="#" class="pagination">2</a>
|
||||
<a href="https://store.steampowered.com/app/2/" data-ds-appid="2"><span class="title"><b>Bold</b> Name</span></a>
|
||||
<a data-ds-appid="4"><span class="title">Four</span><a href="y">inner</a> tail</a>
|
||||
<a data-ds-appid="7" data-ds-appid="8"><span class="title">Twice</span></a>
|
||||
<a data-ds-appid="99"><span class="title">Never closed</span>
|
||||
"""
|
||||
|
||||
|
||||
def test_search_rows_are_read_from_each_anchor():
|
||||
assert discover_new.parse_search_rows(SEARCH_HTML) == [
|
||||
("570", "Dota 2", "Jul 9, 2013"),
|
||||
("1", "Flip & Shop", "Coming soon"),
|
||||
("2", "Bold Name", ""),
|
||||
("4", "Four", ""),
|
||||
("8", "Twice", ""),
|
||||
]
|
||||
|
||||
|
||||
def test_appids_outside_anchors_are_still_recovered():
|
||||
# A markup change that drops the anchors degrades to "fetch details for
|
||||
# everything" instead of finding nothing.
|
||||
assert discover_new.parse_search_rows('<div data-ds-appid="5"></div><div data-ds-appid="x"></div>') == [
|
||||
("5", "", ""),
|
||||
]
|
||||
@@ -82,17 +82,17 @@ def test_the_shape_is_a_fixed_point():
|
||||
("Raffiné Studio\t", "Raffiné Studio"),
|
||||
("Fire from the sky: Cagliari 1943\xa0VR", "Fire from the sky: Cagliari 1943 VR"),
|
||||
("Stuffed Wombat", "Stuffed Wombat"),
|
||||
("Gameforge 4D GmbH", "Gameforge 4D GmbH"),
|
||||
("Triple Fantasy", "Triple Fantasy"),
|
||||
("Wilma Wiiand", "Wilma Wiiand"),
|
||||
("YAMAYURI GAMES", "YAMAYURI GAMES"),
|
||||
("Gameforge 4D GmbH\u202c", "Gameforge 4D GmbH"),
|
||||
("\u200bTriple Fantasy", "Triple Fantasy"),
|
||||
("Wilma Wiiand\u200b\u200b", "Wilma Wiiand"),
|
||||
("YAMAYURI \u200c\u200c\u200d\u200d\u200d\u200b\u200d\u200c\u200cGAMES", "YAMAYURI GAMES"),
|
||||
# Kept: real names in any script, an ideographic space, a single ZWNJ
|
||||
# (Persian spelling) and a single ZWJ (an emoji sequence).
|
||||
("千鳥豚工作室", "千鳥豚工作室"),
|
||||
("鳥獣妖怪戯画 (Choju Yokai Giga)", "鳥獣妖怪戯画 (Choju Yokai Giga)"),
|
||||
("میخواهم", "میخواهم"),
|
||||
("鳥獣妖怪戯画\u3000(Choju Yokai Giga)", "鳥獣妖怪戯画\u3000(Choju Yokai Giga)"),
|
||||
("می\u200cخواهم", "می\u200cخواهم"),
|
||||
("Pocket Rocket Studio 🚀", "Pocket Rocket Studio 🚀"),
|
||||
("👨🚀 Studio", "👨🚀 Studio"),
|
||||
("👨\u200d🚀 Studio", "👨\u200d🚀 Studio"),
|
||||
])
|
||||
def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected):
|
||||
assert clean_text(raw) == expected
|
||||
@@ -100,7 +100,7 @@ def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected
|
||||
|
||||
|
||||
def test_a_blurb_keeps_its_line_breaks():
|
||||
assert clean_text("Let's start typing!\r\nDiscover more ", single_line=False) == \
|
||||
assert clean_text("Let's start typing!\r\nDiscover\u200b more ", single_line=False) == \
|
||||
"Let's start typing!\r\nDiscover more"
|
||||
|
||||
|
||||
@@ -166,8 +166,8 @@ def test_a_record_only_has_its_own_fields_touched():
|
||||
def _messy(appid: int) -> dict:
|
||||
return {
|
||||
"link": f"https://store.steampowered.com/app/{appid}/",
|
||||
"name": "Triple Fantasy ",
|
||||
"developer": ["Wilma Wiiand"],
|
||||
"name": "\u200bTriple Fantasy ",
|
||||
"developer": ["Wilma Wiiand\u200b\u200b"],
|
||||
"publisher": [" "],
|
||||
"release_date": "28 апр. 2025 г.",
|
||||
"description": "an art "game"",
|
||||
|
||||
@@ -366,8 +366,8 @@ def test_a_release_date_is_stored_in_the_one_shape():
|
||||
def test_names_are_compared_cleaned():
|
||||
game = _game(developer=["Wilma Wiiand"], publisher=["Gameforge 4D GmbH"])
|
||||
changed = rsd.apply_app_details(game, {
|
||||
"developers": ["Wilma Wiiand", " "],
|
||||
"publishers": ["Gameforge 4D GmbH"],
|
||||
"developers": ["Wilma Wiiand\u200b\u200b", " "],
|
||||
"publishers": ["Gameforge 4D GmbH\u202c"],
|
||||
})
|
||||
assert "developer" not in changed and "publisher" not in changed
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""How core/scraper.py reads a store page's tags and language table.
|
||||
|
||||
Written against the regexes these parsers used before they were made linear
|
||||
(SonarCloud python:S8786), so each case pins behaviour that already existed.
|
||||
"""
|
||||
from core.scraper import parse_language_table, parse_tags
|
||||
|
||||
|
||||
TAGS_HTML = """
|
||||
<div class="glance_tags popular_tags" data-appid="1">
|
||||
<a href="https://store.steampowered.com/tags/en/Free%20to%20Play/" class="app_tag" style="display: none;">
|
||||
\t\t\t\t\t\tFree to Play\t\t\t\t\t</a><a href="x" class="app_tag" style="display: none;">
|
||||
\t\t\t\t\t\tPoint & Click\t\t\t\t\t</a><a class = "app_tag">action</a><a class="app_tag">Action</a>
|
||||
<a class="app_tag"> </a><a class="app_tag"></a><a class="app_tag"><b>Nested</b></a>
|
||||
<a class="app_tag">+</a><div class="app_tag add_button">+</div><a class="app_tag">RPG > JRPG</a>
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
def test_tags_are_unescaped_trimmed_and_deduplicated():
|
||||
assert parse_tags(TAGS_HTML) == ["Free to Play", "Point & Click", "action", "RPG > JRPG"]
|
||||
|
||||
|
||||
def test_a_page_without_tags_has_none():
|
||||
assert parse_tags('<a class="app_tagger">Nope</a><span class="app_tag">Nope</span>') == []
|
||||
|
||||
|
||||
CHECK = '<td class="checkcol"><span>✔</span></td>'
|
||||
EMPTY = '<td class="checkcol"></td>'
|
||||
|
||||
LANGUAGE_HTML = f"""
|
||||
<table class="game_language_options" cellpadding="0" cellspacing="0">
|
||||
<tr><th style="width: 94px;"></th><th class="checkcol">Interface</th><th class="checkcol">Full Audio</th><th class="checkcol">Subtitles</th></tr>
|
||||
<tr style="" class="">
|
||||
<td style="width: 94px; text-align: left" class="ellipsis">
|
||||
English</td>
|
||||
{CHECK}
|
||||
{CHECK}
|
||||
{CHECK}
|
||||
</tr>
|
||||
<tr class="unsupported"><td>French</td>{CHECK}{CHECK}{CHECK}</tr>
|
||||
<tr CLASS="Lang Unsupported"><td>German</td>{CHECK}{EMPTY}{EMPTY}</tr>
|
||||
<tr data-class="unsupported"><td>Italian</td>{CHECK}{EMPTY}{EMPTY}</tr>
|
||||
<tr class="supported"><td>Spanish - Spain</td>{EMPTY}{EMPTY}{CHECK}</tr>
|
||||
<tr class="unsupported"><td style="width: 94px; text-align: left" class="ellipsis">Japanese</td><td colspan="3">Not supported</td></tr>
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
def test_a_row_marked_unsupported_is_skipped_whatever_its_cells_say():
|
||||
out = parse_language_table(LANGUAGE_HTML)
|
||||
assert out["languages"] == ["English", "Spanish - Spain"]
|
||||
assert out["language_details"] == [
|
||||
{"name": "English", "interface": True, "audio": True, "subtitles": True},
|
||||
{"name": "Spanish - Spain", "interface": False, "audio": False, "subtitles": True},
|
||||
]
|
||||
Reference in new issue
Block a user