diff --git a/scripts/core/fetcher.py b/scripts/core/fetcher.py index d7b4e68e..e8df0056 100644 --- a/scripts/core/fetcher.py +++ b/scripts/core/fetcher.py @@ -219,15 +219,16 @@ def scrape_store_fields(client, appid: str, details: Optional[dict] = None) -> O if status in ("network_error", "not_found"): return None scraped = scrape_store_page(page) if status == "ok" else None - real_page = bool(scraped and (scraped["languages"] or scraped["tags"])) - if real_page and scraped["languages"] and scraped["tags"]: + if scraped is not None and not (scraped["languages"] or scraped["tags"]): + scraped = None # a page with neither tells nothing: all of it comes from the API + if scraped is not None and scraped["languages"] and scraped["tags"]: return scraped item = (client.fetch_store_items([appid]) or {}).get(str(appid)) if item is None: - return scraped if real_page else None + return scraped api = parse_store_item(item, client.fetch_tag_names()) - if not real_page: + if scraped is None: api["has_paid_dlc"] = _paid_dlc(client, appid, details) return api if not scraped["languages"] and api["languages"]: diff --git a/scripts/core/normalize.py b/scripts/core/normalize.py index 92d4feb3..62b9bf9d 100644 --- a/scripts/core/normalize.py +++ b/scripts/core/normalize.py @@ -133,18 +133,18 @@ def normalize_release_date(value): # controls. "Gameforge 4D GmbH" was stored with a stray U+202C after it, and # "Triple Fantasy" with a ZWSP in front. _INVISIBLE = dict.fromkeys(map(ord, ( - "​⁠" - "‪‫‬‭‮" - "⁦⁧⁨⁩" + "\u200b\ufeff\u2060" + "\u202a\u202b\u202c\u202d\u202e" + "\u2066\u2067\u2068\u2069" ))) # Two or more zero-width characters in a row are a watermark, not writing: one # developer name carried 43 of them between "YAMAYURI " and "GAMES". A SINGLE # zero-width joiner is how emoji sequences are built, and a single zero-width # non-joiner is spelling in Persian and the Indic scripts, so those stay. -_ZERO_WIDTH_RUN = re.compile("[​‌‍]{2,}") +_ZERO_WIDTH_RUN = re.compile("[\u200b\u200c\u200d]{2,}") # Plain whitespace, tabs and NBSP ("Cagliari 1943\xa0VR", "ARTFX - GAME\t"), but -# not the ideographic space: "鳥獣妖怪戯画 (Choju Yokai Giga)" spells it that way. -_WHITESPACE = re.compile("[ \t\r\n\f\v ]+") +# not the ideographic space: "鳥獣妖怪戯画\u3000(Choju Yokai Giga)" spells it that way. +_WHITESPACE = re.compile("[ \t\r\n\f\v\xa0]+") def clean_text(value, single_line: bool = True): diff --git a/scripts/core/scraper.py b/scripts/core/scraper.py index 7358d1c8..f042efb9 100644 --- a/scripts/core/scraper.py +++ b/scripts/core/scraper.py @@ -22,7 +22,10 @@ _RE_TABLE_ROW = re.compile(r'(]*>)(.*?)', re.DOTALL | re.IGNORECASE) _RE_TABLE_CELL = re.compile(r']*>(.*?)', re.DOTALL | re.IGNORECASE) _RE_HAS_CHECK = re.compile(r']*>[^<]*') _RE_STRIP_HTML = re.compile(r'<[^>]+>') -_RE_UNSUPPORTED = re.compile(r'class\s*=\s*"[^"]*unsupported[^"]*"', re.IGNORECASE) +# Each class="..." value of a tag; _marked_unsupported() looks inside them. +# One regex with "unsupported" between two [^"]* runs backtracked +# quadratically on a long attribute (SonarCloud python:S8786). +_RE_CLASS_VALUE = re.compile(r'class\s*=\s*"([^"]*)"', re.IGNORECASE) _RE_DLC_SECTION_START = re.compile( r'(?:id\s*=\s*"gameAreaDLCSection"|class\s*=\s*"game_area_dlc_section")', @@ -39,7 +42,9 @@ _RE_DLC_PRICE_TEXT = re.compile( ) _RE_HAS_DIGIT = re.compile(r'\d') -_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>\s*([^<]+?)\s*') +# The text is taken whole and trimmed by parse_tags(), which also drops an +# empty one: `\s*([^<]+?)\s*` around it backtracked on long whitespace. +_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>([^<]*)') # Steam's own page sometimes prints a language's localisation token where its # name belongs: TrackMania Nations Forever lists "#lang_slovakian". A token @@ -49,6 +54,11 @@ _LANGUAGE_TOKENS = {"#lang_slovakian": "Slovak"} # ──── Language table ──── +def _marked_unsupported(tag: str) -> bool: + """Whether any class="..." of a tag names "unsupported", in any case.""" + return any("unsupported" in m.group(1).lower() for m in _RE_CLASS_VALUE.finditer(tag)) + + def parse_language_table(html: str) -> dict: """ Parse #languageTable → {languages: [...], language_details: [...]}. @@ -76,7 +86,7 @@ def parse_language_table(html: str) -> dict: continue # Skip unsupported rows (class on ) - if _RE_UNSUPPORTED.search(tr_tag): + if _marked_unsupported(tr_tag): continue # Skip rows with "not supported" text diff --git a/scripts/core/steam_client.py b/scripts/core/steam_client.py index 8f4da2f0..89bd1417 100644 --- a/scripts/core/steam_client.py +++ b/scripts/core/steam_client.py @@ -82,8 +82,8 @@ class SteamClient: status_forcelist=[], allowed_methods=["GET", "HEAD"]), pool_connections=10, pool_maxsize=10, ) + # https only: every endpoint this client calls is. s.mount("https://", adapter) - s.mount("http://", adapter) return s def _throttle_store(self): diff --git a/scripts/delete_game.py b/scripts/delete_game.py index aaca3dbf..fb1ec21a 100644 --- a/scripts/delete_game.py +++ b/scripts/delete_game.py @@ -92,16 +92,15 @@ def cmd_check(path: str = BLOCKLIST_JSONL) -> int: return 1 if problems else 0 -def cmd_list() -> int: +def cmd_list() -> None: entries = load_blocklist() if not entries: print(f"{BLOCKLIST_JSONL}: nothing blocked.") - return 0 + return for e in entries.values(): note = f" - {e['note']}" if e.get("note") else "" print(f"{e['appid']:>10} {e['blocked_at']} {e['reason']:<13} {e.get('name', '')}{note}") print(f"{len(entries)} blocked game(s).") - return 0 def cmd_add(targets: list[str], reason: str, note: str, apply: bool) -> int: @@ -179,7 +178,8 @@ def main(argv=None) -> int: if args.check: return cmd_check() if args.list: - return cmd_list() + cmd_list() + return 0 if not args.targets: ap.print_help() return 2 diff --git a/scripts/discover_new.py b/scripts/discover_new.py index 1ce408ca..4a2a6143 100644 --- a/scripts/discover_new.py +++ b/scripts/discover_new.py @@ -41,7 +41,6 @@ DEFAULT_STOP_AFTER_KNOWN = 2 POST_BATCH = 100 _RE_APPID = re.compile(r'data-ds-appid="([^"]+)"') -_RE_ROW = re.compile(r']*data-ds-appid="([^"]+)"[^>]*>(.*?)', re.S) _RE_TITLE = re.compile(r'(.*?)', re.S) # NOT `col search_released`: the live markup is # `
`. The `col ` prefix @@ -106,6 +105,35 @@ def safe_stdout(): pass +def _row_anchors(results_html): + """(appid, inner HTML) of each search row: an `` whose tag carries + data-ds-appid (the last one, if twice), up to the next ``. + + One forward pass. It reads exactly what + `]*data-ds-appid="([^"]+)"[^>]*>(.*?)` did, without that regex's + backtracking (SonarCloud python:S8786): a tag ends at its first `>`, and + the scan goes on after a row's ``, or after the tag of an anchor that + is not a row. + """ + pos = 0 + while True: + start = results_html.find("", start) + if close < 0: + return + appids = _RE_APPID.findall(results_html, start, close) + if not appids: + pos = close + 1 + continue + end = results_html.find("", close + 1) + if end < 0: + return + yield appids[-1], results_html[close + 1:end] + pos = end + len("") + + def parse_search_rows(results_html): """[(appid, name, released_text)] from one search page. @@ -115,7 +143,7 @@ def parse_search_rows(results_html): discovery only. """ rows = [] - for appid, inner in _RE_ROW.findall(results_html): + for appid, inner in _row_anchors(results_html): if "," in appid or not appid.isdigit(): continue # bundle/package id title_m = _RE_TITLE.search(inner) diff --git a/scripts/edit_game.py b/scripts/edit_game.py index 0b08715e..abe2dcf6 100644 --- a/scripts/edit_game.py +++ b/scripts/edit_game.py @@ -76,7 +76,7 @@ def cmd_list(): docs = load_overrides() if not docs: print("no overrides") - return 0 + return print(f"{len(docs)} override(s):") for appid, doc in sorted(docs.items()): live = doc.get("fields") or {} @@ -85,7 +85,6 @@ def cmd_list(): print(f" {appid:>8} {name[:44]:<44} {len(live)} active, {len(gone)} retired") for f, e in sorted(live.items()): print(f" {f} = {e.get('value')!r} (was {e.get('was')!r})") - return 0 def cmd_check(): @@ -155,7 +154,8 @@ def main(): safe_stdout() if args.list: - return cmd_list() + cmd_list() + return 0 if args.check: return cmd_check() if not args.appid: diff --git a/scripts/tests/test_discover_new.py b/scripts/tests/test_discover_new.py index 9ab9429e..e151023f 100644 --- a/scripts/tests/test_discover_new.py +++ b/scripts/tests/test_discover_new.py @@ -30,3 +30,40 @@ def test_removed_games_count_as_known(tmp_path, monkeypatch): with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f: f.write(serialize_entry(make_entry("2", "Two", "not_free", "", "2026-10-13T12:00:00Z")) + "\n") assert discover_new.catalogue_appids() == {"1", "2"} + + +# Written against the row regex parse_search_rows used before it became a +# linear scan (SonarCloud python:S8786); every case is behaviour that existed. +SEARCH_HTML = """ + +
+
Dota 2
+
Jul 9, 2013
+
+
+A Bundle +Flip & Shop
Coming soon
+ +Bold Name +Fourinner tail +Twice +Never closed +""" + + +def test_search_rows_are_read_from_each_anchor(): + assert discover_new.parse_search_rows(SEARCH_HTML) == [ + ("570", "Dota 2", "Jul 9, 2013"), + ("1", "Flip & Shop", "Coming soon"), + ("2", "Bold Name", ""), + ("4", "Four", ""), + ("8", "Twice", ""), + ] + + +def test_appids_outside_anchors_are_still_recovered(): + # A markup change that drops the anchors degrades to "fetch details for + # everything" instead of finding nothing. + assert discover_new.parse_search_rows('
') == [ + ("5", "", ""), + ] diff --git a/scripts/tests/test_normalize.py b/scripts/tests/test_normalize.py index fda3f35e..d8b008c7 100644 --- a/scripts/tests/test_normalize.py +++ b/scripts/tests/test_normalize.py @@ -82,17 +82,17 @@ def test_the_shape_is_a_fixed_point(): ("Raffiné Studio\t", "Raffiné Studio"), ("Fire from the sky: Cagliari 1943\xa0VR", "Fire from the sky: Cagliari 1943 VR"), ("Stuffed Wombat", "Stuffed Wombat"), - ("Gameforge 4D GmbH‬", "Gameforge 4D GmbH"), - ("​Triple Fantasy", "Triple Fantasy"), - ("Wilma Wiiand​​", "Wilma Wiiand"), - ("YAMAYURI ‌‌‍‍‍​‍‌‌GAMES", "YAMAYURI GAMES"), + ("Gameforge 4D GmbH\u202c", "Gameforge 4D GmbH"), + ("\u200bTriple Fantasy", "Triple Fantasy"), + ("Wilma Wiiand\u200b\u200b", "Wilma Wiiand"), + ("YAMAYURI \u200c\u200c\u200d\u200d\u200d\u200b\u200d\u200c\u200cGAMES", "YAMAYURI GAMES"), # Kept: real names in any script, an ideographic space, a single ZWNJ # (Persian spelling) and a single ZWJ (an emoji sequence). ("千鳥豚工作室", "千鳥豚工作室"), - ("鳥獣妖怪戯画 (Choju Yokai Giga)", "鳥獣妖怪戯画 (Choju Yokai Giga)"), - ("می‌خواهم", "می‌خواهم"), + ("鳥獣妖怪戯画\u3000(Choju Yokai Giga)", "鳥獣妖怪戯画\u3000(Choju Yokai Giga)"), + ("می\u200cخواهم", "می\u200cخواهم"), ("Pocket Rocket Studio 🚀", "Pocket Rocket Studio 🚀"), - ("👨‍🚀 Studio", "👨‍🚀 Studio"), + ("👨\u200d🚀 Studio", "👨\u200d🚀 Studio"), ]) def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected): assert clean_text(raw) == expected @@ -100,7 +100,7 @@ def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected def test_a_blurb_keeps_its_line_breaks(): - assert clean_text("Let's start typing!\r\nDiscover​ more ", single_line=False) == \ + assert clean_text("Let's start typing!\r\nDiscover\u200b more ", single_line=False) == \ "Let's start typing!\r\nDiscover more" @@ -166,8 +166,8 @@ def test_a_record_only_has_its_own_fields_touched(): def _messy(appid: int) -> dict: return { "link": f"https://store.steampowered.com/app/{appid}/", - "name": "​Triple Fantasy ", - "developer": ["Wilma Wiiand​​"], + "name": "\u200bTriple Fantasy ", + "developer": ["Wilma Wiiand\u200b\u200b"], "publisher": [" "], "release_date": "28 апр. 2025 г.", "description": "an art "game"", diff --git a/scripts/tests/test_refresh_store_data.py b/scripts/tests/test_refresh_store_data.py index e5d07ba6..3cd1d91b 100644 --- a/scripts/tests/test_refresh_store_data.py +++ b/scripts/tests/test_refresh_store_data.py @@ -366,8 +366,8 @@ def test_a_release_date_is_stored_in_the_one_shape(): def test_names_are_compared_cleaned(): game = _game(developer=["Wilma Wiiand"], publisher=["Gameforge 4D GmbH"]) changed = rsd.apply_app_details(game, { - "developers": ["Wilma Wiiand​​", " "], - "publishers": ["Gameforge 4D GmbH‬"], + "developers": ["Wilma Wiiand\u200b\u200b", " "], + "publishers": ["Gameforge 4D GmbH\u202c"], }) assert "developer" not in changed and "publisher" not in changed diff --git a/scripts/tests/test_scraper.py b/scripts/tests/test_scraper.py new file mode 100644 index 00000000..000dbedd --- /dev/null +++ b/scripts/tests/test_scraper.py @@ -0,0 +1,56 @@ +"""How core/scraper.py reads a store page's tags and language table. + +Written against the regexes these parsers used before they were made linear +(SonarCloud python:S8786), so each case pins behaviour that already existed. +""" +from core.scraper import parse_language_table, parse_tags + + +TAGS_HTML = """ +
+""" + + +def test_tags_are_unescaped_trimmed_and_deduplicated(): + assert parse_tags(TAGS_HTML) == ["Free to Play", "Point & Click", "action", "RPG > JRPG"] + + +def test_a_page_without_tags_has_none(): + assert parse_tags('NopeNope') == [] + + +CHECK = '✔' +EMPTY = '' + +LANGUAGE_HTML = f""" + + + + +{CHECK} +{CHECK} +{CHECK} + +{CHECK}{CHECK}{CHECK} +{CHECK}{EMPTY}{EMPTY} +{CHECK}{EMPTY}{EMPTY} +{EMPTY}{EMPTY}{CHECK} + +
InterfaceFull AudioSubtitles
+English
French
German
Italian
Spanish - Spain
JapaneseNot supported
+""" + + +def test_a_row_marked_unsupported_is_skipped_whatever_its_cells_say(): + out = parse_language_table(LANGUAGE_HTML) + assert out["languages"] == ["English", "Spanish - Spain"] + assert out["language_details"] == [ + {"name": "English", "interface": True, "audio": True, "subtitles": True}, + {"name": "Spanish - Spain", "interface": False, "audio": False, "subtitles": True}, + ]