`. The `col ` prefix
@@ -106,6 +105,35 @@ def safe_stdout():
pass
+def _row_anchors(results_html):
+ """(appid, inner HTML) of each search row: an `
` whose tag carries
+ data-ds-appid (the last one, if twice), up to the next ``.
+
+ One forward pass. It reads exactly what
+ `
]*data-ds-appid="([^"]+)"[^>]*>(.*?)` did, without that regex's
+ backtracking (SonarCloud python:S8786): a tag ends at its first `>`, and
+ the scan goes on after a row's ``, or after the tag of an anchor that
+ is not a row.
+ """
+ pos = 0
+ while True:
+ start = results_html.find("
", start)
+ if close < 0:
+ return
+ appids = _RE_APPID.findall(results_html, start, close)
+ if not appids:
+ pos = close + 1
+ continue
+ end = results_html.find("", close + 1)
+ if end < 0:
+ return
+ yield appids[-1], results_html[close + 1:end]
+ pos = end + len("")
+
+
def parse_search_rows(results_html):
"""[(appid, name, released_text)] from one search page.
@@ -115,7 +143,7 @@ def parse_search_rows(results_html):
discovery only.
"""
rows = []
- for appid, inner in _RE_ROW.findall(results_html):
+ for appid, inner in _row_anchors(results_html):
if "," in appid or not appid.isdigit():
continue # bundle/package id
title_m = _RE_TITLE.search(inner)
diff --git a/scripts/edit_game.py b/scripts/edit_game.py
index 0b08715e..abe2dcf6 100644
--- a/scripts/edit_game.py
+++ b/scripts/edit_game.py
@@ -76,7 +76,7 @@ def cmd_list():
docs = load_overrides()
if not docs:
print("no overrides")
- return 0
+ return
print(f"{len(docs)} override(s):")
for appid, doc in sorted(docs.items()):
live = doc.get("fields") or {}
@@ -85,7 +85,6 @@ def cmd_list():
print(f" {appid:>8} {name[:44]:<44} {len(live)} active, {len(gone)} retired")
for f, e in sorted(live.items()):
print(f" {f} = {e.get('value')!r} (was {e.get('was')!r})")
- return 0
def cmd_check():
@@ -155,7 +154,8 @@ def main():
safe_stdout()
if args.list:
- return cmd_list()
+ cmd_list()
+ return 0
if args.check:
return cmd_check()
if not args.appid:
diff --git a/scripts/tests/test_discover_new.py b/scripts/tests/test_discover_new.py
index 9ab9429e..e151023f 100644
--- a/scripts/tests/test_discover_new.py
+++ b/scripts/tests/test_discover_new.py
@@ -30,3 +30,40 @@ def test_removed_games_count_as_known(tmp_path, monkeypatch):
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
f.write(serialize_entry(make_entry("2", "Two", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
assert discover_new.catalogue_appids() == {"1", "2"}
+
+
+# Written against the row regex parse_search_rows used before it became a
+# linear scan (SonarCloud python:S8786); every case is behaviour that existed.
+SEARCH_HTML = """
+
+
+
+
A Bundle
+
Flip & Shop Coming soon
+
+
Bold Name
+
Fourinner tail
+
Twice
+
Never closed
+"""
+
+
+def test_search_rows_are_read_from_each_anchor():
+ assert discover_new.parse_search_rows(SEARCH_HTML) == [
+ ("570", "Dota 2", "Jul 9, 2013"),
+ ("1", "Flip & Shop", "Coming soon"),
+ ("2", "Bold Name", ""),
+ ("4", "Four", ""),
+ ("8", "Twice", ""),
+ ]
+
+
+def test_appids_outside_anchors_are_still_recovered():
+ # A markup change that drops the anchors degrades to "fetch details for
+ # everything" instead of finding nothing.
+ assert discover_new.parse_search_rows('') == [
+ ("5", "", ""),
+ ]
diff --git a/scripts/tests/test_normalize.py b/scripts/tests/test_normalize.py
index fda3f35e..d8b008c7 100644
--- a/scripts/tests/test_normalize.py
+++ b/scripts/tests/test_normalize.py
@@ -82,17 +82,17 @@ def test_the_shape_is_a_fixed_point():
("Raffiné Studio\t", "Raffiné Studio"),
("Fire from the sky: Cagliari 1943\xa0VR", "Fire from the sky: Cagliari 1943 VR"),
("Stuffed Wombat", "Stuffed Wombat"),
- ("Gameforge 4D GmbH", "Gameforge 4D GmbH"),
- ("Triple Fantasy", "Triple Fantasy"),
- ("Wilma Wiiand", "Wilma Wiiand"),
- ("YAMAYURI GAMES", "YAMAYURI GAMES"),
+ ("Gameforge 4D GmbH\u202c", "Gameforge 4D GmbH"),
+ ("\u200bTriple Fantasy", "Triple Fantasy"),
+ ("Wilma Wiiand\u200b\u200b", "Wilma Wiiand"),
+ ("YAMAYURI \u200c\u200c\u200d\u200d\u200d\u200b\u200d\u200c\u200cGAMES", "YAMAYURI GAMES"),
# Kept: real names in any script, an ideographic space, a single ZWNJ
# (Persian spelling) and a single ZWJ (an emoji sequence).
("千鳥豚工作室", "千鳥豚工作室"),
- ("鳥獣妖怪戯画 (Choju Yokai Giga)", "鳥獣妖怪戯画 (Choju Yokai Giga)"),
- ("میخواهم", "میخواهم"),
+ ("鳥獣妖怪戯画\u3000(Choju Yokai Giga)", "鳥獣妖怪戯画\u3000(Choju Yokai Giga)"),
+ ("می\u200cخواهم", "می\u200cخواهم"),
("Pocket Rocket Studio 🚀", "Pocket Rocket Studio 🚀"),
- ("👨🚀 Studio", "👨🚀 Studio"),
+ ("👨\u200d🚀 Studio", "👨\u200d🚀 Studio"),
])
def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected):
assert clean_text(raw) == expected
@@ -100,7 +100,7 @@ def test_names_lose_only_invisible_characters_and_stray_whitespace(raw, expected
def test_a_blurb_keeps_its_line_breaks():
- assert clean_text("Let's start typing!\r\nDiscover more ", single_line=False) == \
+ assert clean_text("Let's start typing!\r\nDiscover\u200b more ", single_line=False) == \
"Let's start typing!\r\nDiscover more"
@@ -166,8 +166,8 @@ def test_a_record_only_has_its_own_fields_touched():
def _messy(appid: int) -> dict:
return {
"link": f"https://store.steampowered.com/app/{appid}/",
- "name": "Triple Fantasy ",
- "developer": ["Wilma Wiiand"],
+ "name": "\u200bTriple Fantasy ",
+ "developer": ["Wilma Wiiand\u200b\u200b"],
"publisher": [" "],
"release_date": "28 апр. 2025 г.",
"description": "an art "game"",
diff --git a/scripts/tests/test_refresh_store_data.py b/scripts/tests/test_refresh_store_data.py
index e5d07ba6..3cd1d91b 100644
--- a/scripts/tests/test_refresh_store_data.py
+++ b/scripts/tests/test_refresh_store_data.py
@@ -366,8 +366,8 @@ def test_a_release_date_is_stored_in_the_one_shape():
def test_names_are_compared_cleaned():
game = _game(developer=["Wilma Wiiand"], publisher=["Gameforge 4D GmbH"])
changed = rsd.apply_app_details(game, {
- "developers": ["Wilma Wiiand", " "],
- "publishers": ["Gameforge 4D GmbH"],
+ "developers": ["Wilma Wiiand\u200b\u200b", " "],
+ "publishers": ["Gameforge 4D GmbH\u202c"],
})
assert "developer" not in changed and "publisher" not in changed
diff --git a/scripts/tests/test_scraper.py b/scripts/tests/test_scraper.py
new file mode 100644
index 00000000..000dbedd
--- /dev/null
+++ b/scripts/tests/test_scraper.py
@@ -0,0 +1,56 @@
+"""How core/scraper.py reads a store page's tags and language table.
+
+Written against the regexes these parsers used before they were made linear
+(SonarCloud python:S8786), so each case pins behaviour that already existed.
+"""
+from core.scraper import parse_language_table, parse_tags
+
+
+TAGS_HTML = """
+
+"""
+
+
+def test_tags_are_unescaped_trimmed_and_deduplicated():
+ assert parse_tags(TAGS_HTML) == ["Free to Play", "Point & Click", "action", "RPG > JRPG"]
+
+
+def test_a_page_without_tags_has_none():
+ assert parse_tags('NopeNope') == []
+
+
+CHECK = '
✔ | '
+EMPTY = '
| '
+
+LANGUAGE_HTML = f"""
+
+ | Interface | Full Audio | Subtitles |
+
+|
+English |
+{CHECK}
+{CHECK}
+{CHECK}
+
+| French | {CHECK}{CHECK}{CHECK}
+| German | {CHECK}{EMPTY}{EMPTY}
+| Italian | {CHECK}{EMPTY}{EMPTY}
+| Spanish - Spain | {EMPTY}{EMPTY}{CHECK}
+| Japanese | Not supported |
+
+"""
+
+
+def test_a_row_marked_unsupported_is_skipped_whatever_its_cells_say():
+ out = parse_language_table(LANGUAGE_HTML)
+ assert out["languages"] == ["English", "Spanish - Spain"]
+ assert out["language_details"] == [
+ {"name": "English", "interface": True, "audio": True, "subtitles": True},
+ {"name": "Spanish - Spain", "interface": False, "audio": False, "subtitles": True},
+ ]