- Linear parsers (python:S8786): the store page's "unsupported" language rows and its tags, and the discovery sweep's search rows, no longer use regexes that backtrack quadratically on long input. New tests, written against the old regexes first, pin what they read, and a differential fuzz (200,000 random inputs) found no case where old and new disagree. - Raw invisible and bidi characters in normalize.py and two test files are written as escapes (text:S6389); every string value is unchanged. - scrape_store_fields() states when there is no usable page (scraped = None) instead of deriving it into a separate flag (pythonbugs:S2259). - cmd_list() in delete_game.py and edit_game.py returns nothing; the dispatch returns 0 (python:S3516). remove_games.main() keeps its int: delete_game.py and its tests use the exit code. - The Steam client mounts its retrying adapter for https only (python:S5332); nothing is fetched over http. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
199 lines
6.7 KiB
Python
199 lines
6.7 KiB
Python
"""
|
|
Store page HTML scraper v2.2.
|
|
|
|
Pre-compiled regexes for hot paths.
|
|
All parsers operate on the same HTML string (fetched once).
|
|
|
|
Text is HTML-unescaped before it is stored: the tag regex used to keep
|
|
"Point & Click", and the site renders tags as text.
|
|
"""
|
|
import re
|
|
from html import unescape as _unescape
|
|
|
|
from .anticheat import parse_page
|
|
|
|
# ──── Pre-compiled patterns ────
|
|
|
|
_RE_LANG_TABLE = re.compile(
|
|
r'<table\s+class\s*=\s*"game_language_options"[^>]*>(.*?)</table>',
|
|
re.DOTALL | re.IGNORECASE,
|
|
)
|
|
_RE_TABLE_ROW = re.compile(r'(<tr[^>]*>)(.*?)</tr>', re.DOTALL | re.IGNORECASE)
|
|
_RE_TABLE_CELL = re.compile(r'<td[^>]*>(.*?)</td>', re.DOTALL | re.IGNORECASE)
|
|
_RE_HAS_CHECK = re.compile(r'<span[^>]*>[^<]*</span>')
|
|
_RE_STRIP_HTML = re.compile(r'<[^>]+>')
|
|
# Each class="..." value of a tag; _marked_unsupported() looks inside them.
|
|
# One regex with "unsupported" between two [^"]* runs backtracked
|
|
# quadratically on a long attribute (SonarCloud python:S8786).
|
|
_RE_CLASS_VALUE = re.compile(r'class\s*=\s*"([^"]*)"', re.IGNORECASE)
|
|
|
|
_RE_DLC_SECTION_START = re.compile(
|
|
r'(?:id\s*=\s*"gameAreaDLCSection"|class\s*=\s*"game_area_dlc_section")',
|
|
re.IGNORECASE,
|
|
)
|
|
_RE_DLC_PRICE_ATTR = re.compile(r'data-price-final\s*=\s*"(\d+)"')
|
|
_RE_DLC_ROW = re.compile(
|
|
r'class\s*=\s*"game_area_dlc_row[^"]*"[^>]*>.*?</a>',
|
|
re.DOTALL | re.IGNORECASE,
|
|
)
|
|
_RE_DLC_PRICE_TEXT = re.compile(
|
|
r'class\s*=\s*"game_area_dlc_price"[^>]*>(.*?)</div>',
|
|
re.DOTALL | re.IGNORECASE,
|
|
)
|
|
_RE_HAS_DIGIT = re.compile(r'\d')
|
|
|
|
# The text is taken whole and trimmed by parse_tags(), which also drops an
|
|
# empty one: `\s*([^<]+?)\s*` around it backtracked on long whitespace.
|
|
_RE_APP_TAG = re.compile(r'class\s*=\s*"app_tag"[^>]*>([^<]*)</a>')
|
|
|
|
# Steam's own page sometimes prints a language's localisation token where its
|
|
# name belongs: TrackMania Nations Forever lists "#lang_slovakian". A token
|
|
# with no entry here is dropped rather than stored as a language.
|
|
_LANGUAGE_TOKENS = {"#lang_slovakian": "Slovak"}
|
|
|
|
|
|
# ──── Language table ────
|
|
|
|
def _marked_unsupported(tag: str) -> bool:
|
|
"""Whether any class="..." of a tag names "unsupported", in any case."""
|
|
return any("unsupported" in m.group(1).lower() for m in _RE_CLASS_VALUE.finditer(tag))
|
|
|
|
|
|
def parse_language_table(html: str) -> dict:
|
|
"""
|
|
Parse #languageTable → {languages: [...], language_details: [...]}.
|
|
Each detail has name, interface, audio, subtitles booleans.
|
|
|
|
Skips rows with:
|
|
- class="unsupported" on <tr>
|
|
- "Not supported" text in row
|
|
- colspan cells (unsupported langs use <td colspan="3">Not supported</td>)
|
|
- All three interface/audio/subtitles = false
|
|
"""
|
|
m = _RE_LANG_TABLE.search(html)
|
|
if not m:
|
|
return {"languages": [], "language_details": []}
|
|
|
|
table_html = m.group(1)
|
|
languages, details = [], []
|
|
|
|
for row_match in _RE_TABLE_ROW.finditer(table_html):
|
|
tr_tag = row_match.group(1) # <tr class="unsupported"> etc.
|
|
row_content = row_match.group(2) # inner HTML
|
|
|
|
# Skip header rows
|
|
if '<th' in row_content:
|
|
continue
|
|
|
|
# Skip unsupported rows (class on <tr>)
|
|
if _marked_unsupported(tr_tag):
|
|
continue
|
|
|
|
# Skip rows with "not supported" text
|
|
row_text = _RE_STRIP_HTML.sub('', row_content).lower()
|
|
if 'not supported' in row_text:
|
|
continue
|
|
|
|
cells = _RE_TABLE_CELL.findall(row_content)
|
|
if len(cells) < 2:
|
|
continue
|
|
|
|
name = _unescape(_RE_STRIP_HTML.sub('', cells[0])).strip()
|
|
if name.startswith("#"):
|
|
name = _LANGUAGE_TOKENS.get(name.lower(), "")
|
|
if not name:
|
|
continue
|
|
|
|
# Rows with colspan (unsupported) only have 2 cells: name + "Not supported"
|
|
# Normal rows have 4 cells: name + interface + audio + subtitles
|
|
if len(cells) < 4:
|
|
# Might be a colspan "Not supported" row that wasn't caught above
|
|
continue
|
|
|
|
interface = bool(_RE_HAS_CHECK.search(cells[1]))
|
|
audio = bool(_RE_HAS_CHECK.search(cells[2]))
|
|
subtitles = bool(_RE_HAS_CHECK.search(cells[3]))
|
|
|
|
# Skip if all three are false (no support at all)
|
|
if not interface and not audio and not subtitles:
|
|
continue
|
|
|
|
languages.append(name)
|
|
details.append({
|
|
"name": name,
|
|
"interface": interface,
|
|
"audio": audio,
|
|
"subtitles": subtitles,
|
|
})
|
|
|
|
return {"languages": languages, "language_details": details}
|
|
|
|
|
|
# ──── DLC pricing ────
|
|
|
|
def parse_has_paid_dlc(html: str) -> bool:
|
|
"""
|
|
Check if the game has PAID DLC by looking at DLC rows on the page.
|
|
|
|
Strategy:
|
|
1. Verify DLC section exists (don't false-positive on non-DLC pages)
|
|
2. Find all game_area_dlc_row elements
|
|
3. Check data-price-final attributes in each row (most reliable)
|
|
4. Fallback: check price text in game_area_dlc_price divs
|
|
|
|
Returns False if no DLC section, or all DLC are free.
|
|
"""
|
|
# Must have a DLC section
|
|
if not _RE_DLC_SECTION_START.search(html):
|
|
return False
|
|
|
|
# Method 1: Check data-price-final in DLC rows
|
|
for row_match in _RE_DLC_ROW.finditer(html):
|
|
row_html = row_match.group(0)
|
|
for price_match in _RE_DLC_PRICE_ATTR.finditer(row_html):
|
|
if int(price_match.group(1)) > 0:
|
|
return True
|
|
|
|
# Method 2: Check price text in any DLC price div on the page
|
|
# (only search after the DLC section marker)
|
|
dlc_start = _RE_DLC_SECTION_START.search(html)
|
|
if dlc_start:
|
|
dlc_area = html[dlc_start.start():]
|
|
for text_match in _RE_DLC_PRICE_TEXT.finditer(dlc_area):
|
|
text = _RE_STRIP_HTML.sub('', text_match.group(1)).strip().lower()
|
|
if text in ("", "free", "n/a", "free to play"):
|
|
continue
|
|
if _RE_HAS_DIGIT.search(text):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
# ──── Tags ────
|
|
|
|
def parse_tags(html: str) -> list[str]:
|
|
"""Extract user-defined tags. Deduplicated, order preserved."""
|
|
tags, seen = [], set()
|
|
for m in _RE_APP_TAG.finditer(html):
|
|
t = _unescape(m.group(1)).strip()
|
|
key = t.lower()
|
|
if t and t != "+" and key not in seen:
|
|
tags.append(t)
|
|
seen.add(key)
|
|
return tags
|
|
|
|
|
|
# ──── Combined ────
|
|
|
|
def scrape_store_page(html: str) -> dict:
|
|
"""Parse languages + DLC pricing + tags + the anti-cheat disclosure
|
|
(core/anticheat.py; None when the page has none) from one HTML string."""
|
|
lang = parse_language_table(html)
|
|
return {
|
|
"languages": lang["languages"],
|
|
"language_details": lang["language_details"],
|
|
"has_paid_dlc": parse_has_paid_dlc(html),
|
|
"tags": parse_tags(html),
|
|
"anti_cheat": parse_page(html),
|
|
}
|