Files
poli0981andClaude Opus 5.5 999d0887ee Fix the SonarQube Cloud findings; release 4.4.2 (#155)
The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud.

Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh.

Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check.

Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41.

[skip-discuss]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 18:16:12 +07:00

488 lines
17 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
scraper.py — Shared scraping utilities for itch.io game data.
Provides:
- Session creation (honest User-Agent, retries on 5xx only)
- fetch_page(): one GET with explicit 429 / Retry-After reporting
- Pacer: polite delays, batch pauses, back-off after a 429
- Free/paid detection, info table parsing (all fields → N/A on missing),
including the AI-disclosure and accessibility rows itch.io added in 2025–26
- Description (the creator's tagline, else the first sentence), thumbnail,
NSFW extraction
- parse_game(): full single-game record from a fetched page
"""
import random
import re
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from html import unescape
from typing import Any
import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
NA = "N/A"
USER_AGENT = "FreeItchGamesBot/4.0 (+https://freeitchgames.win/about)"
HEADERS = {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
}
# (connect, read) timeouts in seconds
TIMEOUT = (5, 15)
# Polite delay range (seconds) between page requests
DELAY_MIN = 2.0
DELAY_MAX = 4.0
# Batch processing: pause longer after N requests
BATCH_SIZE = 20
BATCH_PAUSE_MIN = 15.0
BATCH_PAUSE_MAX = 30.0
# Retry-After values from itch.io are honoured but capped at this many seconds.
RETRY_AFTER_CAP = 300.0
# ---------------------------------------------------------------------------
# Session
# ---------------------------------------------------------------------------
def create_session() -> requests.Session:
"""requests.Session that retries connection errors and 5xx (not 429).
429 is surfaced to the caller (see fetch_page) so it can back off
globally instead of urllib3 sleeping for an uncapped Retry-After.
Backoff between the automatic retries: 0s, 4s.
"""
session = requests.Session()
session.headers.update(HEADERS)
retry = Retry(
total=2,
backoff_factor=2,
status_forcelist=[500, 502, 503, 504],
allowed_methods=["GET", "HEAD"],
respect_retry_after_header=False,
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry)
# Every URL we fetch is canonical https (canonical.py); no adapter for plain http.
session.mount("https://", adapter)
return session
# ---------------------------------------------------------------------------
# Rate-limiting helpers
# ---------------------------------------------------------------------------
def should_batch_pause(index: int) -> bool:
"""Return True when *index* (1-based) hits a batch boundary."""
return index > 0 and index % BATCH_SIZE == 0
class Pacer:
"""Paces sequential requests to one host; slows down after a 429.
wait() goes before every request except the first: a random polite delay,
or a longer batch pause every BATCH_SIZE requests. back_off() sleeps for a
Retry-After and doubles the delay for the rest of the run (max 4x).
`sleep` is injectable so tests never really sleep.
"""
def __init__(self, sleep=time.sleep, rng: random.Random | None = None):
self._sleep = sleep
self._rng = rng or random.Random()
self.factor = 1.0
self.requests = 0
def wait(self) -> None:
if self.requests > 0:
if should_batch_pause(self.requests):
seconds = self._rng.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX)
print(f" ⏸ Batch pause {seconds:.0f}s ...")
else:
seconds = self._rng.uniform(DELAY_MIN, DELAY_MAX) * self.factor
self._sleep(seconds)
self.requests += 1
def back_off(self, seconds: float) -> None:
self.factor = min(self.factor * 2, 4.0)
print(f" ⏳ Rate limited — sleeping {seconds:.0f}s, delay x{self.factor:g} from now on")
self._sleep(seconds)
def parse_retry_after(value: str | None, default: float) -> float:
"""Seconds to wait from a Retry-After header (seconds or HTTP-date), capped."""
if not value:
return min(default, RETRY_AFTER_CAP)
value = value.strip()
try:
seconds = float(value)
except ValueError:
try:
when = parsedate_to_datetime(value)
except TypeError, ValueError:
return min(default, RETRY_AFTER_CAP)
if when.tzinfo is None:
when = when.replace(tzinfo=UTC)
seconds = (when - datetime.now(UTC)).total_seconds()
return max(0.0, min(seconds, RETRY_AFTER_CAP))
# ---------------------------------------------------------------------------
# Fetch
# ---------------------------------------------------------------------------
@dataclass
class Page:
"""Result of one GET. `soup` is set only for HTTP 200."""
status: int | None # None = network error
soup: BeautifulSoup | None = None
retry_after: float = 0.0
error: str = ""
def fetch_page(session: requests.Session, url: str) -> Page:
try:
r = session.get(url, timeout=TIMEOUT, allow_redirects=True)
except requests.RequestException as e:
return Page(status=None, error=type(e).__name__)
page = Page(status=r.status_code)
if r.status_code in (429, 503):
page.retry_after = parse_retry_after(r.headers.get("Retry-After"), default=60.0)
elif r.status_code == 200:
# Parse bytes so BeautifulSoup honours the page's declared charset.
page.soup = BeautifulSoup(r.content, "html.parser")
r.close()
return page
# ---------------------------------------------------------------------------
# Free / Paid detection
# ---------------------------------------------------------------------------
def is_free_game(soup: BeautifulSoup) -> bool:
"""
Determine if a game page represents a **free** game.
Free indicators:
- No buy_row at all (browser-only / direct download)
- Button text is "Download Now" with "Name your own price"
- No <span class="dollars" itemprop="price">
Paid indicators:
- <span class="dollars" itemprop="price"> present
- Button text contains "Buy"
"""
buy_row = soup.find("div", class_="buy_row")
if not buy_row:
# No purchase section → free (browser game or direct link)
return True
# Explicit price tag → paid
price_tag = buy_row.find("span", class_="dollars", attrs={"itemprop": "price"})
if price_tag:
price_text = price_tag.get_text(strip=True)
# "$0.00" edge case → still free
if price_text and price_text not in ("$0.00", "$0.00 USD"):
return False
# Button text hint
buy_btn = buy_row.find("a", class_="buy_btn")
if buy_btn:
btn_text = buy_btn.get_text(strip=True).lower()
if "buy" in btn_text:
return False
return True
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _text(tag) -> str:
"""Visible text with whitespace runs collapsed (a page may break lines inside a link)."""
return " ".join(tag.get_text().split())
def _safe_text(tag, fallback: str = NA) -> str:
if tag is None:
return fallback
text = _text(tag)
return text if text else fallback
def now_iso() -> str:
"""Current UTC timestamp in ISO-8601."""
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def find_title(soup: BeautifulSoup) -> str:
title_tag = soup.find("h1", class_="game_title") or soup.find("h1", attrs={"itemprop": "name"})
return _safe_text(title_tag)
def has_info_panel(soup: BeautifulSoup) -> bool:
wrapper = soup.find("div", class_="info_panel_wrapper")
if not wrapper:
return False
return wrapper.find("table") is not None
def is_password_protected(soup: BeautifulSoup) -> bool:
"""itch.io's "A password is required to view this page" form instead of the game (HTTP 200)."""
return bool(
soup.find("div", class_="game_password_page")
or soup.find("input", attrs={"name": "game_password"})
)
# ---------------------------------------------------------------------------
# Info-table parsing
# ---------------------------------------------------------------------------
# Fields where itch.io lists multiple <a> links → store as JSON array
_LIST_FIELDS = {"Tags", "Platforms", "Languages", "Inputs", "Made with", "Accessibility"}
# Date rows: prefer abbr@title ("23 April 2021 @ 19:30 UTC") over the relative text
_DATE_FIELDS = {"Release date", "Published", "Updated"}
# Stored when the "Content" row says "No generative AI was used" (/games/tag-no-ai).
AI_NONE = "No AI"
def _links_to(a, slug: str) -> bool:
return str(a.get("href", "")).split("?", 1)[0].rstrip("/").endswith(f"/games/{slug}")
def parse_info_table(soup: BeautifulSoup) -> dict[str, Any]:
"""Parse the right-side info panel into a dict.
Multi-value fields (Tags, Platforms, etc.) → list[str]
Single-value fields (Genre, Status, etc.) → str
"AI Disclosure" ("AI Assisted, Graphics, Text") → the classification as
"AI Disclosure", what AI was used for as "AI content" (list); a "Content"
row linking to /games/tag-no-ai also sets "No AI" = True.
Missing fields are NOT inserted — caller handles defaults.
"""
info: dict[str, Any] = {}
wrapper = soup.find("div", class_="info_panel_wrapper")
if not wrapper:
return info
table = wrapper.find("table")
if not table:
return info
for row in table.find_all("tr"):
tds = row.find_all("td")
if len(tds) < 2:
continue
key = _text(tds[0])
value_td = tds[1]
if key in _DATE_FIELDS:
abbr = value_td.find("abbr")
if abbr and abbr.get("title"):
info[key] = abbr["title"]
else:
info[key] = _safe_text(value_td)
continue
# --- Rating: extract from itemprop attributes ---
if key == "Rating":
rating_div = value_td.find("div", class_="aggregate_rating")
if rating_div:
rv = rating_div.find(itemprop="ratingValue")
rc = rating_div.find(itemprop="ratingCount")
info["Rating"] = rv.get("content", NA) if rv else NA
info["RatingCount"] = rc.get("content", NA) if rc else NA
else:
info["Rating"] = NA
info["RatingCount"] = NA
continue
# --- Multi-value fields → list of strings ---
links = value_td.find_all("a")
texts = [_text(a) for a in links]
if key in _LIST_FIELDS:
info[key] = texts
continue
# --- AI Disclosure: the classification first, then what AI was used for ---
if key == "AI Disclosure":
info[key] = texts[0] if texts else _safe_text(value_td)
info["AI content"] = texts[1:]
continue
if key == "Content" and any(_links_to(a, "tag-no-ai") for a in links):
info["No AI"] = True
# --- Genre: use only the FIRST value (primary genre) ---
if key == "Genre":
info[key] = texts[0] if texts else _safe_text(value_td)
continue
# --- Generic single-value: prefer link texts joined, fallback plain ---
value = ", ".join(texts) if links else _text(value_td)
info[key] = value if value else NA
return info
def ai_and_accessibility(info: dict[str, Any]) -> dict[str, Any]:
"""The stored fields from itch.io's newer rows (the daily refresh keeps them current).
ai_disclosure: "AI Assisted" (the "AI Disclosure" row), "No AI" (the
"Content" row's /games/tag-no-ai) or N/A — the creator has not said.
ai_content: what AI was used for ("Graphics", "Text", "Code", "Sounds").
"""
return {
"ai_disclosure": info.get("AI Disclosure") or (AI_NONE if info.get("No AI") else NA),
"ai_content": info.get("AI content", []),
"accessibility": info.get("Accessibility", []),
}
# ---------------------------------------------------------------------------
# Description
# ---------------------------------------------------------------------------
_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'“(])")
DESCRIPTION_MAX = 200
def _cap(text: str) -> str:
"""`text`, cut on a word and marked with "..." when longer than DESCRIPTION_MAX."""
if len(text) <= DESCRIPTION_MAX:
return text
cut = text[: DESCRIPTION_MAX - 3]
return (cut.rsplit(" ", 1)[0] if " " in cut else cut) + "..."
def extract_tagline(soup: BeautifulSoup) -> str:
"""The creator's "short description or tagline" (itch.io puts it in og:description)."""
meta = soup.find("meta", property="og:description")
if not meta:
return NA
text = " ".join(str(meta.get("content", "")).split())
return _cap(text) if text else NA
def extract_first_sentence(soup: BeautifulSoup) -> str:
desc_tag = soup.find("div", class_="formatted_description")
if not desc_tag:
return NA
raw = desc_tag.get_text(strip=True, separator=" ")
full = " ".join(unescape(raw).split())
if not full:
return NA
# First sentence (a terminator followed by whitespace + a capital), capped at 200 chars.
return _cap(_SENTENCE_END.split(full, maxsplit=1)[0])
def extract_description(soup: BeautifulSoup) -> str:
"""The tagline, or the description's first sentence when the creator wrote none."""
tagline = extract_tagline(soup)
return tagline if tagline != NA else extract_first_sentence(soup)
# ---------------------------------------------------------------------------
# Thumbnail
# ---------------------------------------------------------------------------
def extract_thumbnail(soup: BeautifulSoup) -> str:
meta = soup.find("meta", property="og:image")
if meta:
content = str(meta.get("content", ""))
if content.startswith("http"):
return content
ss = soup.find("div", class_="screenshot_list")
if ss:
img = ss.find("img")
if img and str(img.get("src", "")).startswith("http"):
return img["src"]
return NA
# ---------------------------------------------------------------------------
# NSFW detection
# ---------------------------------------------------------------------------
_NSFW_RE = re.compile(
r"\b(adult|nsfw|18\+|erotic\w*|hentai|porn\w*|mature|sexual\w*)(?!\w)", re.IGNORECASE
)
def detect_nsfw(soup: BeautifulSoup, tags, *texts: str) -> str:
"""Detect NSFW from tags, short texts (tagline, first sentence) and itch.io's warning.
`tags` can be list[str] or str. Matches whole words only.
"""
tags_text = " ".join(tags) if isinstance(tags, list) else (tags or "")
if _NSFW_RE.search(tags_text):
return "Yes"
if any(text != NA and _NSFW_RE.search(text) for text in texts):
return "Yes"
# Either notice (a class list in find() would mean the same; ".a.b" would need both).
if soup.select_one("div.view_game_warning, div.mature_content_notice"):
return "Yes"
return "No"
# ---------------------------------------------------------------------------
# Full single-game record
# ---------------------------------------------------------------------------
def parse_game(soup: BeautifulSoup, url: str) -> dict:
"""Build the full record for a fetched game page (plus an `is_free` flag).
Field order matches the stored schema; `updated_at` is appended last and
only when itch.io shows an "Updated" row.
"""
info = parse_info_table(soup)
tags = info.get("Tags", [])
description = extract_description(soup)
release_date = info.get("Release date") or info.get("Published") or NA
# The first sentence too: a tagline rarely says what the description does.
nsfw = detect_nsfw(soup, tags, description, extract_first_sentence(soup))
record = {
"url": url,
"name": find_title(soup),
"is_free": is_free_game(soup),
"dev": info.get("Author") or info.get("Authors", NA),
"description": description,
"genre": info.get("Genre", NA),
"status": info.get("Status", NA),
"publisher": info.get("Publisher", NA),
"release_date": release_date,
"rating": info.get("Rating", NA),
"rating_count": info.get("RatingCount", NA),
"average_session": info.get("Average session", NA),
"nsfw": nsfw,
"thumbnail": extract_thumbnail(soup),
"tags": tags,
"platforms": info.get("Platforms", []),
"languages": info.get("Languages", []),
"inputs": info.get("Inputs", []),
"made_with": info.get("Made with", []),
**ai_and_accessibility(info),
"safe_virus": "?",
"notes": "",
}
if info.get("Updated"):
record["updated_at"] = info["Updated"]
return record