Files
poli0981andClaude Opus 5.5 999d0887ee Fix the SonarQube Cloud findings; release 4.4.2 (#155)
The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud.

Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh.

Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check.

Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41.

[skip-discuss]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 18:16:12 +07:00

268 lines
11 KiB
Python

import update_info
from canonical import canonicalize
from conftest import load_fixture_soup, make_game, read_json, seed_catalog
from json_io import save_json
from scraper import Page
def _run(monkeypatch, pages: dict[str, Page], queue: list, **env):
save_json("scripts/temp_link.json", queue)
for key, value in env.items():
monkeypatch.setenv(key, value)
monkeypatch.setattr(
update_info, "fetch_with_backoff", lambda _s, url, _p, _c: pages.get(url, Page(status=500))
)
update_info.main(["--out", "patch.json"])
return read_json("patch.json")
def test_ingest_outcomes(repo, monkeypatch):
seed_catalog([make_game("known")])
save_json(
"scripts/deleted_games.json",
[
{
"url": "https://dev.itch.io/removed",
"name": "R",
"reason": "Game became paid",
"deleted_at": "2026-01-01T00:00:00Z",
}
],
)
pages = {
"https://dev.itch.io/free": Page(status=200, soup=load_fixture_soup("game_free_full.html")),
"https://dev.itch.io/paid": Page(status=200, soup=load_fixture_soup("game_paid.html")),
"https://dev.itch.io/dead": Page(status=404),
"https://dev.itch.io/locked": Page(
status=200, soup=load_fixture_soup("game_password.html")
),
}
queue = [
"https://DEV.itch.io/free/", # canonicalized
"https://dev.itch.io/free", # duplicate within the queue
"https://dev.itch.io/paid",
"https://dev.itch.io/dead",
"https://dev.itch.io/locked", # password-protected: final, not retried
"https://dev.itch.io/known",
"https://dev.itch.io/removed",
"https://dev.itch.io/flaky", # 500 → retry later
"not a url",
]
patch = _run(monkeypatch, pages, queue)
assert [a["url"] for a in patch["additions"]] == ["https://dev.itch.io/free"]
added = patch["additions"][0]
assert "is_free" not in added and added["added_at"].endswith("Z")
assert "https://dev.itch.io/flaky" not in patch["queue_remove"]
flaky = patch["ingest_state"]["https://dev.itch.io/flaky"]
assert flaky["attempts"] == 1 and flaky["last_error"] == "error: HTTP 500"
stats = patch["stats"]
assert (stats["added"], stats["paid"], stats["dead"], stats["duplicate"]) == (1, 1, 1, 1)
# A password-protected page is final: dropped from the queue, no attempt kept.
assert stats["private"] == 1 and patch["ingest_state"]["https://dev.itch.io/locked"] is None
assert (stats["deleted"], stats["invalid"], stats["retry"]) == (1, 1, 1)
# How each finished link ended, for the admin's batch tracking: retries, invalid
# entries and second spellings leave no outcome.
outcomes = {url: e["outcome"] for url, e in patch["ingest_outcomes"].items()}
assert outcomes == {
"https://dev.itch.io/free": "added",
"https://dev.itch.io/paid": "paid",
"https://dev.itch.io/dead": "dead",
"https://dev.itch.io/locked": "private",
"https://dev.itch.io/known": "duplicate",
"https://dev.itch.io/removed": "deleted",
}
assert all(e["at"].endswith("Z") for e in patch["ingest_outcomes"].values())
# apply_patch compares canonical forms, so each processed link is covered once.
removed = {canonicalize(u) or u for u in patch["queue_remove"]}
assert removed == {
"https://dev.itch.io/free",
"https://dev.itch.io/paid",
"https://dev.itch.io/dead",
"https://dev.itch.io/locked",
"https://dev.itch.io/known",
"https://dev.itch.io/removed",
"not a url",
}
def test_gives_up_after_three_attempts_and_resets_on_requeue(repo, monkeypatch):
seed_catalog([make_game("known")])
url = "https://dev.itch.io/flaky"
save_json("scripts/state/ingest_state.json", {url: {"attempts": 2, "last_error": "x"}})
patch = _run(monkeypatch, {}, [url])
assert url in patch["queue_remove"]
assert patch["ingest_state"][url]["attempts"] == 3 and "gave_up" in patch["ingest_state"][url]
assert patch["ingest_outcomes"][url]["outcome"] == "gave_up"
save_json("scripts/state/ingest_state.json", {url: patch["ingest_state"][url]})
patch = _run(monkeypatch, {}, [url])
assert (
patch["ingest_state"][url]["attempts"] == 1 and "gave_up" not in patch["ingest_state"][url]
)
def test_workflow_input_url_is_added(repo, monkeypatch):
seed_catalog([make_game("known")])
pages = {
"https://dev.itch.io/free": Page(status=200, soup=load_fixture_soup("game_free_full.html"))
}
patch = _run(monkeypatch, pages, [], INPUT_URL="https://dev.itch.io/free")
assert [a["url"] for a in patch["additions"]] == ["https://dev.itch.io/free"]
def test_empty_queue_writes_empty_patch(repo, monkeypatch):
seed_catalog([make_game("known")])
patch = _run(monkeypatch, {}, [])
assert patch["additions"] == [] and patch["queue_remove"] == []
def _apply(patch):
import apply_patch
from patch import write_patch
write_patch("applied.json", patch)
assert apply_patch.main(["applied.json"]) == 0
return read_json("scripts/temp_link.json")
def test_duplicate_spelling_does_not_drop_a_link_that_must_be_retried(repo, monkeypatch):
seed_catalog([make_game("known")])
queue = ["https://dev.itch.io/flaky", "https://dev.itch.io/flaky/"]
patch = _run(monkeypatch, {"https://dev.itch.io/flaky": Page(status=502)}, queue)
assert patch["queue_remove"] == []
assert _apply(patch) == queue
def test_duplicate_spelling_survives_a_rate_limit_stop(repo, monkeypatch):
seed_catalog([make_game("known")])
queue = ["https://dev.itch.io/a", "https://dev.itch.io/a"]
save_json("scripts/temp_link.json", queue)
monkeypatch.setattr(update_info, "fetch_with_backoff", lambda *_a: None)
update_info.main(["--out", "patch.json"])
assert _apply(read_json("patch.json")) == queue
def test_duplicate_spellings_all_go_once_the_first_copy_finishes(repo, monkeypatch):
seed_catalog([make_game("known")])
queue = ["https://dev.itch.io/gone", "https://DEV.itch.io/gone/"]
patch = _run(monkeypatch, {"https://dev.itch.io/gone": Page(status=404)}, queue)
assert _apply(patch) == []
def test_dispatched_url_is_processed_first_and_queued_until_done(repo, monkeypatch):
seed_catalog([make_game("known")])
seen = []
def fetch(_s, url, _p, _c):
seen.append(url)
return Page(status=500)
save_json("scripts/temp_link.json", ["https://dev.itch.io/backlog"])
monkeypatch.setenv("INPUT_URL", "https://dev.itch.io/Dispatched/")
monkeypatch.setattr(update_info, "fetch_with_backoff", fetch)
update_info.main(["--out", "patch.json"])
patch = read_json("patch.json")
assert seen[0] == "https://dev.itch.io/dispatched"
assert patch["queue_add"] == ["https://dev.itch.io/dispatched"]
# Transient failure: the dispatched URL now waits in the queue for the next run.
assert _apply(patch) == ["https://dev.itch.io/backlog", "https://dev.itch.io/dispatched"]
def test_dispatched_url_that_finishes_leaves_no_trace_in_the_queue(repo, monkeypatch):
seed_catalog([make_game("known")])
pages = {
"https://dev.itch.io/free": Page(status=200, soup=load_fixture_soup("game_free_full.html"))
}
patch = _run(monkeypatch, pages, [], INPUT_URL="https://dev.itch.io/free")
assert _apply(patch) == []
def test_non_string_queue_entries_are_cleaned_not_fatal(repo, monkeypatch):
seed_catalog([make_game("known")])
queue = [{"url": "https://dev.itch.io/wrapped"}, None, 5, "https://dev.itch.io/keep"]
patch = _run(monkeypatch, {}, queue)
assert patch["stats"]["invalid"] == 3
# The {"url": ...} entry is recovered as a plain URL; the rest are dropped.
assert _apply(patch) == ["https://dev.itch.io/wrapped", "https://dev.itch.io/keep"]
def test_failures_count_only_when_spaced_out(repo, monkeypatch):
"""Back-to-back runs (a burst of queue pushes) can't use up the attempts."""
import apply_patch
from patch import write_patch
seed_catalog([make_game("known")])
url = "https://dev.itch.io/flaky"
save_json("scripts/temp_link.json", [url])
monkeypatch.setattr(update_info, "fetch_with_backoff", lambda *_a: Page(status=502))
attempts = []
for stamp in (
"2026-09-18T01:00:00Z",
"2026-09-18T01:05:00Z", # 5 min later: not counted
"2026-09-18T02:00:00Z", # still inside the gap
"2026-09-18T07:30:00Z", # 6.5 h after the first: 2
"2026-09-18T14:00:00Z", # 3 -> given up
):
monkeypatch.setattr(update_info, "now_iso", lambda s=stamp: s)
update_info.main(["--out", "patch.json"])
write_patch("patch.json", read_json("patch.json"))
assert apply_patch.main(["patch.json"]) == 0
entry = read_json("scripts/state/ingest_state.json").get(url, {})
attempts.append((entry.get("attempts"), "gave_up" in entry))
assert attempts == [(1, False), (1, False), (1, False), (2, False), (3, True)]
assert read_json("scripts/temp_link.json") == []
def test_throttled_page_is_not_an_attempt(repo, monkeypatch):
seed_catalog([make_game("known")])
url = "https://dev.itch.io/dispatched"
for _ in range(4):
patch = _run(monkeypatch, {url: Page(status=429)}, [], INPUT_URL=url)
assert url not in patch["ingest_state"] and url not in patch["queue_remove"]
assert patch["queue_add"] == [url]
def _deleted(slug):
return {
"url": f"https://dev.itch.io/{slug}",
"name": slug,
"reason": "Game became paid",
"deleted_at": "2026-03-13T00:00:00Z",
}
def test_unblocked_game_that_is_free_again_leaves_the_deleted_log(repo, monkeypatch):
seed_catalog([make_game("known")])
url = "https://dev.itch.io/free"
save_json("scripts/deleted_games.json", [_deleted("free"), _deleted("other")])
save_json("scripts/state/unblocked.json", [url])
pages = {url: Page(status=200, soup=load_fixture_soup("game_free_full.html"))}
patch = _run(monkeypatch, pages, [url])
assert [a["url"] for a in patch["additions"]] == [url] and patch["unlog"] == [url]
_apply(patch)
assert [e["url"] for e in read_json("scripts/deleted_games.json")] == [
"https://dev.itch.io/other"
]
assert read_json("scripts/state/unblocked.json") == []
def test_unblocked_game_that_is_still_paid_keeps_its_removal_record(repo, monkeypatch):
seed_catalog([make_game("known")])
url = "https://dev.itch.io/paid"
save_json("scripts/deleted_games.json", [_deleted("paid")])
save_json("scripts/state/unblocked.json", [url])
pages = {url: Page(status=200, soup=load_fixture_soup("game_paid.html"))}
patch = _run(monkeypatch, pages, [url])
assert patch["additions"] == [] and patch["unblock_remove"] == [url]
_apply(patch)
assert [e["url"] for e in read_json("scripts/deleted_games.json")] == [url]
assert read_json("scripts/state/unblocked.json") == []
def test_deleted_game_without_unblock_is_still_skipped(repo, monkeypatch):
seed_catalog([make_game("known")])
save_json("scripts/deleted_games.json", [_deleted("paid")])
patch = _run(monkeypatch, {}, ["https://dev.itch.io/paid"])
assert patch["stats"]["deleted"] == 1 and patch["unlog"] == []