diff --git a/CLAUDE.md b/CLAUDE.md index 2e18cf1..c706fb3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,7 +75,7 @@ Each game has **20** fields (plus optional `added_at` / `updated_at`, appended b | Workflow | Trigger | Notes | |---|---|---| | `update.yml` | Push to `main` touching `scripts/temp_link.json` + daily 01:23 UTC + `workflow_dispatch` (`url`) | Ingest queue → patch → `commit_push.sh`. Workflow-level `concurrency: ingest` with **`queue: max`** (the default queue replaces a pending run, which would drop its dispatched `url`). A dispatched `url` is processed first and written to `temp_link.json` (`queue_add`) until it reaches a final outcome. Failed fetches count as an attempt only ≥ 6 h apart (`last_attempt`; 3 → given up), and a 429 counts as none. GitHub-App pushes (extension, admin Worker) trigger it; its own `GITHUB_TOKEN` push does not (no loop). Keep the filename. | -| `refresh.yml` | Daily 02:47 UTC + `workflow_dispatch` (`budget`, `url`, `allow_mass_removal`, `full`) | scan (read-only, 55-min step timeout, checks out the branch as it is when the job starts — not the queued SHA — so runs waiting in the queue see the previous run's state; patch artifact uploaded `if: always()`) → apply (`!cancelled()`: a failed / timed-out scan's checkpoint is applied, a cancelled run's is not) → notify. Workflow-level `concurrency: refresh-pipeline` + `queue: max` runs one whole scan → apply at a time (scheduled, manual and force-update runs alike). Removal needs the same 404/410 or paid strike seen again ≥ 20 h after the first sighting (`strike_at`). Checkpoints never carry removals. **Mass-change guard:** a run whose removals or new strikes exceed max(10, 10 %) of the games it checked withholds all its removals and marks them — and, on a strike spike, all its strikes — `withheld_at`; later runs keep withholding those (they take at most half a run's budget) until a healthy check clears them, and alert (`mass_change`) — that pattern means itch.io markup changed; dispatch with `allow_mass_removal` only after checking on itch.io. `apply_patch.py` re-checks the same per-run limit and max(25, 5 %) of the catalog unless `ALLOW_MASS_REMOVAL=1`. | +| `refresh.yml` | Daily 02:47 UTC + `workflow_dispatch` (`budget`, `url`, `allow_mass_removal`, `full`) | scan (read-only, 55-min step timeout, checks out the branch as it is when the job starts — not the queued SHA — so runs waiting in the queue see the previous run's state; patch artifact uploaded `if: always()`) → apply (`!cancelled()`: a failed / timed-out scan's checkpoint is applied, a cancelled run's is not) → notify. Workflow-level `concurrency: refresh-pipeline` + `queue: max` runs one whole scan → apply at a time (scheduled, manual and force-update runs alike). Removal needs the same 404/410, paid or `private` (itch.io's password form, `is_password_protected`; ingest drops such links as final) strike seen again ≥ 20 h after the first sighting (`strike_at`). Checkpoints never carry removals. **Mass-change guard:** a run whose removals or new strikes exceed max(10, 10 %) of the games it checked withholds all its removals and marks them — and, on a strike spike, all its strikes — `withheld_at`; later runs keep withholding those (they take at most half a run's budget) until a healthy check clears them, and alert (`mass_change`) — that pattern means itch.io markup changed; dispatch with `allow_mass_removal` only after checking on itch.io. `apply_patch.py` re-checks the same per-run limit and max(25, 5 %) of the catalog unless `ALLOW_MASS_REMOVAL=1`. | | `force_update.yml` | `workflow_dispatch` ONLY (`url`, `budget`, `allow_mass_removal`) | Dispatches `refresh.yml` with `full: true` (`actions: write`; it used to call it through `uses:`, which failed at startup with no annotation): re-scrapes every scraper field for one URL or the next batch (rotates on `full` timestamps). Dispatch it N times in a row to cover N consecutive batches (FIFO, each from the latest main). Always preserves `safe_virus` / `notes` / `nsfw`. Canonical tool for repairing mojibake. | | `notify-failure.yml` | `workflow_call` | Discord (`DISCORD_CI_WEBHOOK`) on failure / **cancelled** (job-level timeouts end as cancelled) / rate-limited / mass change / failed Cloudflare build. | | `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` / `npm run check:tauri` (gotcha 31) + `npm run cf:types -- --check` + lint + knip + vitest + build + `wrangler deploy --dry-run --env=""`. | @@ -98,7 +98,7 @@ git push origin vX.Y.Z gh release edit vX.Y.Z --draft=false ``` -Draft-first: **never** `gh release create` by hand before CI — tauri-action v1 refuses to upload into a published release, and announcements would go out before installers exist. Merge PRs into `main` with squash or a merge commit, never by fast-forward pushing a branch: its commits' preview-build check runs would then sit on main's own line and confuse `deploy-status.yml`. Tag from `main` only. Release notes go to `changelog/.x/ver..x.md` plus a row in the `CHANGELOG.md` index (each row keeps the anchor of the old single-file heading as ``: published release notes link to them). Versions live in `webapp/src/lib/about.ts` (`APP.version`), `webapp/package.json`, `webapp/src-tauri/tauri.conf.json` and `Cargo.toml` (+ `Cargo.lock`) — keep them equal. GPG signing is wired through gpg-agent (`commit.gpgsign=true`, so `git tag -s` and `git commit` both sign; the pinentry must be answered — in unattended sessions commits fail rather than going unsigned). The historical key `03F965C2E2DB5C6B` was exposed in the browser extension's storage (Sept 2026) and is being replaced — use the maintainer's current key. +Draft-first: **never** `gh release create` by hand before CI — tauri-action v1 refuses to upload into a published release, and announcements would go out before installers exist. Merge PRs into `main` with squash or a merge commit, never by fast-forward pushing a branch: its commits' preview-build check runs would then sit on main's own line and confuse `deploy-status.yml`. Tag from `main` only. Release notes go to `changelog/.x/ver..x.md` plus a row in the `CHANGELOG.md` index (each row keeps the anchor of the old single-file heading as ``: published release notes link to them). Versions live in `webapp/src/lib/about.ts` (`APP.version`), `webapp/package.json`, `webapp/src-tauri/tauri.conf.json` and `Cargo.toml` (+ `Cargo.lock`) — keep them equal. GPG signing is wired through gpg-agent (`commit.gpgsign=true`, so `git tag -s` and `git commit` both sign; the pinentry must be answered — in unattended sessions commits fail rather than going unsigned). Signing key since 2026-10-10: `838884804847BE7AE5FCF0A96AD1AB0921B96825` (ed25519 certify-only primary, generated locally and never published before; signing subkey `ED0BFD3005662D5D`; `user.signingkey` = the primary fingerprint, global + repo). The old key `03F965C2E2DB5C6B` is **revoked** (compromised: it sat in the browser extension's storage, Sept 2026); tags up to v4.3.0 and older commits carry it. The owner's offline backup and revocation certificates live outside the repo. ## Licenses & policies @@ -147,7 +147,7 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri ## Current state (Oct 2026) -**v4.3.0** (2026-10-10, PR #117; released the same day): itch.io's newer info rows in the catalog — `ai_disclosure` / `ai_content` (generative-AI disclosure) and `accessibility` — on the game page, as `/games` filters (`ai`, `access`) and as charts (Overview: "Generative AI"; Discovery: "Accessibility Features", "What AI Was Used For"); `description` = the creator's tagline. The whole catalog was re-scraped the same day with `force_update.yml` (6 × 600 + 10, ≈14.5 games/min; an itch.io outage ~05:20–06:50Z only cost retries): 3,236 of 3,251 games carry the new fields — No AI 53 %, AI Assisted 11 %, not disclosed 36 %; accessibility on 14 % (Subtitles 257, One button 119); the other 15 are 14 dead-page strikes awaiting confirmation and one password-protected page (`everium.itch.io/underthethorns`, a `parse_fail` on every check). Fixed on the way, same day: #118 (`force_update.yml` had never worked in v4 — it now dispatches `refresh.yml`, whose scan reads the latest main so queued batches rotate), #119 (`tauri-plugin-opener` 2.7: every first v4.3.0 release build had failed; the tag was moved before publishing), #120 (the 5 open Dependabot alerts; `check:tauri` CI guard; weekly cargo Dependabot), then Dependabot #122–#127. +**v4.3.0** (2026-10-10, PR #117; released the same day): itch.io's newer info rows in the catalog — `ai_disclosure` / `ai_content` (generative-AI disclosure) and `accessibility` — on the game page, as `/games` filters (`ai`, `access`) and as charts (Overview: "Generative AI"; Discovery: "Accessibility Features", "What AI Was Used For"); `description` = the creator's tagline. The whole catalog was re-scraped the same day with `force_update.yml` (6 × 600 + 10, ≈14.5 games/min; an itch.io outage ~05:20–06:50Z only cost retries): 3,236 of 3,251 games carry the new fields — No AI 53 %, AI Assisted 11 %, not disclosed 36 %; accessibility on 14 % (Subtitles 257, One button 119); the other 15 are 14 dead-page strikes awaiting confirmation and one password-protected page (`everium.itch.io/underthethorns`; it failed every check until #129 taught the refresh to strike such pages). Fixed on the way, same day: #118 (`force_update.yml` had never worked in v4 — it now dispatches `refresh.yml`, whose scan reads the latest main so queued batches rotate), #119 (`tauri-plugin-opener` 2.7: every first v4.3.0 release build had failed; the tag was moved before publishing), #120 (the 5 open Dependabot alerts; `check:tauri` CI guard; weekly cargo Dependabot), then Dependabot #122–#127. **v4.2.0** (2026-10-03, PR #103): catalog packs + lazy descriptions (first visit ≈ 264 KB instead of ≈ 520 KB), compressed IndexedDB cache, `/data/*.json` only for apps 4.0–4.1.1 (410 + repository links for everyone else), `/img` cache fixes, themed scrollbars + "Hide scrollbars", policies (`LEGAL_VERSION` 2026-10-03: data reuse goes through the repository). Checked on the live site the same day: TLS 1.2/1.3 only, HSTS preload header (domain not on the preload list), Cloudflare compression already on (zstd/br), Smart Tiered Cache on but irrelevant here (`docs/operations.md` §1 step 12). @@ -155,4 +155,4 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri **v4.1.0** (2026-09-24): the Turnstile verification gate for the web app, readable chart tooltips, the changelog split into `changelog/`, a docs refresh, updated Privacy/ToS/EULA (`LEGAL_VERSION` 2026-09-24). The gate ships switched on in `wrangler.jsonc` but stays inert until `GATE_BOT_TOKEN`, the Transform Rule and `GATE_SECRET` are in place (`docs/operations.md` §1 step 14) — `curl -s https://freeitchgames.win/api/verify` says `"enabled"`. -**v4.0.0** (2026-09-18) moved the site to the Cloudflare Worker at https://freeitchgames.win (PRs #81, #82, #88). The whole chain is live: Access for `/admin`, the GitHub App (verified commits to `scripts/temp_link.json` → `update.yml`), Turnstile, `ADMIN_EMAILS`; D1 `freeitchgames`, R2 `freeitchgames-thumbs`, Image Transformations (source `img.itch.zone`); Web Analytics is injected by the zone. `/api/ingest` stays unused — the browser extension is archived. **Still open (owner):** the legal-text review items; a new GPG signing key (commits are still signed with the exposed `03F965C2E2DB5C6B`); deleting the unused repo secrets `GH_TOKEN`, `TELEGRAM_BOT_TOKEN`, `DISCUSSION_*`. +**v4.0.0** (2026-09-18) moved the site to the Cloudflare Worker at https://freeitchgames.win (PRs #81, #82, #88). The whole chain is live: Access for `/admin`, the GitHub App (verified commits to `scripts/temp_link.json` → `update.yml`), Turnstile, `ADMIN_EMAILS`; D1 `freeitchgames`, R2 `freeitchgames-thumbs`, Image Transformations (source `img.itch.zone`); Web Analytics is injected by the zone. `/api/ingest` stays unused — the browser extension is archived. **Still open (owner):** the legal-text review items; add the new GPG key's public half on GitHub (until then new signatures show Unverified) and remove the revoked `03F965C2E2DB5C6B` there; deleting the unused repo secrets `GH_TOKEN`, `TELEGRAM_BOT_TOKEN`, `DISCUSSION_*`. diff --git a/README.md b/README.md index 6151422..fbd1aea 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ docs/ # Policies, dev environment, third-party list (+ Vietnam | Job | Schedule | Purpose | |---|---|---| | Ingest (`update.yml`) | On queue change + daily 01:23 UTC | Scrape approved links from `temp_link.json`, skip paid / dead / removed | -| Refresh (`refresh.yml`) | Daily 02:47 UTC | Re-check 1/7 of the catalog: dead links, now-paid games, rating, status | +| Refresh (`refresh.yml`) | Daily 02:47 UTC | Re-check 1/7 of the catalog: dead links, now-paid or password-protected games, rating, status | | Force update (`force_update.yml`) | Manual | Re-scrape every field (one URL or the next batch); keeps `safe_virus` / `notes` / `nsfw` | | RSS discovery (Cloudflare Worker) | Every 4 hours | Poll one itch.io feed per run; new games go to the review queue | | Python CI / Webapp CI | Pull requests (Python CI also on pushes to `main` that touch it) | Lint, tests, data validation / type-check, tests, build | diff --git a/README.vi.md b/README.vi.md index f252aa8..635b128 100644 --- a/README.vi.md +++ b/README.vi.md @@ -170,7 +170,7 @@ docs/ # Chính sách, môi trường dev, danh sách bên th | Tác vụ | Lịch | Mục đích | |---|---|---| | Ingest (`update.yml`) | Khi hàng chờ đổi + hằng ngày 01:23 UTC | Cào link đã duyệt từ `temp_link.json`, bỏ qua game trả phí / đã chết / từng bị gỡ | -| Refresh (`refresh.yml`) | Hằng ngày 02:47 UTC | Kiểm tra lại 1/7 danh mục: link chết, game chuyển sang trả phí, rating, trạng thái | +| Refresh (`refresh.yml`) | Hằng ngày 02:47 UTC | Kiểm tra lại 1/7 danh mục: link chết, game chuyển sang trả phí hoặc bị đặt mật khẩu, rating, trạng thái | | Force update (`force_update.yml`) | Thủ công | Cào lại mọi trường (một URL hoặc lô tiếp theo); giữ nguyên `safe_virus` / `notes` / `nsfw` | | Phát hiện qua RSS (Cloudflare Worker) | Mỗi 4 giờ | Đọc một feed itch.io mỗi lượt; game mới vào hàng chờ duyệt | | Python CI / Webapp CI | Pull request (Python CI còn chạy khi push vào `main` chạm tới phần Python) | Lint, test, validate dữ liệu / type-check, test, build | diff --git a/scripts/refresh.py b/scripts/refresh.py index d520193..5da5c99 100644 --- a/scripts/refresh.py +++ b/scripts/refresh.py @@ -10,6 +10,7 @@ then the least recently checked — about 1/7 of the catalog by default, so every game is re-checked weekly. From that single page it derives: - alive: 404/410 → "dead" strike - price: paid → "paid" strike + - access: itch.io's password form instead of the game → "private" strike - rating, rating_count, status, ai_disclosure / ai_content / accessibility (+ backfills release_date / thumbnail / updated_at when missing) A game is removed only when the same strike is seen again at least @@ -58,6 +59,7 @@ from scraper import ( find_title, has_info_panel, is_free_game, + is_password_protected, now_iso, parse_game, parse_info_table, @@ -80,12 +82,18 @@ MASS_FLOOR = 10 MASS_SHARE = 0.10 REASON_PAID = "Game became paid" +REASON_PRIVATE = "Game page is password-protected" def reason_dead(code: int) -> str: return f"Game page no longer exists (HTTP {code})" +def removal_kind(reason: str) -> str: + """The strike kind ("dead", "paid", "private") a removal reason was written for.""" + return {REASON_PAID: "paid", REASON_PRIVATE: "private"}.get(reason, "dead") + + # --------------------------------------------------------------------------- # Target selection # --------------------------------------------------------------------------- @@ -227,6 +235,8 @@ def check_game( return strike("dead", reason_dead(page.status)) soup = page.soup + if soup is not None and is_password_protected(soup): + return strike("private", REASON_PRIVATE) if soup is None or find_title(soup) == NA: entry["fails"] = entry.get("fails", 0) + 1 return "parse_fail", {}, None, entry @@ -281,7 +291,10 @@ def _write_outputs(stats: dict) -> None: ("Checked", stats["checked"]), ("Fields changed (games)", stats["changed"]), ("New strikes", stats["strikes"]), - ("Removed (dead / paid)", f"{stats['removed_dead']} / {stats['removed_paid']}"), + ( + "Removed (dead / paid / private)", + f"{stats['removed_dead']} / {stats['removed_paid']} / {stats['removed_private']}", + ), ("Removals withheld (mass-change guard)", stats["withheld"]), ("Errors / parse failures", f"{stats['errors']} / {stats['parse_fail']}"), ("HTTP 429/503", stats["http_429"]), @@ -309,7 +322,7 @@ def keep_struck(patch: dict, state: dict, removals: list[dict], stats: dict, mar entry["full"] = removal["deleted_at"] patch["refresh_state"][removal["url"]] = entry stats["removed"] -= 1 - stats["removed_" + ("paid" if removal["reason"] == REASON_PAID else "dead")] -= 1 + stats["removed_" + removal_kind(removal["reason"])] -= 1 patch["removals"] = [r for r in patch["removals"] if r["url"] not in urls] @@ -425,6 +438,7 @@ def main(argv: list[str] | None = None) -> int: "removed", "removed_dead", "removed_paid", + "removed_private", "errors", "parse_fail", "http_429", diff --git a/scripts/scraper.py b/scripts/scraper.py index 74da97d..b4479db 100644 --- a/scripts/scraper.py +++ b/scripts/scraper.py @@ -236,6 +236,14 @@ def has_info_panel(soup: BeautifulSoup) -> bool: return bool(wrapper and wrapper.find("table")) +def is_password_protected(soup: BeautifulSoup) -> bool: + """itch.io's "A password is required to view this page" form instead of the game (HTTP 200).""" + return bool( + soup.find("div", class_="game_password_page") + or soup.find("input", attrs={"name": "game_password"}) + ) + + # --------------------------------------------------------------------------- # Info-table parsing # --------------------------------------------------------------------------- diff --git a/scripts/update_info.py b/scripts/update_info.py index c2d1210..6aacef1 100644 --- a/scripts/update_info.py +++ b/scripts/update_info.py @@ -6,7 +6,8 @@ Flow: 2. Canonicalize; drop invalid URLs, duplicates, games already in the catalog and games in scripts/deleted_games.json (the admin restores a deleted game by removing its log entry when approving it again) - 3. Fetch each page → keep only free games (with an `added_at` stamp) + 3. Fetch each page → keep only free games (with an `added_at` stamp); a + password-protected page ends the link like a dead one 4. Emit a patch (see patch.py). URLs that finished (added / paid / dead / skipped) are dropped from the queue; transient failures stay queued and are retried; a link is given up after MAX_ATTEMPTS failures at least @@ -37,7 +38,7 @@ from data_store import get_all_urls, load_all_games from json_io import load_json from patch import new_patch, write_patch from refresh import fetch_with_backoff -from scraper import NA, Pacer, create_session, now_iso, parse_game +from scraper import NA, Pacer, create_session, is_password_protected, now_iso, parse_game TEMP_LINK = "scripts/temp_link.json" DELETED_LOG = "scripts/deleted_games.json" @@ -87,7 +88,18 @@ def main(argv: list[str] | None = None) -> int: print(f"Added from workflow input: {input_url}") stats = dict.fromkeys( - ("queued", "added", "paid", "dead", "duplicate", "deleted", "invalid", "retry", "gave_up"), + ( + "queued", + "added", + "paid", + "dead", + "private", + "duplicate", + "deleted", + "invalid", + "retry", + "gave_up", + ), 0, ) stats["queued"] = len(queue) @@ -146,7 +158,9 @@ def main(argv: list[str] | None = None) -> int: continue outcome = "" - if page.status == 200 and page.soup is not None: + if page.status == 200 and page.soup is not None and is_password_protected(page.soup): + outcome = "private" # final: nobody can open it, so retrying won't help + elif page.status == 200 and page.soup is not None: record = parse_game(page.soup, url) if record["name"] == NA: outcome = "error: no title" @@ -186,6 +200,7 @@ def main(argv: list[str] | None = None) -> int: write_patch(args.out, patch) print( f"\nDone — {stats['added']} added, {stats['paid']} paid, {stats['dead']} dead, " + f"{stats['private']} password-protected, " f"{stats['duplicate']} duplicates, {stats['deleted']} previously deleted, " f"{stats['invalid']} invalid, {stats['retry']} to retry, {stats['gave_up']} given up." ) diff --git a/tests/fixtures/game_password.html b/tests/fixtures/game_password.html new file mode 100644 index 0000000..b6160b4 --- /dev/null +++ b/tests/fixtures/game_password.html @@ -0,0 +1,22 @@ + + + + +Secret Garden by Quiet Studio - itch.io + + + +

itch.io

+
+
+

A password is required to view this page

+
+ +
+ +
+
+
+
+ + diff --git a/tests/test_parser.py b/tests/test_parser.py index d681c16..e8c36db 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -11,6 +11,7 @@ from scraper import ( detect_nsfw, extract_description, extract_tagline, + is_password_protected, parse_game, parse_retry_after, ) @@ -115,6 +116,12 @@ def test_page_without_info_panel(): assert record["genre"] == NA and record["tags"] == [] +def test_password_page_is_recognised(): + assert is_password_protected(load_fixture_soup("game_password.html")) + assert not is_password_protected(load_fixture_soup("game_free_full.html")) + assert not is_password_protected(load_fixture_soup("game_no_info.html")) + + def test_mature_notice_marks_nsfw(): record = parse_game(load_fixture_soup("game_nsfw_warning.html"), "https://x.itch.io/y") assert record["nsfw"] == "Yes" diff --git a/tests/test_refresh.py b/tests/test_refresh.py index 597306f..09f41b9 100644 --- a/tests/test_refresh.py +++ b/tests/test_refresh.py @@ -83,6 +83,23 @@ def test_paid_page_strike_then_removal(): assert outcome == "removed_paid" and removal["reason"] == "Game became paid" +def test_password_protected_page_strike_then_removal(): + game = make_game("a") + outcome, updates, removal, entry = refresh.check_game( + game, ok_page("game_password.html"), {"fails": 3}, NOW + ) + assert outcome == "strike_private" and removal is None and not updates + assert entry["strike"] == "private" and "fails" in entry + outcome, _, removal, _ = refresh.check_game( + game, ok_page("game_password.html"), entry, TOMORROW_NOW + ) + assert outcome == "removed_private" + assert removal["reason"] == "Game page is password-protected" + assert refresh.removal_kind(removal["reason"]) == "private" + assert refresh.removal_kind("Game became paid") == "paid" + assert refresh.removal_kind("Game page no longer exists (HTTP 404)") == "dead" + + def test_different_strike_kind_restarts_the_count(): game = make_game("a") entry = {"strike": "paid", "strike_at": "2026-09-10T03:00:00Z"} diff --git a/tests/test_update_info.py b/tests/test_update_info.py index 326501f..345355a 100644 --- a/tests/test_update_info.py +++ b/tests/test_update_info.py @@ -33,12 +33,16 @@ def test_ingest_outcomes(repo, monkeypatch): "https://dev.itch.io/free": Page(status=200, soup=load_fixture_soup("game_free_full.html")), "https://dev.itch.io/paid": Page(status=200, soup=load_fixture_soup("game_paid.html")), "https://dev.itch.io/dead": Page(status=404), + "https://dev.itch.io/locked": Page( + status=200, soup=load_fixture_soup("game_password.html") + ), } queue = [ "https://DEV.itch.io/free/", # canonicalized "https://dev.itch.io/free", # duplicate within the queue "https://dev.itch.io/paid", "https://dev.itch.io/dead", + "https://dev.itch.io/locked", # password-protected: final, not retried "https://dev.itch.io/known", "https://dev.itch.io/removed", "https://dev.itch.io/flaky", # 500 → retry later @@ -54,6 +58,8 @@ def test_ingest_outcomes(repo, monkeypatch): assert flaky["attempts"] == 1 and flaky["last_error"] == "error: HTTP 500" stats = patch["stats"] assert (stats["added"], stats["paid"], stats["dead"], stats["duplicate"]) == (1, 1, 1, 1) + # A password-protected page is final: dropped from the queue, no attempt kept. + assert stats["private"] == 1 and patch["ingest_state"]["https://dev.itch.io/locked"] is None assert (stats["deleted"], stats["invalid"], stats["retry"]) == (1, 1, 1) # apply_patch compares canonical forms, so each processed link is covered once. removed = {canonicalize(u) or u for u in patch["queue_remove"]} @@ -61,6 +67,7 @@ def test_ingest_outcomes(repo, monkeypatch): "https://dev.itch.io/free", "https://dev.itch.io/paid", "https://dev.itch.io/dead", + "https://dev.itch.io/locked", "https://dev.itch.io/known", "https://dev.itch.io/removed", "not a url", diff --git a/webapp/src/lib/analytics.ts b/webapp/src/lib/analytics.ts index 9717231..9f2b1e8 100644 --- a/webapp/src/lib/analytics.ts +++ b/webapp/src/lib/analytics.ts @@ -140,6 +140,7 @@ export function deletionsByMonth(deleted: DeletedGameEntry[]): CountEntry[] { export function deletionReasonCounts(deleted: DeletedGameEntry[]): CountEntry[] { let paid = 0 let removed = 0 + let locked = 0 let other = 0 for (const d of deleted) { const reason = (d.reason ?? '').toLowerCase() @@ -147,6 +148,8 @@ export function deletionReasonCounts(deleted: DeletedGameEntry[]): CountEntry[] paid += 1 } else if (reason.includes('no longer exists')) { removed += 1 + } else if (reason.includes('password-protected')) { + locked += 1 } else { other += 1 } @@ -154,6 +157,7 @@ export function deletionReasonCounts(deleted: DeletedGameEntry[]): CountEntry[] const out: CountEntry[] = [] if (paid) out.push({ key: 'Became paid', count: paid }) if (removed) out.push({ key: 'Page removed', count: removed }) + if (locked) out.push({ key: 'Password-protected', count: locked }) if (other) out.push({ key: 'Other', count: other }) return out } diff --git a/webapp/src/lib/i18n/en.ts b/webapp/src/lib/i18n/en.ts index 74ac598..0e9e434 100644 --- a/webapp/src/lib/i18n/en.ts +++ b/webapp/src/lib/i18n/en.ts @@ -351,6 +351,7 @@ export const en = { "charts.accessibility.title": "Accessibility Features", "charts.accessibility.desc": "Counted per game (a game may list several)", "charts.empty": "No data yet.", + "deleted.reason.private": "Page is password-protected", } as const export type MessageKey = keyof typeof en diff --git a/webapp/src/lib/i18n/vi.ts b/webapp/src/lib/i18n/vi.ts index 0632c4e..c6234f6 100644 --- a/webapp/src/lib/i18n/vi.ts +++ b/webapp/src/lib/i18n/vi.ts @@ -353,4 +353,5 @@ export const vi: Record = { "charts.accessibility.title": "Tính năng hỗ trợ tiếp cận", "charts.accessibility.desc": "Đếm theo từng game (một game có thể có nhiều tính năng)", "charts.empty": "Chưa có dữ liệu.", + "deleted.reason.private": "Trang đã bị đặt mật khẩu", } diff --git a/webapp/src/routes/deleted.tsx b/webapp/src/routes/deleted.tsx index 4029b42..cbae85b 100644 --- a/webapp/src/routes/deleted.tsx +++ b/webapp/src/routes/deleted.tsx @@ -13,6 +13,7 @@ import { useT, type MessageKey } from '@/lib/i18n' function reasonKey(reason: string): MessageKey | null { if (/no longer exists|HTTP 4(?:04|10)/i.test(reason)) return 'deleted.reason.gone' if (/became paid/i.test(reason)) return 'deleted.reason.paid' + if (/password-protected/i.test(reason)) return 'deleted.reason.private' return null }