Fix the SonarQube Cloud findings; release 4.4.2 (#155)

The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud.

Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh.

Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check.

Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41.

[skip-discuss]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 authored and GitHub committed 2026-10-22 18:16:12 +07:00
1 parent 9a67bbc536
commit 999d0887ee
44 files changed
+840 -77

No files matched your search

+9 -2
View File
@@ -10,6 +10,7 @@ on:
- 'tests/**'
- 'data_game/**'
- 'requirements*.txt'
- 'requirements*.in'
- 'pyproject.toml'
- '.github/python-version'
- '.github/workflows/python-ci.yml'
@@ -20,6 +21,7 @@ on:
- 'tests/**'
- 'data_game/**'
- 'requirements*.txt'
- 'requirements*.in'
- 'pyproject.toml'
- '.github/python-version'
- '.github/workflows/python-ci.yml'
@@ -49,10 +51,15 @@ jobs:
# supports Python <= 3.14 only (CLAUDE.md gotcha 39).
python-version-file: .github/python-version
cache: pip
cache-dependency-path: requirements-dev.txt
cache-dependency-path: |
requirements.txt
requirements-dev.txt
# Both hash-locked files at once: pip fails if their pins disagree.
# Wheels only, so no package runs a build script here.
- name: Install dependencies
run: pip install -r requirements-dev.txt
run: |
pip install --require-hashes --only-binary :all: -r requirements.txt -r requirements-dev.txt
- name: Ruff
run: |
+4 -2
View File
@@ -75,7 +75,8 @@ jobs:
cache-dependency-path: requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt
run: |
pip install --require-hashes --only-binary :all: -r requirements.txt
- name: Check a slice of the catalog
id: refresh
@@ -130,7 +131,8 @@ jobs:
cache-dependency-path: requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt
run: |
pip install --require-hashes --only-binary :all: -r requirements.txt
- name: Download patch
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+1 -1
View File
@@ -81,7 +81,7 @@ jobs:
- name: Install npm deps
working-directory: webapp
run: npm ci
run: npm ci --ignore-scripts
- name: Initialize Android project
working-directory: webapp
+1 -1
View File
@@ -121,7 +121,7 @@ jobs:
- name: Install npm deps
working-directory: webapp
run: npm ci
run: npm ci --ignore-scripts
- name: Build Tauri app
uses: tauri-apps/tauri-action@1deb371b0cd8bd54025b384f1cd735e725c4060f # action-v1.0.0
+2 -1
View File
@@ -55,7 +55,8 @@ jobs:
cache-dependency-path: requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt
run: |
pip install --require-hashes --only-binary :all: -r requirements.txt
- name: Scrape queued links
timeout-minutes: 50
+4 -2
View File
@@ -36,8 +36,10 @@ jobs:
cache: npm
cache-dependency-path: webapp/package-lock.json
# No install scripts: esbuild's and workerd's postinstall only check their
# platform binary, which their launchers find without it.
- name: Install dependencies
run: npm ci
run: npm ci --ignore-scripts
# The web build never runs `tauri build`, which refuses mismatched
# crate / npm versions; without this a Dependabot bump on one side
@@ -61,4 +63,4 @@ jobs:
run: npm run build
- name: Validate Workers config (dry run)
run: npx wrangler deploy --dry-run --env=""
run: npm run cf:check
+9
View File
@@ -0,0 +1,9 @@
# SonarQube Cloud automatic analysis
# (https://sonarcloud.io/project/overview?id=poli0981_free-games-itchio-list).
# Automatic analysis reads only a few settings from this file (no per-rule ignores):
# mark a false positive on the issue itself. See CLAUDE.md, gotcha 41.
# itch.io pages saved as served, for the scraper tests: their markup is
# itch.io's (tables without <th>, tabindex on divs, …), and editing it would
# make the tests check something itch.io never sends.
sonar.exclusions=tests/fixtures/**
+2 -1
View File
@@ -3,12 +3,13 @@
All notable changes to this project, newest first. The notes of each release live in
[`changelog/`](changelog/), one file per minor version: `changelog/<major>.x/ver<major>.<minor>.x.md`.
**Latest release: [4.4.1](changelog/4.x/ver4.4.x.md#441---2026-10-16-extension-and-bot-retired) (2026-10-16) — Extension and bot retired.**
**Latest release: [4.4.2](changelog/4.x/ver4.4.x.md#442---2026-10-22-toolchain-upgrade-code-scan-fixes) (2026-10-22) — Toolchain upgrade, code-scan fixes.**
## 4.x
| Version | Date | Highlights |
|---|---|---|
| [4.4.2](changelog/4.x/ver4.4.x.md#442---2026-10-22-toolchain-upgrade-code-scan-fixes) | 2026-10-22 | Toolchain upgrade (TypeScript 7, Node 26, Python 3.15), code-scan fixes |
| [4.4.1](changelog/4.x/ver4.4.x.md#441---2026-10-16-extension-and-bot-retired) | 2026-10-16 | Extension and bot retired (archived repos), ingest API removed |
| [4.4.0](changelog/4.x/ver4.4.x.md#440---2026-10-13-admin-automation-bulk-actions-smoother-charts) | 2026-10-13 | Admin automation, bulk actions, smoother charts |
| [4.3.0](changelog/4.x/ver4.3.x.md#430---2026-10-10-ai-disclosure-accessibility-refreshed-catalog) | 2026-10-10 | AI disclosure, accessibility, refreshed catalog |
+10 -5
View File
@@ -10,7 +10,7 @@ everyone; the old github.io URL only redirects). Three layers share one JSON cat
1. **Python pipeline** — the data layer (GitHub Actions).
- `data_game/game_info_NNN.json` — chunked records, max 500 per file, 3,251 games on 2026-10-10. Serialization is fixed: `json.dumps(..., ensure_ascii=False, indent=4)`, no trailing newline (`json_io.dumps`) — the Worker port (`webapp/worker/catalog.ts`) emits identical bytes (`tests/fixtures/golden/`).
- `data_game/index.json` — chunk manifest (total + per-file counts). `data_game/count_history.json` — date-keyed `{date,total}` series ("games over time" chart). `scripts/deleted_games.json` — removal log (public "Removed" page). `scripts/temp_link.json` — ingest queue.
- `scripts/` — scanners never write the catalog: `update_info.py` (queued URLs) and `refresh.py` (rotating 1/7-per-day check; `--full` re-scrape) emit a URL-keyed **patch** (`patch.py`); `apply_patch.py` applies it to the latest `main` and runs `validate.py`; `bash/commit_push.sh` does fetch → reset → apply → commit → push with retries. Shared: `scraper.py` (session, `Pacer` + 429 back-off, parsing: the info panel incl. itch.io's 2025–26 rows — `AI Disclosure` ("AI Assisted", then what for) / `Content` linking `/games/tag-no-ai` → `ai_disclosure` + `ai_content`, `Accessibility` → `accessibility`; `description` = the creator's tagline (`og:description`), else the first sentence), `data_store.py` (chunks stay in place, rebalance only past thresholds, write-if-changed), `json_io.py`, `canonical.py` (canonical itch.io URL; vectors in `tests/fixtures/url_vectors.json`, shared with the Worker). Pipeline bookkeeping lives in `scripts/state/` (not published), including `ingest_outcomes.json` (how each queued link ended — added / duplicate / paid / dead / private / deleted / gave_up; patch key `ingest_outcomes`; the last 90 days, ≤ 5,000) that the admin's tracking and the cron's reconcile read. Tests: `pytest` (network blocked). Lint: `ruff check scripts tests webapp/scripts`, `ruff format`, `vulture` (config in root `pyproject.toml`). Local env: Python 3.15 (`.github/python-version`, gotcha 39; ruff `target-version = "py315"`), `python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt` (3.15 is in UTF-8 mode by default — no `PYTHONUTF8` needed, the workflows no longer set it).
- `scripts/` — scanners never write the catalog: `update_info.py` (queued URLs) and `refresh.py` (rotating 1/7-per-day check; `--full` re-scrape) emit a URL-keyed **patch** (`patch.py`); `apply_patch.py` applies it to the latest `main` and runs `validate.py`; `bash/commit_push.sh` does fetch → reset → apply → commit → push with retries. Shared: `scraper.py` (session, `Pacer` + 429 back-off, parsing: the info panel incl. itch.io's 2025–26 rows — `AI Disclosure` ("AI Assisted", then what for) / `Content` linking `/games/tag-no-ai` → `ai_disclosure` + `ai_content`, `Accessibility` → `accessibility`; `description` = the creator's tagline (`og:description`), else the first sentence), `data_store.py` (chunks stay in place, rebalance only past thresholds, write-if-changed), `json_io.py`, `canonical.py` (canonical itch.io URL; vectors in `tests/fixtures/url_vectors.json`, shared with the Worker). Pipeline bookkeeping lives in `scripts/state/` (not published), including `ingest_outcomes.json` (how each queued link ended — added / duplicate / paid / dead / private / deleted / gave_up; patch key `ingest_outcomes`; the last 90 days, ≤ 5,000) that the admin's tracking and the cron's reconcile read. Tests: `pytest` (network blocked). Lint: `ruff check scripts tests webapp/scripts`, `ruff format`, `vulture` (config in root `pyproject.toml`). Local env: Python 3.15 (`.github/python-version`, gotcha 39; ruff `target-version = "py315"`), `python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt` (3.15 is in UTF-8 mode by default — no `PYTHONUTF8` needed, the workflows no longer set it). Both requirements files are hash-locked pip-tools compiles of `requirements*.in` — edit the `.in`, then recompile (gotcha 40).
2. **Cloudflare Worker + static assets** — `webapp/worker/` + `webapp/wrangler.jsonc`, deployed by **Cloudflare Workers Builds** from `main` (root dir `webapp`, build `npm run build`, deploy `npx wrangler deploy`; no deploy workflow). The catalog is bundled into the site at build time (`vite-plugins/catalog-data.ts` → `dist/data/pack/` compact packs + the legacy readable JSON in `dist/data/`, plus `data/urls.json` and `sitemap.xml`), so every data commit rebuilds the site.
3. **Web app + Tauri apps** — `webapp/src/` (React 19.3 + TypeScript 7.0 + Vite 8/rolldown + Tailwind + shadcn/ui + React Router 8), and the **Tauri 2 desktop + Android** read-only viewers in `webapp/src-tauri/` (same code; they fetch `https://freeitchgames.win/data/pack` and load covers straight from img.itch.zone). Plus `webapp/admin/` — the maintainer-only admin app (second Vite entry, served at `/admin/`).
@@ -27,6 +27,7 @@ npm run build # → webapp/dist/ (public app + /admin/ + /data + sitema
npm test # vitest (worker/, src/lib); worker/test/* D1 tests spawn a local workerd via getPlatformProxy
npm run lint && npm run knip && npx tsc -b
npm run cf:types # after editing wrangler.jsonc (ignores .dev.vars via types.env; CI runs it with --check)
npm run cf:check # wrangler deploy --dry-run with the project's wrangler (what CI runs; after a build)
npx wrangler d1 migrations apply freeitchgames --local # local D1 (use --remote by hand for production)
npm run tauri:dev # native window; needs Rust toolchain
npm run tauri:build # native installers
@@ -78,14 +79,14 @@ Each game has **20** fields (plus optional `added_at` / `updated_at`, appended b
| `refresh.yml` | Daily 02:47 UTC + `workflow_dispatch` (`budget`, `url`, `allow_mass_removal`, `full`) | scan (read-only, 55-min step timeout, checks out the branch as it is when the job starts — not the queued SHA — so runs waiting in the queue see the previous run's state; patch artifact uploaded `if: always()`) → apply (`!cancelled()`: a failed / timed-out scan's checkpoint is applied, a cancelled run's is not) → notify. Workflow-level `concurrency: refresh-pipeline` + `queue: max` runs one whole scan → apply at a time (scheduled, manual and force-update runs alike). Removal needs the same 404/410, paid or `private` (itch.io's password form, `is_password_protected`; ingest drops such links as final) strike seen again ≥ 20 h after the first sighting (`strike_at`). Checkpoints never carry removals. **Mass-change guard:** a run whose removals or new strikes exceed max(10, 10 %) of the games it checked withholds all its removals and marks them — and, on a strike spike, all its strikes — `withheld_at`; later runs keep withholding those (they take at most half a run's budget) until a healthy check clears them, and alert (`mass_change`) — that pattern means itch.io markup changed; dispatch with `allow_mass_removal` only after checking on itch.io. `apply_patch.py` re-checks the same per-run limit and max(25, 5 %) of the catalog unless `ALLOW_MASS_REMOVAL=1`. |
| `force_update.yml` | `workflow_dispatch` ONLY (`url`, `budget`, `allow_mass_removal`) | Dispatches `refresh.yml` with `full: true` (`actions: write`; it used to call it through `uses:`, which failed at startup with no annotation): re-scrapes every scraper field for one URL or the next batch (rotates on `full` timestamps). Dispatch it N times in a row to cover N consecutive batches (FIFO, each from the latest main). Always preserves `safe_virus` / `notes` / `nsfw`. Canonical tool for repairing mojibake. |
| `notify-failure.yml` | `workflow_call` | Discord (`DISCORD_CI_WEBHOOK`) on failure / **cancelled** (job-level timeouts end as cancelled) / rate-limited / mass change / failed Cloudflare build. |
| `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` / `npm run check:tauri` (gotcha 31) + `npm run cf:types -- --check` + lint + knip + vitest + build + `wrangler deploy --dry-run --env=""`. |
| `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` (both requirements locks installed at once, so drifting pins fail) / `npm ci --ignore-scripts` + `npm run check:tauri` (gotcha 31) + `npm run cf:types -- --check` + lint + knip + vitest + build + `npm run cf:check` (`wrangler deploy --dry-run`). A SonarQube Cloud check also runs on PRs (gotcha 41). |
| `deploy-status.yml` | Hourly + manual | Not a deploy. `bash/cf_build_status.sh` walks main's first-parent history to the latest finished `Workers Builds: free-games-itchio-list` check run (Cloudflare posts it only when a build ends; branch commits carry preview builds under the same name) and alerts once per failed build (cache key per check-run id, saved after the Discord post). Scheduled because GitHub never starts `check_run` workflows for `GITHUB_TOKEN` pushes (the data commits). |
| `release_desktop.yml` | Tag `v*` push, or manual | `create-release` makes the single draft (id passed to tauri-action v1 `releaseId`); release builds use **no** npm/cargo caches (cache-poisoning). Win + macOS aarch64 + macOS x86_64 (cross-compile) + Linux (`ubuntu-22.04` for a low glibc floor). Uploads `.dmg` / `.app.tar.gz` / `.pkg` (macOS), `.msi` / `.exe` (Windows), `.deb` / `.rpm` / `.AppImage` (Linux). macOS `.pkg` `--identifier` reads `tauri.conf.json`. |
| `release_android.yml` | Tag `v*` push, or manual | Tauri **Android** build → arm64-v8a APK, **post-signed** with `zipalign`+`apksigner` (gen/ stays gitignored). Tag → attaches to the same draft Release; manual dispatch → workflow artifact. Secrets: `ANDROID_KEYSTORE_BASE64` / `ANDROID_KEYSTORE_PASSWORD` / `ANDROID_KEY_ALIAS`. |
| `announce-release.yml`, `notify-release-pipeline.yml` | Release published / pipeline runs | Reusable workflows from `poli0981/.github`, SHA-pinned — but at that pin they still check out their notifier script from that repo's `main`. |
| `announce-discussion.yml` | Release published; push to `main` touching `CHANGELOG.md` / `changelog/**` | Local. The release goes to Discussions → Announcements, a changelog push to → General unless the head commit says `[skip-discuss]`. |
All workflows: `permissions: {}` at the top, per-job grants, actions pinned by commit SHA.
All workflows: `permissions: {}` at the top, per-job grants, actions pinned by commit SHA. Python installs are hash-locked and wheels only (`pip install --require-hashes --only-binary :all:`, gotcha 40); npm installs skip install scripts (`npm ci --ignore-scripts` — esbuild's and workerd's launchers find their platform binary without them).
## Release / tag process
@@ -145,17 +146,21 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri
37. **D1 counts every statement of a batch** towards its 1,000 queries per invocation: bind URL lists in `IN (…)` slices of ≤ 90 (100 bound parameters at most), as `setStatus` / `reconcile` / `markIngested` do, never one statement per URL.
38. **npm 11.20+ (Node 26) gates dependency install scripts on `allowScripts` in `webapp/package.json`.** esbuild and workerd are approved by name (their postinstall only checks the platform binary). A new dependency with an install script shows up in `npm install-scripts ls`; review it, then `npm install-scripts approve <pkg> --no-allow-scripts-pin` (name-only, so Dependabot bumps stay covered).
39. **Dependabot's pip updater reads `.python-version` and aborts on a Python it doesn't support** (3.10–3.14 as of Oct 2026): a `3.15` there stopped pip version updates and the pip dependency graph ("Dependabot detected the following Python requirement … '3.15.*'"). So the CI pin lives in `.github/python-version`, which only `setup-python`'s `python-version-file` reads (python-ci, refresh ×2, update); move it back to `.python-version` once Dependabot lists 3.15.
40. **The Python requirements are hash-locked pip-tools compiles** — `requirements.txt` ← `requirements.in`, `requirements-dev.txt` ← `requirements-dev.in` (`-c requirements.txt`) — installed everywhere with `--require-hashes --only-binary :all:`, so a new release of any dependency, indirect ones included, can't reach the jobs that push to `main`, and no sdist build script runs. After editing a `.in`, recompile with **pip-tools on Python 3.14**: `pip-compile --generate-hashes --allow-unsafe --strip-extras requirements.in`, then the same for `requirements-dev.in`. Dependabot takes the Python version from the `# … autogenerated by pip-compile with Python X.Y` header and supports ≤ 3.14 (gotcha 39); the hashes cover every file of a release, so CI's 3.15 still finds its cp315 wheels. `colorama` is listed in `requirements-dev.in` because pytest needs it on Windows only: a lock compiled on Linux (Dependabot's) would otherwise leave it out and the hash-checked local install would fail. Keep those pip commands in a block scalar (`run: |`): in a plain YAML scalar the `: ` of `:all: -r` makes the whole workflow file invalid — GitHub then records a 0-job failed push run named after the file, and a PR simply shows no check for that workflow.
41. **SonarQube Cloud runs Automatic Analysis** (project `poli0981_free-games-itchio-list`, public; PRs get a "SonarCloud Code Analysis" check). It reads only `.sonarcloud.properties` (sources / tests / exclusions — no per-rule ignores; mark a false positive on the issue itself). Two of its suggested fixes are wrong here: S2871 wants `localeCompare` in `.sort()`, but chunk names, migration files and `unblocked.json` keep code-unit order (= Python's `sorted()`) — pass `compareCodeUnits` (`worker/text.ts`); S8906 wants `select_one('div.a.b')` for bs4's `class_=[a, b]`, which means *either* class — write a selector list `div.a, div.b`. Trim trailing characters with `trimEnd()` (`worker/text.ts`), not `/x+$/` (S8786: quadratic backtracking on a long run). Known false positive: css:S8776 on Tailwind v4's `@utility … { &::-webkit-scrollbar … }` (the `&` is required for variants).
## Safe-edit rules
- Don't write `data_game/` from new code paths directly — go through a patch + `apply_patch.py` (or the Worker's `repo.ts`), so concurrent writers never clobber each other and `validate.py` gates every commit.
- Don't commit `webapp/dist/`, `webapp/src-tauri/target/`, `webapp/src-tauri/gen/`, `webapp/src-tauri/icon-source.png`, `.dev.vars`, `username.txt`, `assets/qr/`. `Cargo.lock` **is** committed (since v4).
- Never commit secrets (GitHub App key, admin emails, Turnstile secret, tokens). Data workflows push with the job's `GITHUB_TOKEN`; the Worker writes as the GitHub App.
- When adding an npm dep, also add it to `webapp/src/lib/about.ts` so the About page lists it.
- When adding an npm dep, also add it to `webapp/src/lib/about.ts` so the About page lists it. A Python dep goes into `requirements*.in`, then recompile the locks (gotcha 40).
## Current state (Oct 2026)
**Toolchain** (2026-10-22, on `main`, not released yet — goes into the next release's notes): TypeScript 7.0.2 (`@typescript/native`; `typescript` = the 6.0 API alias, gotcha 6), Node 26 (`webapp/.node-version`, `engines >=26`, `@types/node` 26, Workers Builds `NODE_VERSION` raised to 26 by the owner in the dashboard — `docs/operations.md` §1 step 1), Python 3.15 (`.github/python-version` — gotcha 39, ruff `py315`, `urllib3` declared since `scraper.py` imports it). No source change was needed: TS 7 reports 0 errors, the suites pass on Node 26.12 and Python 3.15.1.
**Toolchain** (2026-10-22, #149 + #154, shipped in 4.4.2): TypeScript 7.0.2 (`@typescript/native`; `typescript` = the 6.0 API alias, gotcha 6), Node 26 (`webapp/.node-version`, `engines >=26`, `@types/node` 26, Workers Builds `NODE_VERSION` raised to 26 by the owner in the dashboard — `docs/operations.md` §1 step 1), Python 3.15 (`.github/python-version` — gotcha 39, ruff `py315`, `urllib3` declared since `scraper.py` imports it). No source change was needed: TS 7 reports 0 errors, the suites pass on Node 26.12 and Python 3.15.1.
**v4.4.2** (2026-10-22): SonarQube Cloud cleanup — the scan's 57 issues: 45 fixed in code, the 11 on the saved itch.io test pages excluded by `.sonarcloud.properties`, css:S8776 (Tailwind `@utility`) left for the owner to mark a false positive (gotcha 41). Worker paths and the gate cookie's padding are trimmed by `trimEnd()` (a `/\/+$/` on the public 404 path was quadratic), the feed / collection parsers have no backtracking regexes left, `compareCodeUnits` makes the code-unit sorts explicit, the NSFW notices are one selector list (each tested). Supply chain: hash-locked Python deps + wheels only in every workflow (gotcha 40), `npm ci --ignore-scripts` in CI and the release builds, `npm run cf:check` instead of `npx wrangler`. `scripts/i18n-edit.mjs` reads its operations from stdin (`node scripts/i18n-edit.mjs < ops.json`). Ships the toolchain upgrade and Dependabot #141–#148, #150, #152, #153.
**v4.4.1** (2026-10-16): the browser extension and the Telegram bot are retired — both repos (`poli0981/itch-f2p-extension`, `poli0981/telegram-scraper-bot`) were already public archives; an Announcements discussion says so. Removed: `POST /api/ingest` (`ingest.ts`, `requireService`, `RL_INGEST`, `ACCESS_AUD_INGEST`, the idempotency store — it never received a game: 0 `ext` rows in production D1), the admin's "Browser extension" auto-approve source and "Extension" label, the About page's Telegram numeric-ID note. The policies (Privacy, ToS, EULA + installer RTF, Disclaimer, Security; EN + VI) only lost the extension / ingest wording: "Last updated" 2026-10-16, `LEGAL_VERSION` deliberately unchanged (nothing new to accept, ToS §14). Unused repo secrets `GH_TOKEN`, `TELEGRAM_BOT_TOKEN`, `DISCUSSION_*` deleted. Ships Dependabot #131 (vite 8.3.4), #135 (tauri crates), #136 (serde_json), #137 (ruff).
+5 -5
View File
@@ -36,7 +36,7 @@ api() {
}
output() {
if [ -n "${GITHUB_OUTPUT:-}" ]; then echo "$1" >>"$GITHUB_OUTPUT"; fi
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "$1" >>"$GITHUB_OUTPUT"; fi
}
commits=$(api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" \
@@ -44,14 +44,14 @@ commits=$(api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" \
declare -A parent=()
sha=""
while read -r commit first_parent; do
[ -n "$commit" ] || continue
[ -n "$sha" ] || sha=$commit
[[ -n "$commit" ]] || continue
[[ -n "$sha" ]] || sha=$commit
parent[$commit]=$first_parent
done <<<"$commits"
now=$(date -u +%s)
for _ in $(seq 1 "$MAX_COMMITS"); do
[ -n "$sha" ] && [ -n "${parent[$sha]+set}" ] || break
[[ -n "$sha" && -n "${parent[$sha]+set}" ]] || break
run=$(api --method GET "repos/$GITHUB_REPOSITORY/commits/$sha/check-runs" \
-f check_name="$CHECK_NAME" -f filter=latest --jq \
'[.check_runs[] | select(.app.slug == "cloudflare-workers-and-pages")] | first
@@ -67,7 +67,7 @@ for _ in $(seq 1 "$MAX_COMMITS"); do
;;
*)
age_h=$(((now - $(date -u -d "$completed" +%s)) / 3600))
if [ "$age_h" -lt "$WINDOW_H" ]; then
if [[ "$age_h" -lt "$WINDOW_H" ]]; then
echo "::error::The Cloudflare build of ${sha:0:7} ended '$conclusion' — freeitchgames.win is not up to date: $url"
output "failed_id=$id"
output "failed_url=$url"
+48 -1
View File
@@ -1,6 +1,53 @@
# Changelog — 4.4.x
Release notes for 4.4.0 and 4.4.1, newest first. Every version: [CHANGELOG.md](../../CHANGELOG.md).
Release notes for 4.4.0 to 4.4.2, newest first. Every version: [CHANGELOG.md](../../CHANGELOG.md).
## [4.4.2] - 2026-10-22 (Toolchain upgrade, code-scan fixes)
A maintenance release: nothing you would notice changes on the site or in the apps. The project now
builds with TypeScript 7, Node 26 and Python 3.15, and the issues a SonarQube Cloud scan of the
repository reported are fixed — the one that mattered was a path check in the Worker's not-found
handler that a crafted address could make expensive.
### Upgrade notes
- **Nothing to accept again**: no policy changed.
- **Desktop and Android apps**: install 4.4.2 over 4.4.1 as usual.
- **Building or running your own copy?** Use Node 26 for the web app and Python 3.15 for the
pipeline. The Python dependencies are now pinned with hashes: `pip install -r requirements-dev.txt`
works as before; to change one, edit `requirements.in` or `requirements-dev.in` and recompile with
pip-tools (the command is at the top of each file).
### Fixed
- **Worker**: trailing slashes on request paths (not-found handler, admin routes, the Access issuer)
and the verification cookie's base64 padding are trimmed by a loop instead of a regular expression
that backtracked — a path with a long run of slashes cost quadratic CPU time on a public route. The
itch.io feed and collection parsers lost three more expressions of that kind; a collection link's
attributes are now read by splitting on quotes.
- **Pipeline**: itch.io's two mature-content notices are matched by one CSS selector, so either of
them still marks a game NSFW (each now has a test). A few checks are written out so the scanner can
follow them, without a change in behaviour, and the scraper no longer sets up plain-http requests
(every page it fetches is https).
### Changed
- **Build tooling** (#149, #154): TypeScript 7.0, the native compiler, runs the type check, with the
6.0 API installed beside it for typescript-eslint. Node 26 builds the web app and the release apps,
and npm runs only the install scripts of esbuild and workerd. Python 3.15 runs the pipeline (UTF-8
mode is now the default); its CI pin moved to `.github/python-version`, out of Dependabot's way.
- **Supply chain**: the pipeline's Python dependencies, indirect ones included, are locked with hashes
(`requirements.txt`, `requirements-dev.txt`, compiled from `requirements*.in`) and installed from
wheels only, so a new release of a dependency can no longer slip into the jobs that commit to the
catalog. CI and the release builds install npm packages with `--ignore-scripts`, and the Workers
config check runs the project's own wrangler (`npm run cf:check`).
- **Code scanning**: `.sonarcloud.properties` keeps the saved itch.io test pages out of the scan;
the maintainer's `i18n-edit` script reads its operations from stdin instead of a file path.
- **Dependencies**: React 19.3.1 (#141), TanStack Query 5.104.4 (#142, #150), Radix UI primitives
(#143), `@tauri-apps/api` 2.12.4, CLI 2.12.3 and `plugin-opener` 2.7.2 (#144) with the matching
crates `tauri-build` 2.7.3, `tauri-plugin-opener` 2.7.2 and `tauri-plugin-single-instance` 2.5.4
(#146), lucide-react 1.59.0 (#152), Vitest 5.0.5 (#153), Ruff 0.17.1 and Pillow 13 (#147),
`actions/setup-java` 6.1.0 and `dtolnay/rust-toolchain` in the release workflows (#148).
## [4.4.1] - 2026-10-16 (Extension and bot retired)
+6 -1
View File
@@ -47,6 +47,11 @@ pip install -r requirements-dev.txt
Python 3.15 reads and writes files as UTF-8 by default (UTF-8 mode, PEP 686), so Windows no longer
needs `PYTHONUTF8=1`.
Both requirements files pin every package with its hashes; pip checks them on install. To add or
change a dependency, edit `requirements.in` or `requirements-dev.in` and recompile the lock with
pip-tools — the command is at the top of each `.in` file (run it on Python 3.14, which Dependabot
can read; CLAUDE.md, gotcha 40).
### Checks (same as `python-ci.yml`)
```sh
@@ -171,7 +176,7 @@ npm run knip # unused files / exports / dependencies
npm test # Vitest: app + Worker; D1 tests start a local workerd
npx tsc -b # type-check only (npm run build runs it too)
npm run build
npx wrangler deploy --dry-run --env="" # validates the Worker config, deploys nothing
npm run cf:check # wrangler deploy --dry-run: validates the Worker config, deploys nothing
```
After editing `wrangler.jsonc`, run `npm run cf:types` and commit the regenerated
+5 -1
View File
@@ -49,6 +49,10 @@ pip install -r requirements-dev.txt
Python 3.15 mặc định đọc và ghi file dưới dạng UTF-8 (UTF-8 mode, PEP 686), nên trên Windows không
cần đặt `PYTHONUTF8=1` nữa.
Hai file requirements ghim mọi package kèm hash; pip kiểm tra hash khi cài. Muốn thêm hoặc đổi một
dependency, sửa `requirements.in` hoặc `requirements-dev.in` rồi compile lại lock bằng pip-tools —
lệnh nằm ở đầu mỗi file `.in` (chạy bằng Python 3.14, bản Dependabot đọc được; CLAUDE.md, gotcha 40).
### Kiểm tra (giống `python-ci.yml`)
```sh
@@ -173,7 +177,7 @@ npm run knip # file / export / dependency không dùng
npm test # Vitest: app + Worker; test D1 khởi động một workerd local
npx tsc -b # chỉ kiểm tra kiểu (npm run build cũng chạy bước này)
npm run build
npx wrangler deploy --dry-run --env="" # kiểm tra cấu hình Worker, không deploy gì
npm run cf:check # wrangler deploy --dry-run: kiểm tra cấu hình Worker, không deploy gì
```
Sau khi sửa `wrangler.jsonc`, chạy `npm run cf:types` và commit file `worker-configuration.d.ts` vừa
+11
View File
@@ -0,0 +1,11 @@
# Lint and test tools on top of the runtime pins (`-c`). requirements-dev.txt is
# the hash-locked compile of this file; regenerate it after an edit (CLAUDE.md, gotcha 40):
# pip-compile --generate-hashes --allow-unsafe --strip-extras requirements-dev.in
-c requirements.txt
-r requirements.in
ruff>=0.17.1
pytest>=9.1.1
vulture>=2.16
pillow>=13.0.0
# pytest needs it on Windows only; listed so a lock compiled on Linux (Dependabot) still installs there.
colorama
+360 -5
View File
@@ -1,5 +1,360 @@
-r requirements.txt
ruff>=0.17.1
pytest>=9.1.1
vulture>=2.16
pillow>=13.0.0
#
# This file is autogenerated by pip-compile with Python 3.14
# by the following command:
#
# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements-dev.in
#
beautifulsoup4==4.15.0 \
--hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \
--hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9
# via
# -c requirements.txt
# -r requirements.in
certifi==2026.10.19 \
--hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \
--hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b
# via
# -c requirements.txt
# requests
charset-normalizer==3.5.2 \
--hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \
--hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \
--hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \
--hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \
--hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \
--hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \
--hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \
--hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \
--hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \
--hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \
--hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \
--hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \
--hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \
--hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \
--hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \
--hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \
--hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \
--hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \
--hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \
--hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \
--hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \
--hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \
--hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \
--hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \
--hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \
--hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \
--hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \
--hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \
--hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \
--hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \
--hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \
--hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \
--hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \
--hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \
--hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \
--hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \
--hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \
--hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \
--hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \
--hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \
--hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \
--hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \
--hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \
--hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \
--hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \
--hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \
--hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \
--hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \
--hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \
--hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \
--hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \
--hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \
--hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \
--hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \
--hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \
--hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \
--hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \
--hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \
--hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \
--hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \
--hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \
--hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \
--hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \
--hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \
--hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \
--hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \
--hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \
--hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \
--hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \
--hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \
--hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \
--hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \
--hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \
--hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \
--hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \
--hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \
--hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \
--hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \
--hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \
--hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \
--hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \
--hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \
--hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \
--hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \
--hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \
--hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \
--hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \
--hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \
--hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \
--hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \
--hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \
--hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \
--hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \
--hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \
--hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \
--hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \
--hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \
--hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \
--hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \
--hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \
--hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \
--hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \
--hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \
--hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \
--hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \
--hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \
--hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \
--hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \
--hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \
--hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \
--hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \
--hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \
--hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \
--hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \
--hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \
--hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \
--hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \
--hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \
--hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \
--hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \
--hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \
--hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \
--hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \
--hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \
--hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \
--hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \
--hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \
--hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \
--hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \
--hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \
--hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \
--hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \
--hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \
--hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \
--hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \
--hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \
--hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \
--hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \
--hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \
--hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \
--hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \
--hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \
--hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \
--hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \
--hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \
--hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \
--hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \
--hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \
--hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \
--hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \
--hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \
--hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \
--hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \
--hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \
--hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \
--hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \
--hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \
--hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \
--hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \
--hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \
--hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \
--hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \
--hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \
--hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \
--hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \
--hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \
--hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \
--hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \
--hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \
--hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \
--hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \
--hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f
# via
# -c requirements.txt
# requests
colorama==0.4.6 \
--hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
--hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
# via
# -r requirements-dev.in
# pytest
idna==3.20 \
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
# via
# -c requirements.txt
# requests
iniconfig==2.3.1 \
--hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \
--hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7
# via pytest
packaging==26.3 \
--hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \
--hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
# via pytest
pillow==13.0.0 \
--hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \
--hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \
--hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \
--hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \
--hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \
--hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \
--hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \
--hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \
--hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \
--hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \
--hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \
--hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \
--hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \
--hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \
--hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \
--hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \
--hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \
--hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \
--hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \
--hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \
--hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \
--hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \
--hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \
--hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \
--hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \
--hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \
--hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \
--hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \
--hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \
--hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \
--hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \
--hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \
--hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \
--hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \
--hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \
--hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \
--hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \
--hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \
--hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \
--hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \
--hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \
--hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \
--hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \
--hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \
--hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \
--hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \
--hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \
--hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \
--hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \
--hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \
--hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \
--hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \
--hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \
--hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \
--hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \
--hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \
--hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \
--hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \
--hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \
--hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \
--hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \
--hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \
--hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \
--hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \
--hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \
--hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \
--hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \
--hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \
--hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \
--hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \
--hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \
--hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \
--hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \
--hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \
--hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \
--hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \
--hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \
--hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \
--hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \
--hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \
--hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \
--hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \
--hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe
# via -r requirements-dev.in
pluggy==1.7.0 \
--hash=sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec \
--hash=sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8
# via pytest
pygments==2.21.0 \
--hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \
--hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c
# via pytest
pytest==9.1.1 \
--hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \
--hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c
# via -r requirements-dev.in
requests==2.34.2 \
--hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \
--hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed
# via
# -c requirements.txt
# -r requirements.in
ruff==0.17.1 \
--hash=sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca \
--hash=sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a \
--hash=sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a \
--hash=sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74 \
--hash=sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4 \
--hash=sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640 \
--hash=sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f \
--hash=sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7 \
--hash=sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785 \
--hash=sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1 \
--hash=sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05 \
--hash=sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab \
--hash=sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978 \
--hash=sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb \
--hash=sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467 \
--hash=sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189 \
--hash=sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4 \
--hash=sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a
# via -r requirements-dev.in
soupsieve==3.0.4 \
--hash=sha256:8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf \
--hash=sha256:924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0
# via
# -c requirements.txt
# beautifulsoup4
typing-extensions==4.16.0 \
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
# via
# -c requirements.txt
# beautifulsoup4
urllib3==2.8.0 \
--hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \
--hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63
# via
# -c requirements.txt
# -r requirements.in
# requests
vulture==2.16 \
--hash=sha256:6e0f1c312cef1c87856957e5c2ca9608834a7c794c2180477f30bf0e4cc58eee \
--hash=sha256:f8d9f6e2af03011664a3c6c240c9765b3f392917d3135fddca6d6a68d359f717
# via -r requirements-dev.in
+6
View File
@@ -0,0 +1,6 @@
# Runtime dependencies of the data pipeline (scripts/). requirements.txt is the
# hash-locked compile of this file; regenerate it after an edit (CLAUDE.md, gotcha 40):
# pip-compile --generate-hashes --allow-unsafe --strip-extras requirements.in
requests>=2.34.2,<3
beautifulsoup4>=4.15.0,<5
urllib3>=2.8.0,<3
+210 -3
View File
@@ -1,3 +1,210 @@
requests>=2.34.2,<3
beautifulsoup4>=4.15.0,<5
urllib3>=2.8.0,<3
#
# This file is autogenerated by pip-compile with Python 3.14
# by the following command:
#
# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements.in
#
beautifulsoup4==4.15.0 \
--hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \
--hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9
# via -r requirements.in
certifi==2026.10.19 \
--hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \
--hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b
# via requests
charset-normalizer==3.5.2 \
--hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \
--hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \
--hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \
--hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \
--hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \
--hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \
--hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \
--hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \
--hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \
--hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \
--hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \
--hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \
--hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \
--hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \
--hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \
--hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \
--hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \
--hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \
--hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \
--hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \
--hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \
--hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \
--hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \
--hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \
--hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \
--hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \
--hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \
--hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \
--hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \
--hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \
--hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \
--hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \
--hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \
--hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \
--hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \
--hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \
--hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \
--hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \
--hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \
--hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \
--hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \
--hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \
--hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \
--hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \
--hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \
--hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \
--hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \
--hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \
--hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \
--hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \
--hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \
--hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \
--hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \
--hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \
--hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \
--hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \
--hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \
--hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \
--hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \
--hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \
--hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \
--hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \
--hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \
--hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \
--hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \
--hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \
--hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \
--hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \
--hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \
--hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \
--hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \
--hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \
--hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \
--hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \
--hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \
--hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \
--hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \
--hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \
--hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \
--hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \
--hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \
--hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \
--hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \
--hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \
--hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \
--hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \
--hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \
--hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \
--hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \
--hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \
--hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \
--hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \
--hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \
--hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \
--hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \
--hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \
--hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \
--hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \
--hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \
--hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \
--hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \
--hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \
--hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \
--hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \
--hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \
--hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \
--hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \
--hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \
--hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \
--hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \
--hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \
--hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \
--hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \
--hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \
--hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \
--hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \
--hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \
--hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \
--hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \
--hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \
--hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \
--hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \
--hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \
--hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \
--hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \
--hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \
--hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \
--hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \
--hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \
--hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \
--hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \
--hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \
--hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \
--hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \
--hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \
--hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \
--hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \
--hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \
--hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \
--hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \
--hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \
--hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \
--hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \
--hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \
--hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \
--hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \
--hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \
--hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \
--hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \
--hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \
--hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \
--hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \
--hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \
--hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \
--hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \
--hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \
--hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \
--hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \
--hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \
--hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \
--hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \
--hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \
--hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \
--hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \
--hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \
--hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \
--hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \
--hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \
--hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \
--hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \
--hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \
--hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f
# via requests
idna==3.20 \
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
# via requests
requests==2.34.2 \
--hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \
--hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed
# via -r requirements.in
soupsieve==3.0.4 \
--hash=sha256:8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf \
--hash=sha256:924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0
# via beautifulsoup4
typing-extensions==4.16.0 \
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
# via beautifulsoup4
urllib3==2.8.0 \
--hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \
--hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63
# via
# -r requirements.in
# requests
+13 -6
View File
@@ -77,8 +77,8 @@ def create_session() -> requests.Session:
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry)
# Every URL we fetch is canonical https (canonical.py); no adapter for plain http.
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
@@ -233,7 +233,9 @@ def find_title(soup: BeautifulSoup) -> str:
def has_info_panel(soup: BeautifulSoup) -> bool:
wrapper = soup.find("div", class_="info_panel_wrapper")
return bool(wrapper and wrapper.find("table"))
if not wrapper:
return False
return wrapper.find("table") is not None
def is_password_protected(soup: BeautifulSoup) -> bool:
@@ -370,7 +372,9 @@ def _cap(text: str) -> str:
def extract_tagline(soup: BeautifulSoup) -> str:
"""The creator's "short description or tagline" (itch.io puts it in og:description)."""
meta = soup.find("meta", property="og:description")
text = " ".join(str(meta.get("content", "")).split()) if meta else ""
if not meta:
return NA
text = " ".join(str(meta.get("content", "")).split())
return _cap(text) if text else NA
@@ -399,8 +403,10 @@ def extract_description(soup: BeautifulSoup) -> str:
# ---------------------------------------------------------------------------
def extract_thumbnail(soup: BeautifulSoup) -> str:
meta = soup.find("meta", property="og:image")
if meta and str(meta.get("content", "")).startswith("http"):
return meta["content"]
if meta:
content = str(meta.get("content", ""))
if content.startswith("http"):
return content
ss = soup.find("div", class_="screenshot_list")
if ss:
@@ -429,7 +435,8 @@ def detect_nsfw(soup: BeautifulSoup, tags, *texts: str) -> str:
return "Yes"
if any(text != NA and _NSFW_RE.search(text) for text in texts):
return "Yes"
if soup.find("div", class_=["view_game_warning", "mature_content_notice"]):
# Either notice (a class list in find() would mean the same; ".a.b" would need both).
if soup.select_one("div.view_game_warning, div.mature_content_notice"):
return "Yes"
return "No"
+3 -4
View File
@@ -78,7 +78,7 @@ def record_outcome(patch: dict, url: str, outcome: str) -> None:
patch["ingest_outcomes"][url] = {"outcome": outcome, "at": now_iso()}
def main(argv: list[str] | None = None) -> int:
def main(argv: list[str] | None = None) -> None:
ap = argparse.ArgumentParser(description="Scrape queued links into a patch.")
ap.add_argument("--out", required=True, help="patch file to write")
ap.add_argument("--deadline-min", type=float, default=50.0)
@@ -115,7 +115,7 @@ def main(argv: list[str] | None = None) -> int:
if not queue:
print("No links to process (temp_link.json empty and no INPUT_URL).")
write_patch(args.out, patch)
return 0
return
catalog_urls = get_all_urls(load_all_games())
deleted_urls = {e.get("url") for e in load_json(DELETED_LOG)}
@@ -216,8 +216,7 @@ def main(argv: list[str] | None = None) -> int:
f"{stats['duplicate']} duplicates, {stats['deleted']} previously deleted, "
f"{stats['invalid']} invalid, {stats['retry']} to retry, {stats['gave_up']} given up."
)
return 0
if __name__ == "__main__":
sys.exit(main())
main()
+8
View File
@@ -127,6 +127,14 @@ def test_mature_notice_marks_nsfw():
assert record["nsfw"] == "Yes"
@pytest.mark.parametrize("notice", ["view_game_warning", "mature_content_notice"])
def test_either_mature_notice_alone_marks_nsfw(notice):
from bs4 import BeautifulSoup
soup = BeautifulSoup(f'<div class="{notice}"></div>', "html.parser")
assert detect_nsfw(soup, [], NA) == "Yes"
@pytest.mark.parametrize(
("tags", "description", "expected"),
[
+4 -4
View File
@@ -12,7 +12,7 @@ def _run(monkeypatch, pages: dict[str, Page], queue: list, **env):
monkeypatch.setattr(
update_info, "fetch_with_backoff", lambda _s, url, _p, _c: pages.get(url, Page(status=500))
)
assert update_info.main(["--out", "patch.json"]) == 0
update_info.main(["--out", "patch.json"])
return read_json("patch.json")
@@ -139,7 +139,7 @@ def test_duplicate_spelling_survives_a_rate_limit_stop(repo, monkeypatch):
queue = ["https://dev.itch.io/a", "https://dev.itch.io/a"]
save_json("scripts/temp_link.json", queue)
monkeypatch.setattr(update_info, "fetch_with_backoff", lambda *_a: None)
assert update_info.main(["--out", "patch.json"]) == 0
update_info.main(["--out", "patch.json"])
assert _apply(read_json("patch.json")) == queue
@@ -161,7 +161,7 @@ def test_dispatched_url_is_processed_first_and_queued_until_done(repo, monkeypat
save_json("scripts/temp_link.json", ["https://dev.itch.io/backlog"])
monkeypatch.setenv("INPUT_URL", "https://dev.itch.io/Dispatched/")
monkeypatch.setattr(update_info, "fetch_with_backoff", fetch)
assert update_info.main(["--out", "patch.json"]) == 0
update_info.main(["--out", "patch.json"])
patch = read_json("patch.json")
assert seen[0] == "https://dev.itch.io/dispatched"
assert patch["queue_add"] == ["https://dev.itch.io/dispatched"]
@@ -205,7 +205,7 @@ def test_failures_count_only_when_spaced_out(repo, monkeypatch):
"2026-09-18T14:00:00Z", # 3 -> given up
):
monkeypatch.setattr(update_info, "now_iso", lambda s=stamp: s)
assert update_info.main(["--out", "patch.json"]) == 0
update_info.main(["--out", "patch.json"])
write_patch("patch.json", read_json("patch.json"))
assert apply_patch.main(["patch.json"]) == 0
entry = read_json("scripts/state/ingest_state.json").get(url, {})
+1 -1
View File
@@ -78,7 +78,7 @@ npm run lint # ESLint
npm run knip # unused files / exports / deps
npm test # Vitest: app + Worker (D1 tests run a local workerd)
npm run build # includes tsc -b (or run npx tsc -b on its own)
npx wrangler deploy --dry-run --env=""
npm run cf:check # wrangler deploy --dry-run (deploys nothing)
```
After editing `wrangler.jsonc`, run `npm run cf:types` and commit the regenerated
+1 -1
View File
@@ -44,7 +44,7 @@ export const useTracking = create<TrackingState>()(
set((s) =>
urls.length === 0
? s
: { batches: capBatches([{ id: `${at}-${Math.random().toString(36).slice(2, 8)}`, label, at, urls }, ...s.batches]) },
: { batches: capBatches([{ id: `${at}-${crypto.randomUUID().slice(0, 8)}`, label, at, urls }, ...s.batches]) },
),
forget: (id) => set((s) => ({ batches: s.batches.filter((b) => b.id !== id) })),
}),
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "webapp",
"version": "4.4.1",
"version": "4.4.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "webapp",
"version": "4.4.1",
"version": "4.4.2",
"license": "MIT",
"dependencies": {
"@fontsource-variable/geist": "^5.3.0",
+2 -1
View File
@@ -1,7 +1,7 @@
{
"name": "webapp",
"private": true,
"version": "4.4.1",
"version": "4.4.2",
"license": "MIT",
"type": "module",
"engines": {
@@ -19,6 +19,7 @@
"test": "vitest run",
"check:tauri": "node scripts/check-tauri-versions.mjs",
"cf:types": "wrangler types --env-file=types.env --strict-vars=false",
"cf:check": "wrangler deploy --dry-run --env=",
"cf:dev": "wrangler dev"
},
"dependencies": {
+9 -2
View File
@@ -2,11 +2,14 @@
// (src/lib/i18n/en.ts and vi.ts) with the TypeScript parser, so multi-line
// values and section comments survive.
//
// node scripts/i18n-edit.mjs ops.json
// node scripts/i18n-edit.mjs < ops.json # PowerShell: Get-Content ops.json | node scripts/i18n-edit.mjs
// ops.json: { "remove": ["exact.key", "prefix.*"],
// "set": { "key": { "en": "…", "vi": "…" } } }
// node scripts/i18n-edit.mjs --unused # list keys never referenced in src/
//
// The operations come on stdin, not as a file path: the script only ever
// touches the two dictionaries.
//
// New keys are appended at the end of the object; the compiler enforces that
// vi.ts has exactly the keys of en.ts (Record<MessageKey, string>).
import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs'
@@ -50,7 +53,11 @@ if (process.argv[2] === '--unused') {
process.exit(0)
}
const ops = JSON.parse(readFileSync(process.argv[2], 'utf8'))
if (process.stdin.isTTY) {
console.error('usage: node scripts/i18n-edit.mjs < ops.json (or --unused)')
process.exit(2)
}
const ops = JSON.parse(readFileSync(0, 'utf8'))
const matches = (key, pattern) => (pattern.endsWith('.*') ? key.startsWith(pattern.slice(0, -1)) : key === pattern)
for (const [lang, path] of Object.entries(FILES)) {
+1 -1
View File
@@ -1033,7 +1033,7 @@ dependencies = [
[[package]]
name = "free-games-itchio-webapp"
version = "4.4.1"
version = "4.4.2"
dependencies = [
"serde",
"serde_json",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "free-games-itchio-webapp"
version = "4.4.1"
version = "4.4.2"
description = "Itch.io free games catalog viewer"
authors = ["poli0981"]
license = "MIT"
+1 -1
View File
@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "Itch.io Free Games DB",
"version": "4.4.1",
"version": "4.4.2",
"identifier": "com.poli0981.freegamesitchio",
"build": {
"beforeDevCommand": "npm run dev",
+4 -1
View File
@@ -23,7 +23,10 @@ export function useSeoHead(): void {
const { pathname } = useLocation()
useEffect(() => {
if (isTauri()) return
const path = pathname.length > 1 ? pathname.replace(/\/+$/, '') : '/'
// A loop, not /\/+$/ (which backtracks on a long run of slashes).
let end = pathname.length
while (end > 1 && pathname[end - 1] === '/') end--
const path = pathname.slice(0, end)
const canonical = headTag('link[rel="canonical"]', () => {
const link = document.createElement('link')
link.rel = 'canonical'
+1 -1
View File
@@ -8,7 +8,7 @@ export interface ThirdParty {
export const APP = {
name: 'Free Itch Games',
version: '4.4.1',
version: '4.4.2',
repo: 'https://github.com/poli0981/free-games-itchio-list',
license: 'code MIT · data CC BY 4.0',
} as const
+2 -2
View File
@@ -46,8 +46,8 @@ export function aiDisclosureCounts(games: Game[]): CountEntry[] {
}
function parseRating(rating: string): number | null {
const n = parseFloat(rating)
return isFinite(n) ? n : null
const n = Number.parseFloat(rating)
return Number.isFinite(n) ? n : null
}
export function classifyOnline(games: Game[]): { online: number; offline: number } {
+2 -1
View File
@@ -8,6 +8,7 @@
* here: signature against the team's JWKS, issuer, and the application's AUD.
*/
import { createRemoteJWKSet, jwtVerify, type JWTPayload } from 'jose'
import { trimEnd } from './text'
export interface AccessEnv {
ACCESS_TEAM_DOMAIN: string // https://<team>.cloudflareaccess.com
@@ -39,7 +40,7 @@ async function verify(request: Request, env: AccessEnv, audience: string): Promi
const token = request.headers.get('cf-access-jwt-assertion')
if (!token) throw new AccessError('missing Access token')
// The issuer is the bare origin; tolerate a trailing slash pasted from the dashboard.
const team = env.ACCESS_TEAM_DOMAIN.trim().replace(/\/+$/, '')
const team = trimEnd(env.ACCESS_TEAM_DOMAIN.trim(), '/')
try {
const { payload } = await jwtVerify(token, jwks(team), { issuer: team, audience })
return payload
+2 -1
View File
@@ -50,6 +50,7 @@ import {
type QueueStore,
} from './queue'
import { repoOps, type IngestView, type RepoOps } from './repo'
import { trimEnd } from './text'
const STATUSES: readonly CandidateStatus[] = ['pending', 'rejected', 'queued', 'ingested', 'failed']
const THUMB_WIDTHS = [160, 640]
@@ -538,7 +539,7 @@ export async function adminApi(request: Request, url: URL, ctx: AdminContext): P
}
async function route(request: Request, url: URL, ctx: AdminContext): Promise<Response> {
const route = `${request.method} ${url.pathname.replace(/\/+$/, '')}`
const route = `${request.method} ${trimEnd(url.pathname, '/')}`
switch (route) {
case 'GET /api/admin/me':
return json({ email: ctx.actor, writes: ctx.repo !== null, repo: ctx.repoName ?? null })
+2 -1
View File
@@ -11,6 +11,7 @@
const CHUNK_SIZE = 500
const MIN_FILL = 250
const DEFAULT_SIZES = { chunkSize: CHUNK_SIZE, minFill: MIN_FILL }
/** The only fields a maintainer edits; everything else is scraper-owned. */
const EDITABLE_FIELDS = ['safe_virus', 'notes', 'nsfw'] as const
@@ -181,7 +182,7 @@ export function applyAdminChange(
edits: Map<string, FieldEdit>,
removals: { url: string; reason: string }[],
now: Date,
sizes: { chunkSize: number; minFill: number } = { chunkSize: CHUNK_SIZE, minFill: MIN_FILL },
sizes: { chunkSize: number; minFill: number } = DEFAULT_SIZES,
): CatalogChange {
const games = files.chunks.flat().map((g) => ({ ...g }))
const editedUrls: string[] = []
+11 -2
View File
@@ -80,9 +80,18 @@ export function collectionPageUrl(id: string, slug: string, page: number): strin
return `${collectionUrl(id, slug)}?format=json${page > 1 ? `&page=${page}` : ''}`
}
const ATTRIBUTE_NAME = /^[a-zA-Z_:][-a-zA-Z0-9_:.]*$/
/** A tag's double-quoted attributes, read by splitting on the quotes (a name="value" regex backtracks). */
function attributes(tag: string): Map<string, string> {
const out = new Map<string, string>()
for (const m of tag.matchAll(/([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*"([^"]*)"/g)) out.set(m[1].toLowerCase(), m[2])
const parts = tag.split('"') // outside, value, outside, value, …; an unclosed value is dropped
for (let i = 1; i < parts.length - 1; i += 2) {
const before = parts[i - 1].trimEnd()
if (!before.endsWith('=')) continue
const name = before.slice(0, -1).trimEnd().split(/\s+/).pop() ?? ''
if (ATTRIBUTE_NAME.test(name)) out.set(name.toLowerCase(), parts[i])
}
return out
}
@@ -92,7 +101,7 @@ function hasClasses(attrs: Map<string, string>, ...names: string[]): boolean {
}
function text(html: string): string {
return decodeEntities(html.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim()
return decodeEntities(html.replace(/<[^<>]*>/g, ' ')).replace(/\s+/g, ' ').trim()
}
function parseCell(cell: string): CollectionItem | null {
+2 -2
View File
@@ -55,13 +55,13 @@ export function parseFeed(xml: string): FeedItem[] {
const url = tag(block, 'link') ?? tag(block, 'guid')
if (!url) continue
const rawTitle = tag(block, 'title') ?? ''
const brackets = [...rawTitle.matchAll(/\[([^\]]+)\]/g)].map((m) => m[1].trim())
const brackets = [...rawTitle.matchAll(/\[([^[\]]+)\]/g)].map((m) => m[1].trim())
const genre = brackets.find((b) => !NOT_GENRE.has(b.toLowerCase())) ?? null
const price = tag(block, 'price') ?? ''
const image = tag(block, 'imageurl')
items.push({
url,
title: tag(block, 'plainTitle') ?? rawTitle.replace(/\s*\[[^\]]*\]/g, '').trim(),
title: tag(block, 'plainTitle') ?? rawTitle.replace(/\[[^[\]]*\]/g, '').replace(/\s+/g, ' ').trim(),
image: image?.startsWith('https://img.itch.zone/') ? image : null,
genre,
free: /^[^0-9]*0+(?:[.,]0+)?$/.test(price) || brackets.some((b) => b.toLowerCase() === 'free'),
+3 -4
View File
@@ -21,6 +21,7 @@
import { missingConfig, type WorkerEnv } from './env'
import { challengePage, crawlerUnavailable } from './gate-page'
import { HttpError, LOCAL_HOSTS, errorResponse, isObject, json, rateKey, readJson } from './http'
import { trimEnd } from './text'
import { siteverify } from './turnstile'
const COOKIE = '__Host-fig_gate'
@@ -98,10 +99,8 @@ async function macKey(secret: string): Promise<CryptoKey> {
const macInput = (iat: number) => encoder.encode(`fig-gate|v1|${iat}`)
function toBase64Url(bytes: ArrayBuffer): string {
return btoa(String.fromCharCode(...new Uint8Array(bytes)))
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/, '')
const b64 = btoa(String.fromCharCode(...new Uint8Array(bytes)))
return trimEnd(b64.replace(/\+/g, '-').replace(/\//g, '_'), '=')
}
function fromBase64Url(text: string): Uint8Array {
+3 -2
View File
@@ -18,6 +18,7 @@ import {
} from './catalog'
import { GITHUB_CONFIG, missingConfig, type WorkerEnv } from './env'
import { commitWithRetry, headOid, listDir, readFile, type CommitInput, type GitHubEnv } from './github'
import { compareCodeUnits } from './text'
const TEMP_LINK = 'scripts/temp_link.json'
const DELETED_LOG = 'scripts/deleted_games.json'
@@ -56,7 +57,7 @@ async function readJsonFile<T>(env: GitHubEnv, path: string, head: string, fallb
/** The catalog files at `head`, chunks in the same order as data_store.py. */
async function readCatalog(env: GitHubEnv, head: string): Promise<CatalogFiles> {
const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort()
const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort(compareCodeUnits)
const [chunks, index, history, deleted] = await Promise.all([
Promise.all(names.map((n) => readJsonFile<Game[]>(env, `data_game/${n}`, head, []))),
readJsonFile<CatalogIndex | null>(env, 'data_game/index.json', head, null),
@@ -92,7 +93,7 @@ export function withQueued(queueText: string | null, urls: string[]): { text: st
export function withUnblocked(currentText: string | null, urls: string[]): { text: string; added: number } {
const parsed: unknown = currentText === null ? [] : JSON.parse(currentText)
const current = Array.isArray(parsed) ? parsed.filter((u): u is string => typeof u === 'string') : []
const next = [...new Set([...current, ...urls])].sort()
const next = [...new Set([...current, ...urls])].sort(compareCodeUnits)
return { text: dumps(next), added: next.length - current.length }
}
+3 -2
View File
@@ -16,6 +16,7 @@
*/
import { catalogUrls, type DataSource } from './data'
import { json } from './http'
import { trimEnd } from './text'
// Keep in sync with the <Routes> in src/App.tsx.
const ROUTES = new Set(['/', '/games', '/charts', '/removed', '/settings', '/about', '/suggest'])
@@ -52,7 +53,7 @@ async function isGameSlug(slug: string, src: DataSource): Promise<boolean> {
}
async function isAppRoute(pathname: string, src: DataSource): Promise<boolean> {
const path = pathname.length > 1 ? pathname.replace(/\/+$/, '') : pathname
const path = pathname.length > 1 ? trimEnd(pathname, '/') : pathname
if (ROUTES.has(path) || ERROR_PREVIEW_ROUTE.test(path)) return true
const game = GAME_ROUTE.exec(path)
return game !== null && (await isGameSlug(game[1], src))
@@ -72,7 +73,7 @@ export async function handleMiss(request: Request, url: URL, assets: Fetcher, ga
const cors = url.pathname.startsWith('/data/') ? { 'Access-Control-Allow-Origin': '*' } : undefined
return json({ error: 'not_found' }, 404, cors)
}
const moved = MOVED[url.pathname.replace(/\/+$/, '')]
const moved = MOVED[trimEnd(url.pathname, '/')]
if (moved) return Response.redirect(new URL(moved + url.search, url).toString(), 301)
// Before the shell is fetched, so a conditional request (If-None-Match)
// without a pass gets the verification page, never a 304 for a cached shell.
+3 -2
View File
@@ -3,6 +3,7 @@
* for tests of the SQL in queue.ts. Runs workerd through wrangler's platform proxy.
*/
import { getPlatformProxy } from 'wrangler'
import { compareCodeUnits } from '../text'
const MIGRATIONS = import.meta.glob<string>('../migrations/*.sql', { query: '?raw', import: 'default', eager: true })
@@ -11,9 +12,9 @@ export async function testDb(): Promise<{ db: D1Database; dispose: () => Promise
configPath: new URL('./wrangler.test.jsonc', import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'),
persist: false,
})
for (const name of Object.keys(MIGRATIONS).sort()) {
for (const name of Object.keys(MIGRATIONS).sort(compareCodeUnits)) {
const statements = MIGRATIONS[name]
.replace(/--.*$/gm, '')
.replace(/--[^\r\n]*/g, '')
.split(';')
.map((s) => s.trim())
.filter(Boolean)
+6
View File
@@ -35,6 +35,12 @@ describe('parseFeed', () => {
expect(items[2].free).toBe(false)
expect(items[3].image).toBeNull() // only img.itch.zone covers are kept
})
it('falls back to the title without its [tags] when there is no plainTitle', () => {
const feed = `<rss><channel><item><title>Night Shift [Free] [Horror] [Windows]</title>
<link>https://studio.itch.io/night-shift</link><price>$0.00</price></item></channel></rss>`
expect(parseFeed(feed)[0]).toMatchObject({ title: 'Night Shift', genre: 'Horror', free: true })
})
})
describe('pickFeed', () => {
+40
View File
@@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest'
import { compareCodeUnits, decodeEntities, trimEnd } from './text'
describe('decodeEntities', () => {
it('decodes decimal, hex and named references', () => {
expect(decodeEntities('&#65;&#x42;&#X43; &lt;b&gt; Rock &amp; Roll&nbsp;')).toBe('ABC <b> Rock & Roll')
})
})
describe('trimEnd', () => {
it.each([
['/games/', '/', '/games'],
['/games///', '/', '/games'],
['/games', '/', '/games'],
['///', '/', ''],
['', '/', ''],
['c2lnbg==', '=', 'c2lnbg'],
])('%j without trailing %j', (text, char, expected) => {
expect(trimEnd(text, char)).toBe(expected)
})
it('takes linear time on a long run that is not at the end (a /\\/+$/ regex did not)', () => {
const path = `/${'/'.repeat(200_000)}x`
expect(trimEnd(path, '/')).toBe(path)
})
})
describe('compareCodeUnits', () => {
it('orders like a bare sort(): code units, not the locale', () => {
expect(['game_info_010.json', 'alpha', 'game_info_002.json', 'Zeta', 'b', 'B'].sort(compareCodeUnits)).toEqual([
'B',
'Zeta',
'alpha',
'b',
'game_info_002.json',
'game_info_010.json',
])
expect(compareCodeUnits('same', 'same')).toBe(0)
})
})
+17 -1
View File
@@ -2,7 +2,7 @@
export function decodeEntities(text: string): string {
return text
.replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n)))
.replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(parseInt(n, 16)))
.replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(Number.parseInt(n, 16)))
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&lt;/g, '<')
@@ -11,3 +11,19 @@ export function decodeEntities(text: string): string {
.replace(/&amp;/g, '&')
.trim()
}
/**
* `text` without its trailing `char`s. A loop, not `/x+$/`: that regex retries
* from every character of a long run that is not at the end (quadratic).
*/
export function trimEnd(text: string, char: string): string {
let end = text.length
while (end > 0 && text[end - 1] === char) end--
return text.slice(0, end)
}
/** Code-unit order, like a bare `sort()` and, for these ASCII names, Python's `sorted()`; not locale-aware. */
export function compareCodeUnits(a: string, b: string): number {
if (a === b) return 0
return a < b ? -1 : 1
}