From 999d0887eecc79822ae5bdf2e535046a231ecb23 Mon Sep 17 00:00:00 2001 From: poli0981 <127664709+poli0981@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:16:12 +0700 Subject: [PATCH] Fix the SonarQube Cloud findings; release 4.4.2 (#155) The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud. Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh. Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check. Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41. [skip-discuss] Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-ci.yml | 11 +- .github/workflows/refresh.yml | 6 +- .github/workflows/release_android.yml | 2 +- .github/workflows/release_desktop.yml | 2 +- .github/workflows/update.yml | 3 +- .github/workflows/webapp-ci.yml | 6 +- .sonarcloud.properties | 9 + CHANGELOG.md | 3 +- CLAUDE.md | 15 +- bash/cf_build_status.sh | 10 +- changelog/4.x/ver4.4.x.md | 49 +++- docs/dev_env.md | 7 +- docs/i18n/vi/dev_env.md | 6 +- requirements-dev.in | 11 + requirements-dev.txt | 365 +++++++++++++++++++++++++- requirements.in | 6 + requirements.txt | 213 ++++++++++++++- scripts/scraper.py | 19 +- scripts/update_info.py | 7 +- tests/test_parser.py | 8 + tests/test_update_info.py | 8 +- webapp/README.md | 2 +- webapp/admin/tracking.ts | 2 +- webapp/package-lock.json | 4 +- webapp/package.json | 3 +- webapp/scripts/i18n-edit.mjs | 11 +- webapp/src-tauri/Cargo.lock | 2 +- webapp/src-tauri/Cargo.toml | 2 +- webapp/src-tauri/tauri.conf.json | 2 +- webapp/src/hooks/useSeoHead.ts | 5 +- webapp/src/lib/about.ts | 2 +- webapp/src/lib/analytics.ts | 4 +- webapp/worker/access.ts | 3 +- webapp/worker/admin.ts | 3 +- webapp/worker/catalog.ts | 3 +- webapp/worker/collections.ts | 13 +- webapp/worker/discover.ts | 4 +- webapp/worker/gate.ts | 7 +- webapp/worker/repo.ts | 5 +- webapp/worker/spa.ts | 5 +- webapp/worker/test/d1.ts | 5 +- webapp/worker/test/discover.test.ts | 6 + webapp/worker/text.test.ts | 40 +++ webapp/worker/text.ts | 18 +- 44 files changed, 840 insertions(+), 77 deletions(-) create mode 100644 .sonarcloud.properties create mode 100644 requirements-dev.in create mode 100644 requirements.in create mode 100644 webapp/worker/text.test.ts diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 2b73515..934d901 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -10,6 +10,7 @@ on: - 'tests/**' - 'data_game/**' - 'requirements*.txt' + - 'requirements*.in' - 'pyproject.toml' - '.github/python-version' - '.github/workflows/python-ci.yml' @@ -20,6 +21,7 @@ on: - 'tests/**' - 'data_game/**' - 'requirements*.txt' + - 'requirements*.in' - 'pyproject.toml' - '.github/python-version' - '.github/workflows/python-ci.yml' @@ -49,10 +51,15 @@ jobs: # supports Python <= 3.14 only (CLAUDE.md gotcha 39). python-version-file: .github/python-version cache: pip - cache-dependency-path: requirements-dev.txt + cache-dependency-path: | + requirements.txt + requirements-dev.txt + # Both hash-locked files at once: pip fails if their pins disagree. + # Wheels only, so no package runs a build script here. - name: Install dependencies - run: pip install -r requirements-dev.txt + run: | + pip install --require-hashes --only-binary :all: -r requirements.txt -r requirements-dev.txt - name: Ruff run: | diff --git a/.github/workflows/refresh.yml b/.github/workflows/refresh.yml index fb96099..0110faf 100644 --- a/.github/workflows/refresh.yml +++ b/.github/workflows/refresh.yml @@ -75,7 +75,8 @@ jobs: cache-dependency-path: requirements.txt - name: Install dependencies - run: pip install -r requirements.txt + run: | + pip install --require-hashes --only-binary :all: -r requirements.txt - name: Check a slice of the catalog id: refresh @@ -130,7 +131,8 @@ jobs: cache-dependency-path: requirements.txt - name: Install dependencies - run: pip install -r requirements.txt + run: | + pip install --require-hashes --only-binary :all: -r requirements.txt - name: Download patch uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 diff --git a/.github/workflows/release_android.yml b/.github/workflows/release_android.yml index 5ef40ba..56e8217 100644 --- a/.github/workflows/release_android.yml +++ b/.github/workflows/release_android.yml @@ -81,7 +81,7 @@ jobs: - name: Install npm deps working-directory: webapp - run: npm ci + run: npm ci --ignore-scripts - name: Initialize Android project working-directory: webapp diff --git a/.github/workflows/release_desktop.yml b/.github/workflows/release_desktop.yml index a32cde8..1cf2cdf 100644 --- a/.github/workflows/release_desktop.yml +++ b/.github/workflows/release_desktop.yml @@ -121,7 +121,7 @@ jobs: - name: Install npm deps working-directory: webapp - run: npm ci + run: npm ci --ignore-scripts - name: Build Tauri app uses: tauri-apps/tauri-action@1deb371b0cd8bd54025b384f1cd735e725c4060f # action-v1.0.0 diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 6ead227..d1d94f9 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -55,7 +55,8 @@ jobs: cache-dependency-path: requirements.txt - name: Install dependencies - run: pip install -r requirements.txt + run: | + pip install --require-hashes --only-binary :all: -r requirements.txt - name: Scrape queued links timeout-minutes: 50 diff --git a/.github/workflows/webapp-ci.yml b/.github/workflows/webapp-ci.yml index 88c00b0..d6a257c 100644 --- a/.github/workflows/webapp-ci.yml +++ b/.github/workflows/webapp-ci.yml @@ -36,8 +36,10 @@ jobs: cache: npm cache-dependency-path: webapp/package-lock.json + # No install scripts: esbuild's and workerd's postinstall only check their + # platform binary, which their launchers find without it. - name: Install dependencies - run: npm ci + run: npm ci --ignore-scripts # The web build never runs `tauri build`, which refuses mismatched # crate / npm versions; without this a Dependabot bump on one side @@ -61,4 +63,4 @@ jobs: run: npm run build - name: Validate Workers config (dry run) - run: npx wrangler deploy --dry-run --env="" + run: npm run cf:check diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 0000000..1f476be --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,9 @@ +# SonarQube Cloud automatic analysis +# (https://sonarcloud.io/project/overview?id=poli0981_free-games-itchio-list). +# Automatic analysis reads only a few settings from this file (no per-rule ignores): +# mark a false positive on the issue itself. See CLAUDE.md, gotcha 41. + +# itch.io pages saved as served, for the scraper tests: their markup is +# itch.io's (tables without , tabindex on divs, …), and editing it would +# make the tests check something itch.io never sends. +sonar.exclusions=tests/fixtures/** diff --git a/CHANGELOG.md b/CHANGELOG.md index 2284360..bbbff16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,13 @@ All notable changes to this project, newest first. The notes of each release live in [`changelog/`](changelog/), one file per minor version: `changelog/.x/ver..x.md`. -**Latest release: [4.4.1](changelog/4.x/ver4.4.x.md#441---2026-10-16-extension-and-bot-retired) (2026-10-16) — Extension and bot retired.** +**Latest release: [4.4.2](changelog/4.x/ver4.4.x.md#442---2026-10-22-toolchain-upgrade-code-scan-fixes) (2026-10-22) — Toolchain upgrade, code-scan fixes.** ## 4.x | Version | Date | Highlights | |---|---|---| +| [4.4.2](changelog/4.x/ver4.4.x.md#442---2026-10-22-toolchain-upgrade-code-scan-fixes) | 2026-10-22 | Toolchain upgrade (TypeScript 7, Node 26, Python 3.15), code-scan fixes | | [4.4.1](changelog/4.x/ver4.4.x.md#441---2026-10-16-extension-and-bot-retired) | 2026-10-16 | Extension and bot retired (archived repos), ingest API removed | | [4.4.0](changelog/4.x/ver4.4.x.md#440---2026-10-13-admin-automation-bulk-actions-smoother-charts) | 2026-10-13 | Admin automation, bulk actions, smoother charts | | [4.3.0](changelog/4.x/ver4.3.x.md#430---2026-10-10-ai-disclosure-accessibility-refreshed-catalog) | 2026-10-10 | AI disclosure, accessibility, refreshed catalog | diff --git a/CLAUDE.md b/CLAUDE.md index 00f8c16..1e5329d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ everyone; the old github.io URL only redirects). Three layers share one JSON cat 1. **Python pipeline** — the data layer (GitHub Actions). - `data_game/game_info_NNN.json` — chunked records, max 500 per file, 3,251 games on 2026-10-10. Serialization is fixed: `json.dumps(..., ensure_ascii=False, indent=4)`, no trailing newline (`json_io.dumps`) — the Worker port (`webapp/worker/catalog.ts`) emits identical bytes (`tests/fixtures/golden/`). - `data_game/index.json` — chunk manifest (total + per-file counts). `data_game/count_history.json` — date-keyed `{date,total}` series ("games over time" chart). `scripts/deleted_games.json` — removal log (public "Removed" page). `scripts/temp_link.json` — ingest queue. - - `scripts/` — scanners never write the catalog: `update_info.py` (queued URLs) and `refresh.py` (rotating 1/7-per-day check; `--full` re-scrape) emit a URL-keyed **patch** (`patch.py`); `apply_patch.py` applies it to the latest `main` and runs `validate.py`; `bash/commit_push.sh` does fetch → reset → apply → commit → push with retries. Shared: `scraper.py` (session, `Pacer` + 429 back-off, parsing: the info panel incl. itch.io's 2025–26 rows — `AI Disclosure` ("AI Assisted", then what for) / `Content` linking `/games/tag-no-ai` → `ai_disclosure` + `ai_content`, `Accessibility` → `accessibility`; `description` = the creator's tagline (`og:description`), else the first sentence), `data_store.py` (chunks stay in place, rebalance only past thresholds, write-if-changed), `json_io.py`, `canonical.py` (canonical itch.io URL; vectors in `tests/fixtures/url_vectors.json`, shared with the Worker). Pipeline bookkeeping lives in `scripts/state/` (not published), including `ingest_outcomes.json` (how each queued link ended — added / duplicate / paid / dead / private / deleted / gave_up; patch key `ingest_outcomes`; the last 90 days, ≤ 5,000) that the admin's tracking and the cron's reconcile read. Tests: `pytest` (network blocked). Lint: `ruff check scripts tests webapp/scripts`, `ruff format`, `vulture` (config in root `pyproject.toml`). Local env: Python 3.15 (`.github/python-version`, gotcha 39; ruff `target-version = "py315"`), `python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt` (3.15 is in UTF-8 mode by default — no `PYTHONUTF8` needed, the workflows no longer set it). + - `scripts/` — scanners never write the catalog: `update_info.py` (queued URLs) and `refresh.py` (rotating 1/7-per-day check; `--full` re-scrape) emit a URL-keyed **patch** (`patch.py`); `apply_patch.py` applies it to the latest `main` and runs `validate.py`; `bash/commit_push.sh` does fetch → reset → apply → commit → push with retries. Shared: `scraper.py` (session, `Pacer` + 429 back-off, parsing: the info panel incl. itch.io's 2025–26 rows — `AI Disclosure` ("AI Assisted", then what for) / `Content` linking `/games/tag-no-ai` → `ai_disclosure` + `ai_content`, `Accessibility` → `accessibility`; `description` = the creator's tagline (`og:description`), else the first sentence), `data_store.py` (chunks stay in place, rebalance only past thresholds, write-if-changed), `json_io.py`, `canonical.py` (canonical itch.io URL; vectors in `tests/fixtures/url_vectors.json`, shared with the Worker). Pipeline bookkeeping lives in `scripts/state/` (not published), including `ingest_outcomes.json` (how each queued link ended — added / duplicate / paid / dead / private / deleted / gave_up; patch key `ingest_outcomes`; the last 90 days, ≤ 5,000) that the admin's tracking and the cron's reconcile read. Tests: `pytest` (network blocked). Lint: `ruff check scripts tests webapp/scripts`, `ruff format`, `vulture` (config in root `pyproject.toml`). Local env: Python 3.15 (`.github/python-version`, gotcha 39; ruff `target-version = "py315"`), `python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt` (3.15 is in UTF-8 mode by default — no `PYTHONUTF8` needed, the workflows no longer set it). Both requirements files are hash-locked pip-tools compiles of `requirements*.in` — edit the `.in`, then recompile (gotcha 40). 2. **Cloudflare Worker + static assets** — `webapp/worker/` + `webapp/wrangler.jsonc`, deployed by **Cloudflare Workers Builds** from `main` (root dir `webapp`, build `npm run build`, deploy `npx wrangler deploy`; no deploy workflow). The catalog is bundled into the site at build time (`vite-plugins/catalog-data.ts` → `dist/data/pack/` compact packs + the legacy readable JSON in `dist/data/`, plus `data/urls.json` and `sitemap.xml`), so every data commit rebuilds the site. 3. **Web app + Tauri apps** — `webapp/src/` (React 19.3 + TypeScript 7.0 + Vite 8/rolldown + Tailwind + shadcn/ui + React Router 8), and the **Tauri 2 desktop + Android** read-only viewers in `webapp/src-tauri/` (same code; they fetch `https://freeitchgames.win/data/pack` and load covers straight from img.itch.zone). Plus `webapp/admin/` — the maintainer-only admin app (second Vite entry, served at `/admin/`). @@ -27,6 +27,7 @@ npm run build # → webapp/dist/ (public app + /admin/ + /data + sitema npm test # vitest (worker/, src/lib); worker/test/* D1 tests spawn a local workerd via getPlatformProxy npm run lint && npm run knip && npx tsc -b npm run cf:types # after editing wrangler.jsonc (ignores .dev.vars via types.env; CI runs it with --check) +npm run cf:check # wrangler deploy --dry-run with the project's wrangler (what CI runs; after a build) npx wrangler d1 migrations apply freeitchgames --local # local D1 (use --remote by hand for production) npm run tauri:dev # native window; needs Rust toolchain npm run tauri:build # native installers @@ -78,14 +79,14 @@ Each game has **20** fields (plus optional `added_at` / `updated_at`, appended b | `refresh.yml` | Daily 02:47 UTC + `workflow_dispatch` (`budget`, `url`, `allow_mass_removal`, `full`) | scan (read-only, 55-min step timeout, checks out the branch as it is when the job starts — not the queued SHA — so runs waiting in the queue see the previous run's state; patch artifact uploaded `if: always()`) → apply (`!cancelled()`: a failed / timed-out scan's checkpoint is applied, a cancelled run's is not) → notify. Workflow-level `concurrency: refresh-pipeline` + `queue: max` runs one whole scan → apply at a time (scheduled, manual and force-update runs alike). Removal needs the same 404/410, paid or `private` (itch.io's password form, `is_password_protected`; ingest drops such links as final) strike seen again ≥ 20 h after the first sighting (`strike_at`). Checkpoints never carry removals. **Mass-change guard:** a run whose removals or new strikes exceed max(10, 10 %) of the games it checked withholds all its removals and marks them — and, on a strike spike, all its strikes — `withheld_at`; later runs keep withholding those (they take at most half a run's budget) until a healthy check clears them, and alert (`mass_change`) — that pattern means itch.io markup changed; dispatch with `allow_mass_removal` only after checking on itch.io. `apply_patch.py` re-checks the same per-run limit and max(25, 5 %) of the catalog unless `ALLOW_MASS_REMOVAL=1`. | | `force_update.yml` | `workflow_dispatch` ONLY (`url`, `budget`, `allow_mass_removal`) | Dispatches `refresh.yml` with `full: true` (`actions: write`; it used to call it through `uses:`, which failed at startup with no annotation): re-scrapes every scraper field for one URL or the next batch (rotates on `full` timestamps). Dispatch it N times in a row to cover N consecutive batches (FIFO, each from the latest main). Always preserves `safe_virus` / `notes` / `nsfw`. Canonical tool for repairing mojibake. | | `notify-failure.yml` | `workflow_call` | Discord (`DISCORD_CI_WEBHOOK`) on failure / **cancelled** (job-level timeouts end as cancelled) / rate-limited / mass change / failed Cloudflare build. | -| `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` / `npm run check:tauri` (gotcha 31) + `npm run cf:types -- --check` + lint + knip + vitest + build + `wrangler deploy --dry-run --env=""`. | +| `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` (both requirements locks installed at once, so drifting pins fail) / `npm ci --ignore-scripts` + `npm run check:tauri` (gotcha 31) + `npm run cf:types -- --check` + lint + knip + vitest + build + `npm run cf:check` (`wrangler deploy --dry-run`). A SonarQube Cloud check also runs on PRs (gotcha 41). | | `deploy-status.yml` | Hourly + manual | Not a deploy. `bash/cf_build_status.sh` walks main's first-parent history to the latest finished `Workers Builds: free-games-itchio-list` check run (Cloudflare posts it only when a build ends; branch commits carry preview builds under the same name) and alerts once per failed build (cache key per check-run id, saved after the Discord post). Scheduled because GitHub never starts `check_run` workflows for `GITHUB_TOKEN` pushes (the data commits). | | `release_desktop.yml` | Tag `v*` push, or manual | `create-release` makes the single draft (id passed to tauri-action v1 `releaseId`); release builds use **no** npm/cargo caches (cache-poisoning). Win + macOS aarch64 + macOS x86_64 (cross-compile) + Linux (`ubuntu-22.04` for a low glibc floor). Uploads `.dmg` / `.app.tar.gz` / `.pkg` (macOS), `.msi` / `.exe` (Windows), `.deb` / `.rpm` / `.AppImage` (Linux). macOS `.pkg` `--identifier` reads `tauri.conf.json`. | | `release_android.yml` | Tag `v*` push, or manual | Tauri **Android** build → arm64-v8a APK, **post-signed** with `zipalign`+`apksigner` (gen/ stays gitignored). Tag → attaches to the same draft Release; manual dispatch → workflow artifact. Secrets: `ANDROID_KEYSTORE_BASE64` / `ANDROID_KEYSTORE_PASSWORD` / `ANDROID_KEY_ALIAS`. | | `announce-release.yml`, `notify-release-pipeline.yml` | Release published / pipeline runs | Reusable workflows from `poli0981/.github`, SHA-pinned — but at that pin they still check out their notifier script from that repo's `main`. | | `announce-discussion.yml` | Release published; push to `main` touching `CHANGELOG.md` / `changelog/**` | Local. The release goes to Discussions → Announcements, a changelog push to → General unless the head commit says `[skip-discuss]`. | -All workflows: `permissions: {}` at the top, per-job grants, actions pinned by commit SHA. +All workflows: `permissions: {}` at the top, per-job grants, actions pinned by commit SHA. Python installs are hash-locked and wheels only (`pip install --require-hashes --only-binary :all:`, gotcha 40); npm installs skip install scripts (`npm ci --ignore-scripts` — esbuild's and workerd's launchers find their platform binary without them). ## Release / tag process @@ -145,17 +146,21 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri 37. **D1 counts every statement of a batch** towards its 1,000 queries per invocation: bind URL lists in `IN (…)` slices of ≤ 90 (100 bound parameters at most), as `setStatus` / `reconcile` / `markIngested` do, never one statement per URL. 38. **npm 11.20+ (Node 26) gates dependency install scripts on `allowScripts` in `webapp/package.json`.** esbuild and workerd are approved by name (their postinstall only checks the platform binary). A new dependency with an install script shows up in `npm install-scripts ls`; review it, then `npm install-scripts approve --no-allow-scripts-pin` (name-only, so Dependabot bumps stay covered). 39. **Dependabot's pip updater reads `.python-version` and aborts on a Python it doesn't support** (3.10–3.14 as of Oct 2026): a `3.15` there stopped pip version updates and the pip dependency graph ("Dependabot detected the following Python requirement … '3.15.*'"). So the CI pin lives in `.github/python-version`, which only `setup-python`'s `python-version-file` reads (python-ci, refresh ×2, update); move it back to `.python-version` once Dependabot lists 3.15. +40. **The Python requirements are hash-locked pip-tools compiles** — `requirements.txt` ← `requirements.in`, `requirements-dev.txt` ← `requirements-dev.in` (`-c requirements.txt`) — installed everywhere with `--require-hashes --only-binary :all:`, so a new release of any dependency, indirect ones included, can't reach the jobs that push to `main`, and no sdist build script runs. After editing a `.in`, recompile with **pip-tools on Python 3.14**: `pip-compile --generate-hashes --allow-unsafe --strip-extras requirements.in`, then the same for `requirements-dev.in`. Dependabot takes the Python version from the `# … autogenerated by pip-compile with Python X.Y` header and supports ≤ 3.14 (gotcha 39); the hashes cover every file of a release, so CI's 3.15 still finds its cp315 wheels. `colorama` is listed in `requirements-dev.in` because pytest needs it on Windows only: a lock compiled on Linux (Dependabot's) would otherwise leave it out and the hash-checked local install would fail. Keep those pip commands in a block scalar (`run: |`): in a plain YAML scalar the `: ` of `:all: -r` makes the whole workflow file invalid — GitHub then records a 0-job failed push run named after the file, and a PR simply shows no check for that workflow. +41. **SonarQube Cloud runs Automatic Analysis** (project `poli0981_free-games-itchio-list`, public; PRs get a "SonarCloud Code Analysis" check). It reads only `.sonarcloud.properties` (sources / tests / exclusions — no per-rule ignores; mark a false positive on the issue itself). Two of its suggested fixes are wrong here: S2871 wants `localeCompare` in `.sort()`, but chunk names, migration files and `unblocked.json` keep code-unit order (= Python's `sorted()`) — pass `compareCodeUnits` (`worker/text.ts`); S8906 wants `select_one('div.a.b')` for bs4's `class_=[a, b]`, which means *either* class — write a selector list `div.a, div.b`. Trim trailing characters with `trimEnd()` (`worker/text.ts`), not `/x+$/` (S8786: quadratic backtracking on a long run). Known false positive: css:S8776 on Tailwind v4's `@utility … { &::-webkit-scrollbar … }` (the `&` is required for variants). ## Safe-edit rules - Don't write `data_game/` from new code paths directly — go through a patch + `apply_patch.py` (or the Worker's `repo.ts`), so concurrent writers never clobber each other and `validate.py` gates every commit. - Don't commit `webapp/dist/`, `webapp/src-tauri/target/`, `webapp/src-tauri/gen/`, `webapp/src-tauri/icon-source.png`, `.dev.vars`, `username.txt`, `assets/qr/`. `Cargo.lock` **is** committed (since v4). - Never commit secrets (GitHub App key, admin emails, Turnstile secret, tokens). Data workflows push with the job's `GITHUB_TOKEN`; the Worker writes as the GitHub App. -- When adding an npm dep, also add it to `webapp/src/lib/about.ts` so the About page lists it. +- When adding an npm dep, also add it to `webapp/src/lib/about.ts` so the About page lists it. A Python dep goes into `requirements*.in`, then recompile the locks (gotcha 40). ## Current state (Oct 2026) -**Toolchain** (2026-10-22, on `main`, not released yet — goes into the next release's notes): TypeScript 7.0.2 (`@typescript/native`; `typescript` = the 6.0 API alias, gotcha 6), Node 26 (`webapp/.node-version`, `engines >=26`, `@types/node` 26, Workers Builds `NODE_VERSION` raised to 26 by the owner in the dashboard — `docs/operations.md` §1 step 1), Python 3.15 (`.github/python-version` — gotcha 39, ruff `py315`, `urllib3` declared since `scraper.py` imports it). No source change was needed: TS 7 reports 0 errors, the suites pass on Node 26.12 and Python 3.15.1. +**Toolchain** (2026-10-22, #149 + #154, shipped in 4.4.2): TypeScript 7.0.2 (`@typescript/native`; `typescript` = the 6.0 API alias, gotcha 6), Node 26 (`webapp/.node-version`, `engines >=26`, `@types/node` 26, Workers Builds `NODE_VERSION` raised to 26 by the owner in the dashboard — `docs/operations.md` §1 step 1), Python 3.15 (`.github/python-version` — gotcha 39, ruff `py315`, `urllib3` declared since `scraper.py` imports it). No source change was needed: TS 7 reports 0 errors, the suites pass on Node 26.12 and Python 3.15.1. + +**v4.4.2** (2026-10-22): SonarQube Cloud cleanup — the scan's 57 issues: 45 fixed in code, the 11 on the saved itch.io test pages excluded by `.sonarcloud.properties`, css:S8776 (Tailwind `@utility`) left for the owner to mark a false positive (gotcha 41). Worker paths and the gate cookie's padding are trimmed by `trimEnd()` (a `/\/+$/` on the public 404 path was quadratic), the feed / collection parsers have no backtracking regexes left, `compareCodeUnits` makes the code-unit sorts explicit, the NSFW notices are one selector list (each tested). Supply chain: hash-locked Python deps + wheels only in every workflow (gotcha 40), `npm ci --ignore-scripts` in CI and the release builds, `npm run cf:check` instead of `npx wrangler`. `scripts/i18n-edit.mjs` reads its operations from stdin (`node scripts/i18n-edit.mjs < ops.json`). Ships the toolchain upgrade and Dependabot #141–#148, #150, #152, #153. **v4.4.1** (2026-10-16): the browser extension and the Telegram bot are retired — both repos (`poli0981/itch-f2p-extension`, `poli0981/telegram-scraper-bot`) were already public archives; an Announcements discussion says so. Removed: `POST /api/ingest` (`ingest.ts`, `requireService`, `RL_INGEST`, `ACCESS_AUD_INGEST`, the idempotency store — it never received a game: 0 `ext` rows in production D1), the admin's "Browser extension" auto-approve source and "Extension" label, the About page's Telegram numeric-ID note. The policies (Privacy, ToS, EULA + installer RTF, Disclaimer, Security; EN + VI) only lost the extension / ingest wording: "Last updated" 2026-10-16, `LEGAL_VERSION` deliberately unchanged (nothing new to accept, ToS §14). Unused repo secrets `GH_TOKEN`, `TELEGRAM_BOT_TOKEN`, `DISCUSSION_*` deleted. Ships Dependabot #131 (vite 8.3.4), #135 (tauri crates), #136 (serde_json), #137 (ruff). diff --git a/bash/cf_build_status.sh b/bash/cf_build_status.sh index 4a91f71..c5d8ff0 100755 --- a/bash/cf_build_status.sh +++ b/bash/cf_build_status.sh @@ -36,7 +36,7 @@ api() { } output() { - if [ -n "${GITHUB_OUTPUT:-}" ]; then echo "$1" >>"$GITHUB_OUTPUT"; fi + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then echo "$1" >>"$GITHUB_OUTPUT"; fi } commits=$(api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" \ @@ -44,14 +44,14 @@ commits=$(api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" \ declare -A parent=() sha="" while read -r commit first_parent; do - [ -n "$commit" ] || continue - [ -n "$sha" ] || sha=$commit + [[ -n "$commit" ]] || continue + [[ -n "$sha" ]] || sha=$commit parent[$commit]=$first_parent done <<<"$commits" now=$(date -u +%s) for _ in $(seq 1 "$MAX_COMMITS"); do - [ -n "$sha" ] && [ -n "${parent[$sha]+set}" ] || break + [[ -n "$sha" && -n "${parent[$sha]+set}" ]] || break run=$(api --method GET "repos/$GITHUB_REPOSITORY/commits/$sha/check-runs" \ -f check_name="$CHECK_NAME" -f filter=latest --jq \ '[.check_runs[] | select(.app.slug == "cloudflare-workers-and-pages")] | first @@ -67,7 +67,7 @@ for _ in $(seq 1 "$MAX_COMMITS"); do ;; *) age_h=$(((now - $(date -u -d "$completed" +%s)) / 3600)) - if [ "$age_h" -lt "$WINDOW_H" ]; then + if [[ "$age_h" -lt "$WINDOW_H" ]]; then echo "::error::The Cloudflare build of ${sha:0:7} ended '$conclusion' — freeitchgames.win is not up to date: $url" output "failed_id=$id" output "failed_url=$url" diff --git a/changelog/4.x/ver4.4.x.md b/changelog/4.x/ver4.4.x.md index fb11dc6..f8fda10 100644 --- a/changelog/4.x/ver4.4.x.md +++ b/changelog/4.x/ver4.4.x.md @@ -1,6 +1,53 @@ # Changelog — 4.4.x -Release notes for 4.4.0 and 4.4.1, newest first. Every version: [CHANGELOG.md](../../CHANGELOG.md). +Release notes for 4.4.0 to 4.4.2, newest first. Every version: [CHANGELOG.md](../../CHANGELOG.md). + +## [4.4.2] - 2026-10-22 (Toolchain upgrade, code-scan fixes) + +A maintenance release: nothing you would notice changes on the site or in the apps. The project now +builds with TypeScript 7, Node 26 and Python 3.15, and the issues a SonarQube Cloud scan of the +repository reported are fixed — the one that mattered was a path check in the Worker's not-found +handler that a crafted address could make expensive. + +### Upgrade notes + +- **Nothing to accept again**: no policy changed. +- **Desktop and Android apps**: install 4.4.2 over 4.4.1 as usual. +- **Building or running your own copy?** Use Node 26 for the web app and Python 3.15 for the + pipeline. The Python dependencies are now pinned with hashes: `pip install -r requirements-dev.txt` + works as before; to change one, edit `requirements.in` or `requirements-dev.in` and recompile with + pip-tools (the command is at the top of each file). + +### Fixed + +- **Worker**: trailing slashes on request paths (not-found handler, admin routes, the Access issuer) + and the verification cookie's base64 padding are trimmed by a loop instead of a regular expression + that backtracked — a path with a long run of slashes cost quadratic CPU time on a public route. The + itch.io feed and collection parsers lost three more expressions of that kind; a collection link's + attributes are now read by splitting on quotes. +- **Pipeline**: itch.io's two mature-content notices are matched by one CSS selector, so either of + them still marks a game NSFW (each now has a test). A few checks are written out so the scanner can + follow them, without a change in behaviour, and the scraper no longer sets up plain-http requests + (every page it fetches is https). + +### Changed + +- **Build tooling** (#149, #154): TypeScript 7.0, the native compiler, runs the type check, with the + 6.0 API installed beside it for typescript-eslint. Node 26 builds the web app and the release apps, + and npm runs only the install scripts of esbuild and workerd. Python 3.15 runs the pipeline (UTF-8 + mode is now the default); its CI pin moved to `.github/python-version`, out of Dependabot's way. +- **Supply chain**: the pipeline's Python dependencies, indirect ones included, are locked with hashes + (`requirements.txt`, `requirements-dev.txt`, compiled from `requirements*.in`) and installed from + wheels only, so a new release of a dependency can no longer slip into the jobs that commit to the + catalog. CI and the release builds install npm packages with `--ignore-scripts`, and the Workers + config check runs the project's own wrangler (`npm run cf:check`). +- **Code scanning**: `.sonarcloud.properties` keeps the saved itch.io test pages out of the scan; + the maintainer's `i18n-edit` script reads its operations from stdin instead of a file path. +- **Dependencies**: React 19.3.1 (#141), TanStack Query 5.104.4 (#142, #150), Radix UI primitives + (#143), `@tauri-apps/api` 2.12.4, CLI 2.12.3 and `plugin-opener` 2.7.2 (#144) with the matching + crates `tauri-build` 2.7.3, `tauri-plugin-opener` 2.7.2 and `tauri-plugin-single-instance` 2.5.4 + (#146), lucide-react 1.59.0 (#152), Vitest 5.0.5 (#153), Ruff 0.17.1 and Pillow 13 (#147), + `actions/setup-java` 6.1.0 and `dtolnay/rust-toolchain` in the release workflows (#148). ## [4.4.1] - 2026-10-16 (Extension and bot retired) diff --git a/docs/dev_env.md b/docs/dev_env.md index 1cf40d6..9f73139 100644 --- a/docs/dev_env.md +++ b/docs/dev_env.md @@ -47,6 +47,11 @@ pip install -r requirements-dev.txt Python 3.15 reads and writes files as UTF-8 by default (UTF-8 mode, PEP 686), so Windows no longer needs `PYTHONUTF8=1`. +Both requirements files pin every package with its hashes; pip checks them on install. To add or +change a dependency, edit `requirements.in` or `requirements-dev.in` and recompile the lock with +pip-tools — the command is at the top of each `.in` file (run it on Python 3.14, which Dependabot +can read; CLAUDE.md, gotcha 40). + ### Checks (same as `python-ci.yml`) ```sh @@ -171,7 +176,7 @@ npm run knip # unused files / exports / dependencies npm test # Vitest: app + Worker; D1 tests start a local workerd npx tsc -b # type-check only (npm run build runs it too) npm run build -npx wrangler deploy --dry-run --env="" # validates the Worker config, deploys nothing +npm run cf:check # wrangler deploy --dry-run: validates the Worker config, deploys nothing ``` After editing `wrangler.jsonc`, run `npm run cf:types` and commit the regenerated diff --git a/docs/i18n/vi/dev_env.md b/docs/i18n/vi/dev_env.md index c70d597..01dcc26 100644 --- a/docs/i18n/vi/dev_env.md +++ b/docs/i18n/vi/dev_env.md @@ -49,6 +49,10 @@ pip install -r requirements-dev.txt Python 3.15 mặc định đọc và ghi file dưới dạng UTF-8 (UTF-8 mode, PEP 686), nên trên Windows không cần đặt `PYTHONUTF8=1` nữa. +Hai file requirements ghim mọi package kèm hash; pip kiểm tra hash khi cài. Muốn thêm hoặc đổi một +dependency, sửa `requirements.in` hoặc `requirements-dev.in` rồi compile lại lock bằng pip-tools — +lệnh nằm ở đầu mỗi file `.in` (chạy bằng Python 3.14, bản Dependabot đọc được; CLAUDE.md, gotcha 40). + ### Kiểm tra (giống `python-ci.yml`) ```sh @@ -173,7 +177,7 @@ npm run knip # file / export / dependency không dùng npm test # Vitest: app + Worker; test D1 khởi động một workerd local npx tsc -b # chỉ kiểm tra kiểu (npm run build cũng chạy bước này) npm run build -npx wrangler deploy --dry-run --env="" # kiểm tra cấu hình Worker, không deploy gì +npm run cf:check # wrangler deploy --dry-run: kiểm tra cấu hình Worker, không deploy gì ``` Sau khi sửa `wrangler.jsonc`, chạy `npm run cf:types` và commit file `worker-configuration.d.ts` vừa diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..1c314c2 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,11 @@ +# Lint and test tools on top of the runtime pins (`-c`). requirements-dev.txt is +# the hash-locked compile of this file; regenerate it after an edit (CLAUDE.md, gotcha 40): +# pip-compile --generate-hashes --allow-unsafe --strip-extras requirements-dev.in +-c requirements.txt +-r requirements.in +ruff>=0.17.1 +pytest>=9.1.1 +vulture>=2.16 +pillow>=13.0.0 +# pytest needs it on Windows only; listed so a lock compiled on Linux (Dependabot) still installs there. +colorama diff --git a/requirements-dev.txt b/requirements-dev.txt index 9a43c53..af516b0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,360 @@ --r requirements.txt -ruff>=0.17.1 -pytest>=9.1.1 -vulture>=2.16 -pillow>=13.0.0 +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements-dev.in +# +beautifulsoup4==4.15.0 \ + --hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \ + --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 + # via + # -c requirements.txt + # -r requirements.in +certifi==2026.10.19 \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b + # via + # -c requirements.txt + # requests +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via + # -c requirements.txt + # requests +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via + # -r requirements-dev.in + # pytest +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # -c requirements.txt + # requests +iniconfig==2.3.1 \ + --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ + --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 + # via pytest +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via pytest +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via -r requirements-dev.in +pluggy==1.7.0 \ + --hash=sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec \ + --hash=sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8 + # via pytest +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via pytest +pytest==9.1.1 \ + --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + # via -r requirements-dev.in +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # -c requirements.txt + # -r requirements.in +ruff==0.17.1 \ + --hash=sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca \ + --hash=sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a \ + --hash=sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a \ + --hash=sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74 \ + --hash=sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4 \ + --hash=sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640 \ + --hash=sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f \ + --hash=sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7 \ + --hash=sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785 \ + --hash=sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1 \ + --hash=sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05 \ + --hash=sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab \ + --hash=sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978 \ + --hash=sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb \ + --hash=sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467 \ + --hash=sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189 \ + --hash=sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4 \ + --hash=sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a + # via -r requirements-dev.in +soupsieve==3.0.4 \ + --hash=sha256:8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf \ + --hash=sha256:924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0 + # via + # -c requirements.txt + # beautifulsoup4 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # -c requirements.txt + # beautifulsoup4 +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # -c requirements.txt + # -r requirements.in + # requests +vulture==2.16 \ + --hash=sha256:6e0f1c312cef1c87856957e5c2ca9608834a7c794c2180477f30bf0e4cc58eee \ + --hash=sha256:f8d9f6e2af03011664a3c6c240c9765b3f392917d3135fddca6d6a68d359f717 + # via -r requirements-dev.in diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..6b3dcb7 --- /dev/null +++ b/requirements.in @@ -0,0 +1,6 @@ +# Runtime dependencies of the data pipeline (scripts/). requirements.txt is the +# hash-locked compile of this file; regenerate it after an edit (CLAUDE.md, gotcha 40): +# pip-compile --generate-hashes --allow-unsafe --strip-extras requirements.in +requests>=2.34.2,<3 +beautifulsoup4>=4.15.0,<5 +urllib3>=2.8.0,<3 diff --git a/requirements.txt b/requirements.txt index b6c4fe4..b9c4dfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,210 @@ -requests>=2.34.2,<3 -beautifulsoup4>=4.15.0,<5 -urllib3>=2.8.0,<3 +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements.in +# +beautifulsoup4==4.15.0 \ + --hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \ + --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 + # via -r requirements.in +certifi==2026.10.19 \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b + # via requests +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via requests +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via -r requirements.in +soupsieve==3.0.4 \ + --hash=sha256:8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf \ + --hash=sha256:924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0 + # via beautifulsoup4 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via beautifulsoup4 +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # -r requirements.in + # requests diff --git a/scripts/scraper.py b/scripts/scraper.py index b4479db..2191bf1 100644 --- a/scripts/scraper.py +++ b/scripts/scraper.py @@ -77,8 +77,8 @@ def create_session() -> requests.Session: raise_on_status=False, ) adapter = HTTPAdapter(max_retries=retry) + # Every URL we fetch is canonical https (canonical.py); no adapter for plain http. session.mount("https://", adapter) - session.mount("http://", adapter) return session @@ -233,7 +233,9 @@ def find_title(soup: BeautifulSoup) -> str: def has_info_panel(soup: BeautifulSoup) -> bool: wrapper = soup.find("div", class_="info_panel_wrapper") - return bool(wrapper and wrapper.find("table")) + if not wrapper: + return False + return wrapper.find("table") is not None def is_password_protected(soup: BeautifulSoup) -> bool: @@ -370,7 +372,9 @@ def _cap(text: str) -> str: def extract_tagline(soup: BeautifulSoup) -> str: """The creator's "short description or tagline" (itch.io puts it in og:description).""" meta = soup.find("meta", property="og:description") - text = " ".join(str(meta.get("content", "")).split()) if meta else "" + if not meta: + return NA + text = " ".join(str(meta.get("content", "")).split()) return _cap(text) if text else NA @@ -399,8 +403,10 @@ def extract_description(soup: BeautifulSoup) -> str: # --------------------------------------------------------------------------- def extract_thumbnail(soup: BeautifulSoup) -> str: meta = soup.find("meta", property="og:image") - if meta and str(meta.get("content", "")).startswith("http"): - return meta["content"] + if meta: + content = str(meta.get("content", "")) + if content.startswith("http"): + return content ss = soup.find("div", class_="screenshot_list") if ss: @@ -429,7 +435,8 @@ def detect_nsfw(soup: BeautifulSoup, tags, *texts: str) -> str: return "Yes" if any(text != NA and _NSFW_RE.search(text) for text in texts): return "Yes" - if soup.find("div", class_=["view_game_warning", "mature_content_notice"]): + # Either notice (a class list in find() would mean the same; ".a.b" would need both). + if soup.select_one("div.view_game_warning, div.mature_content_notice"): return "Yes" return "No" diff --git a/scripts/update_info.py b/scripts/update_info.py index 8fde4e7..f9ea586 100644 --- a/scripts/update_info.py +++ b/scripts/update_info.py @@ -78,7 +78,7 @@ def record_outcome(patch: dict, url: str, outcome: str) -> None: patch["ingest_outcomes"][url] = {"outcome": outcome, "at": now_iso()} -def main(argv: list[str] | None = None) -> int: +def main(argv: list[str] | None = None) -> None: ap = argparse.ArgumentParser(description="Scrape queued links into a patch.") ap.add_argument("--out", required=True, help="patch file to write") ap.add_argument("--deadline-min", type=float, default=50.0) @@ -115,7 +115,7 @@ def main(argv: list[str] | None = None) -> int: if not queue: print("No links to process (temp_link.json empty and no INPUT_URL).") write_patch(args.out, patch) - return 0 + return catalog_urls = get_all_urls(load_all_games()) deleted_urls = {e.get("url") for e in load_json(DELETED_LOG)} @@ -216,8 +216,7 @@ def main(argv: list[str] | None = None) -> int: f"{stats['duplicate']} duplicates, {stats['deleted']} previously deleted, " f"{stats['invalid']} invalid, {stats['retry']} to retry, {stats['gave_up']} given up." ) - return 0 if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/tests/test_parser.py b/tests/test_parser.py index e8c36db..c9bb755 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -127,6 +127,14 @@ def test_mature_notice_marks_nsfw(): assert record["nsfw"] == "Yes" +@pytest.mark.parametrize("notice", ["view_game_warning", "mature_content_notice"]) +def test_either_mature_notice_alone_marks_nsfw(notice): + from bs4 import BeautifulSoup + + soup = BeautifulSoup(f'
', "html.parser") + assert detect_nsfw(soup, [], NA) == "Yes" + + @pytest.mark.parametrize( ("tags", "description", "expected"), [ diff --git a/tests/test_update_info.py b/tests/test_update_info.py index e684049..2901628 100644 --- a/tests/test_update_info.py +++ b/tests/test_update_info.py @@ -12,7 +12,7 @@ def _run(monkeypatch, pages: dict[str, Page], queue: list, **env): monkeypatch.setattr( update_info, "fetch_with_backoff", lambda _s, url, _p, _c: pages.get(url, Page(status=500)) ) - assert update_info.main(["--out", "patch.json"]) == 0 + update_info.main(["--out", "patch.json"]) return read_json("patch.json") @@ -139,7 +139,7 @@ def test_duplicate_spelling_survives_a_rate_limit_stop(repo, monkeypatch): queue = ["https://dev.itch.io/a", "https://dev.itch.io/a"] save_json("scripts/temp_link.json", queue) monkeypatch.setattr(update_info, "fetch_with_backoff", lambda *_a: None) - assert update_info.main(["--out", "patch.json"]) == 0 + update_info.main(["--out", "patch.json"]) assert _apply(read_json("patch.json")) == queue @@ -161,7 +161,7 @@ def test_dispatched_url_is_processed_first_and_queued_until_done(repo, monkeypat save_json("scripts/temp_link.json", ["https://dev.itch.io/backlog"]) monkeypatch.setenv("INPUT_URL", "https://dev.itch.io/Dispatched/") monkeypatch.setattr(update_info, "fetch_with_backoff", fetch) - assert update_info.main(["--out", "patch.json"]) == 0 + update_info.main(["--out", "patch.json"]) patch = read_json("patch.json") assert seen[0] == "https://dev.itch.io/dispatched" assert patch["queue_add"] == ["https://dev.itch.io/dispatched"] @@ -205,7 +205,7 @@ def test_failures_count_only_when_spaced_out(repo, monkeypatch): "2026-09-18T14:00:00Z", # 3 -> given up ): monkeypatch.setattr(update_info, "now_iso", lambda s=stamp: s) - assert update_info.main(["--out", "patch.json"]) == 0 + update_info.main(["--out", "patch.json"]) write_patch("patch.json", read_json("patch.json")) assert apply_patch.main(["patch.json"]) == 0 entry = read_json("scripts/state/ingest_state.json").get(url, {}) diff --git a/webapp/README.md b/webapp/README.md index d9a3684..c86ab42 100644 --- a/webapp/README.md +++ b/webapp/README.md @@ -78,7 +78,7 @@ npm run lint # ESLint npm run knip # unused files / exports / deps npm test # Vitest: app + Worker (D1 tests run a local workerd) npm run build # includes tsc -b (or run npx tsc -b on its own) -npx wrangler deploy --dry-run --env="" +npm run cf:check # wrangler deploy --dry-run (deploys nothing) ``` After editing `wrangler.jsonc`, run `npm run cf:types` and commit the regenerated diff --git a/webapp/admin/tracking.ts b/webapp/admin/tracking.ts index 5615299..5b5cf53 100644 --- a/webapp/admin/tracking.ts +++ b/webapp/admin/tracking.ts @@ -44,7 +44,7 @@ export const useTracking = create()( set((s) => urls.length === 0 ? s - : { batches: capBatches([{ id: `${at}-${Math.random().toString(36).slice(2, 8)}`, label, at, urls }, ...s.batches]) }, + : { batches: capBatches([{ id: `${at}-${crypto.randomUUID().slice(0, 8)}`, label, at, urls }, ...s.batches]) }, ), forget: (id) => set((s) => ({ batches: s.batches.filter((b) => b.id !== id) })), }), diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 2edb54d..a885a92 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "webapp", - "version": "4.4.1", + "version": "4.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "webapp", - "version": "4.4.1", + "version": "4.4.2", "license": "MIT", "dependencies": { "@fontsource-variable/geist": "^5.3.0", diff --git a/webapp/package.json b/webapp/package.json index 3b3ee6f..565f5c8 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,7 +1,7 @@ { "name": "webapp", "private": true, - "version": "4.4.1", + "version": "4.4.2", "license": "MIT", "type": "module", "engines": { @@ -19,6 +19,7 @@ "test": "vitest run", "check:tauri": "node scripts/check-tauri-versions.mjs", "cf:types": "wrangler types --env-file=types.env --strict-vars=false", + "cf:check": "wrangler deploy --dry-run --env=", "cf:dev": "wrangler dev" }, "dependencies": { diff --git a/webapp/scripts/i18n-edit.mjs b/webapp/scripts/i18n-edit.mjs index 28177a4..5e63b50 100644 --- a/webapp/scripts/i18n-edit.mjs +++ b/webapp/scripts/i18n-edit.mjs @@ -2,11 +2,14 @@ // (src/lib/i18n/en.ts and vi.ts) with the TypeScript parser, so multi-line // values and section comments survive. // -// node scripts/i18n-edit.mjs ops.json +// node scripts/i18n-edit.mjs < ops.json # PowerShell: Get-Content ops.json | node scripts/i18n-edit.mjs // ops.json: { "remove": ["exact.key", "prefix.*"], // "set": { "key": { "en": "…", "vi": "…" } } } // node scripts/i18n-edit.mjs --unused # list keys never referenced in src/ // +// The operations come on stdin, not as a file path: the script only ever +// touches the two dictionaries. +// // New keys are appended at the end of the object; the compiler enforces that // vi.ts has exactly the keys of en.ts (Record). import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs' @@ -50,7 +53,11 @@ if (process.argv[2] === '--unused') { process.exit(0) } -const ops = JSON.parse(readFileSync(process.argv[2], 'utf8')) +if (process.stdin.isTTY) { + console.error('usage: node scripts/i18n-edit.mjs < ops.json (or --unused)') + process.exit(2) +} +const ops = JSON.parse(readFileSync(0, 'utf8')) const matches = (key, pattern) => (pattern.endsWith('.*') ? key.startsWith(pattern.slice(0, -1)) : key === pattern) for (const [lang, path] of Object.entries(FILES)) { diff --git a/webapp/src-tauri/Cargo.lock b/webapp/src-tauri/Cargo.lock index 3d4a7c5..81b09e5 100644 --- a/webapp/src-tauri/Cargo.lock +++ b/webapp/src-tauri/Cargo.lock @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "free-games-itchio-webapp" -version = "4.4.1" +version = "4.4.2" dependencies = [ "serde", "serde_json", diff --git a/webapp/src-tauri/Cargo.toml b/webapp/src-tauri/Cargo.toml index ce1ca54..891bc1a 100644 --- a/webapp/src-tauri/Cargo.toml +++ b/webapp/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "free-games-itchio-webapp" -version = "4.4.1" +version = "4.4.2" description = "Itch.io free games catalog viewer" authors = ["poli0981"] license = "MIT" diff --git a/webapp/src-tauri/tauri.conf.json b/webapp/src-tauri/tauri.conf.json index 0651834..8378205 100644 --- a/webapp/src-tauri/tauri.conf.json +++ b/webapp/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Itch.io Free Games DB", - "version": "4.4.1", + "version": "4.4.2", "identifier": "com.poli0981.freegamesitchio", "build": { "beforeDevCommand": "npm run dev", diff --git a/webapp/src/hooks/useSeoHead.ts b/webapp/src/hooks/useSeoHead.ts index b1412ea..432d4cb 100644 --- a/webapp/src/hooks/useSeoHead.ts +++ b/webapp/src/hooks/useSeoHead.ts @@ -23,7 +23,10 @@ export function useSeoHead(): void { const { pathname } = useLocation() useEffect(() => { if (isTauri()) return - const path = pathname.length > 1 ? pathname.replace(/\/+$/, '') : '/' + // A loop, not /\/+$/ (which backtracks on a long run of slashes). + let end = pathname.length + while (end > 1 && pathname[end - 1] === '/') end-- + const path = pathname.slice(0, end) const canonical = headTag('link[rel="canonical"]', () => { const link = document.createElement('link') link.rel = 'canonical' diff --git a/webapp/src/lib/about.ts b/webapp/src/lib/about.ts index 475dc4b..66cf556 100644 --- a/webapp/src/lib/about.ts +++ b/webapp/src/lib/about.ts @@ -8,7 +8,7 @@ export interface ThirdParty { export const APP = { name: 'Free Itch Games', - version: '4.4.1', + version: '4.4.2', repo: 'https://github.com/poli0981/free-games-itchio-list', license: 'code MIT · data CC BY 4.0', } as const diff --git a/webapp/src/lib/analytics.ts b/webapp/src/lib/analytics.ts index 9f2b1e8..2e37f20 100644 --- a/webapp/src/lib/analytics.ts +++ b/webapp/src/lib/analytics.ts @@ -46,8 +46,8 @@ export function aiDisclosureCounts(games: Game[]): CountEntry[] { } function parseRating(rating: string): number | null { - const n = parseFloat(rating) - return isFinite(n) ? n : null + const n = Number.parseFloat(rating) + return Number.isFinite(n) ? n : null } export function classifyOnline(games: Game[]): { online: number; offline: number } { diff --git a/webapp/worker/access.ts b/webapp/worker/access.ts index 2d2203e..315c239 100644 --- a/webapp/worker/access.ts +++ b/webapp/worker/access.ts @@ -8,6 +8,7 @@ * here: signature against the team's JWKS, issuer, and the application's AUD. */ import { createRemoteJWKSet, jwtVerify, type JWTPayload } from 'jose' +import { trimEnd } from './text' export interface AccessEnv { ACCESS_TEAM_DOMAIN: string // https://.cloudflareaccess.com @@ -39,7 +40,7 @@ async function verify(request: Request, env: AccessEnv, audience: string): Promi const token = request.headers.get('cf-access-jwt-assertion') if (!token) throw new AccessError('missing Access token') // The issuer is the bare origin; tolerate a trailing slash pasted from the dashboard. - const team = env.ACCESS_TEAM_DOMAIN.trim().replace(/\/+$/, '') + const team = trimEnd(env.ACCESS_TEAM_DOMAIN.trim(), '/') try { const { payload } = await jwtVerify(token, jwks(team), { issuer: team, audience }) return payload diff --git a/webapp/worker/admin.ts b/webapp/worker/admin.ts index 45fda38..f906ec2 100644 --- a/webapp/worker/admin.ts +++ b/webapp/worker/admin.ts @@ -50,6 +50,7 @@ import { type QueueStore, } from './queue' import { repoOps, type IngestView, type RepoOps } from './repo' +import { trimEnd } from './text' const STATUSES: readonly CandidateStatus[] = ['pending', 'rejected', 'queued', 'ingested', 'failed'] const THUMB_WIDTHS = [160, 640] @@ -538,7 +539,7 @@ export async function adminApi(request: Request, url: URL, ctx: AdminContext): P } async function route(request: Request, url: URL, ctx: AdminContext): Promise { - const route = `${request.method} ${url.pathname.replace(/\/+$/, '')}` + const route = `${request.method} ${trimEnd(url.pathname, '/')}` switch (route) { case 'GET /api/admin/me': return json({ email: ctx.actor, writes: ctx.repo !== null, repo: ctx.repoName ?? null }) diff --git a/webapp/worker/catalog.ts b/webapp/worker/catalog.ts index fbb40b1..e9ca92e 100644 --- a/webapp/worker/catalog.ts +++ b/webapp/worker/catalog.ts @@ -11,6 +11,7 @@ const CHUNK_SIZE = 500 const MIN_FILL = 250 +const DEFAULT_SIZES = { chunkSize: CHUNK_SIZE, minFill: MIN_FILL } /** The only fields a maintainer edits; everything else is scraper-owned. */ const EDITABLE_FIELDS = ['safe_virus', 'notes', 'nsfw'] as const @@ -181,7 +182,7 @@ export function applyAdminChange( edits: Map, removals: { url: string; reason: string }[], now: Date, - sizes: { chunkSize: number; minFill: number } = { chunkSize: CHUNK_SIZE, minFill: MIN_FILL }, + sizes: { chunkSize: number; minFill: number } = DEFAULT_SIZES, ): CatalogChange { const games = files.chunks.flat().map((g) => ({ ...g })) const editedUrls: string[] = [] diff --git a/webapp/worker/collections.ts b/webapp/worker/collections.ts index 7f23fe1..7925606 100644 --- a/webapp/worker/collections.ts +++ b/webapp/worker/collections.ts @@ -80,9 +80,18 @@ export function collectionPageUrl(id: string, slug: string, page: number): strin return `${collectionUrl(id, slug)}?format=json${page > 1 ? `&page=${page}` : ''}` } +const ATTRIBUTE_NAME = /^[a-zA-Z_:][-a-zA-Z0-9_:.]*$/ + +/** A tag's double-quoted attributes, read by splitting on the quotes (a name="value" regex backtracks). */ function attributes(tag: string): Map { const out = new Map() - for (const m of tag.matchAll(/([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*"([^"]*)"/g)) out.set(m[1].toLowerCase(), m[2]) + const parts = tag.split('"') // outside, value, outside, value, …; an unclosed value is dropped + for (let i = 1; i < parts.length - 1; i += 2) { + const before = parts[i - 1].trimEnd() + if (!before.endsWith('=')) continue + const name = before.slice(0, -1).trimEnd().split(/\s+/).pop() ?? '' + if (ATTRIBUTE_NAME.test(name)) out.set(name.toLowerCase(), parts[i]) + } return out } @@ -92,7 +101,7 @@ function hasClasses(attrs: Map, ...names: string[]): boolean { } function text(html: string): string { - return decodeEntities(html.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim() + return decodeEntities(html.replace(/<[^<>]*>/g, ' ')).replace(/\s+/g, ' ').trim() } function parseCell(cell: string): CollectionItem | null { diff --git a/webapp/worker/discover.ts b/webapp/worker/discover.ts index 593ccae..d995901 100644 --- a/webapp/worker/discover.ts +++ b/webapp/worker/discover.ts @@ -55,13 +55,13 @@ export function parseFeed(xml: string): FeedItem[] { const url = tag(block, 'link') ?? tag(block, 'guid') if (!url) continue const rawTitle = tag(block, 'title') ?? '' - const brackets = [...rawTitle.matchAll(/\[([^\]]+)\]/g)].map((m) => m[1].trim()) + const brackets = [...rawTitle.matchAll(/\[([^[\]]+)\]/g)].map((m) => m[1].trim()) const genre = brackets.find((b) => !NOT_GENRE.has(b.toLowerCase())) ?? null const price = tag(block, 'price') ?? '' const image = tag(block, 'imageurl') items.push({ url, - title: tag(block, 'plainTitle') ?? rawTitle.replace(/\s*\[[^\]]*\]/g, '').trim(), + title: tag(block, 'plainTitle') ?? rawTitle.replace(/\[[^[\]]*\]/g, '').replace(/\s+/g, ' ').trim(), image: image?.startsWith('https://img.itch.zone/') ? image : null, genre, free: /^[^0-9]*0+(?:[.,]0+)?$/.test(price) || brackets.some((b) => b.toLowerCase() === 'free'), diff --git a/webapp/worker/gate.ts b/webapp/worker/gate.ts index eed9295..99ce049 100644 --- a/webapp/worker/gate.ts +++ b/webapp/worker/gate.ts @@ -21,6 +21,7 @@ import { missingConfig, type WorkerEnv } from './env' import { challengePage, crawlerUnavailable } from './gate-page' import { HttpError, LOCAL_HOSTS, errorResponse, isObject, json, rateKey, readJson } from './http' +import { trimEnd } from './text' import { siteverify } from './turnstile' const COOKIE = '__Host-fig_gate' @@ -98,10 +99,8 @@ async function macKey(secret: string): Promise { const macInput = (iat: number) => encoder.encode(`fig-gate|v1|${iat}`) function toBase64Url(bytes: ArrayBuffer): string { - return btoa(String.fromCharCode(...new Uint8Array(bytes))) - .replace(/\+/g, '-') - .replace(/\//g, '_') - .replace(/=+$/, '') + const b64 = btoa(String.fromCharCode(...new Uint8Array(bytes))) + return trimEnd(b64.replace(/\+/g, '-').replace(/\//g, '_'), '=') } function fromBase64Url(text: string): Uint8Array { diff --git a/webapp/worker/repo.ts b/webapp/worker/repo.ts index 75fba6d..032635c 100644 --- a/webapp/worker/repo.ts +++ b/webapp/worker/repo.ts @@ -18,6 +18,7 @@ import { } from './catalog' import { GITHUB_CONFIG, missingConfig, type WorkerEnv } from './env' import { commitWithRetry, headOid, listDir, readFile, type CommitInput, type GitHubEnv } from './github' +import { compareCodeUnits } from './text' const TEMP_LINK = 'scripts/temp_link.json' const DELETED_LOG = 'scripts/deleted_games.json' @@ -56,7 +57,7 @@ async function readJsonFile(env: GitHubEnv, path: string, head: string, fallb /** The catalog files at `head`, chunks in the same order as data_store.py. */ async function readCatalog(env: GitHubEnv, head: string): Promise { - const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort() + const names = (await listDir(env, 'data_game', head)).filter((n) => CHUNK_FILE.test(n)).sort(compareCodeUnits) const [chunks, index, history, deleted] = await Promise.all([ Promise.all(names.map((n) => readJsonFile(env, `data_game/${n}`, head, []))), readJsonFile(env, 'data_game/index.json', head, null), @@ -92,7 +93,7 @@ export function withQueued(queueText: string | null, urls: string[]): { text: st export function withUnblocked(currentText: string | null, urls: string[]): { text: string; added: number } { const parsed: unknown = currentText === null ? [] : JSON.parse(currentText) const current = Array.isArray(parsed) ? parsed.filter((u): u is string => typeof u === 'string') : [] - const next = [...new Set([...current, ...urls])].sort() + const next = [...new Set([...current, ...urls])].sort(compareCodeUnits) return { text: dumps(next), added: next.length - current.length } } diff --git a/webapp/worker/spa.ts b/webapp/worker/spa.ts index 3b64db2..bca4d5c 100644 --- a/webapp/worker/spa.ts +++ b/webapp/worker/spa.ts @@ -16,6 +16,7 @@ */ import { catalogUrls, type DataSource } from './data' import { json } from './http' +import { trimEnd } from './text' // Keep in sync with the in src/App.tsx. const ROUTES = new Set(['/', '/games', '/charts', '/removed', '/settings', '/about', '/suggest']) @@ -52,7 +53,7 @@ async function isGameSlug(slug: string, src: DataSource): Promise { } async function isAppRoute(pathname: string, src: DataSource): Promise { - const path = pathname.length > 1 ? pathname.replace(/\/+$/, '') : pathname + const path = pathname.length > 1 ? trimEnd(pathname, '/') : pathname if (ROUTES.has(path) || ERROR_PREVIEW_ROUTE.test(path)) return true const game = GAME_ROUTE.exec(path) return game !== null && (await isGameSlug(game[1], src)) @@ -72,7 +73,7 @@ export async function handleMiss(request: Request, url: URL, assets: Fetcher, ga const cors = url.pathname.startsWith('/data/') ? { 'Access-Control-Allow-Origin': '*' } : undefined return json({ error: 'not_found' }, 404, cors) } - const moved = MOVED[url.pathname.replace(/\/+$/, '')] + const moved = MOVED[trimEnd(url.pathname, '/')] if (moved) return Response.redirect(new URL(moved + url.search, url).toString(), 301) // Before the shell is fetched, so a conditional request (If-None-Match) // without a pass gets the verification page, never a 304 for a cached shell. diff --git a/webapp/worker/test/d1.ts b/webapp/worker/test/d1.ts index c4bdeff..4d8385e 100644 --- a/webapp/worker/test/d1.ts +++ b/webapp/worker/test/d1.ts @@ -3,6 +3,7 @@ * for tests of the SQL in queue.ts. Runs workerd through wrangler's platform proxy. */ import { getPlatformProxy } from 'wrangler' +import { compareCodeUnits } from '../text' const MIGRATIONS = import.meta.glob('../migrations/*.sql', { query: '?raw', import: 'default', eager: true }) @@ -11,9 +12,9 @@ export async function testDb(): Promise<{ db: D1Database; dispose: () => Promise configPath: new URL('./wrangler.test.jsonc', import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'), persist: false, }) - for (const name of Object.keys(MIGRATIONS).sort()) { + for (const name of Object.keys(MIGRATIONS).sort(compareCodeUnits)) { const statements = MIGRATIONS[name] - .replace(/--.*$/gm, '') + .replace(/--[^\r\n]*/g, '') .split(';') .map((s) => s.trim()) .filter(Boolean) diff --git a/webapp/worker/test/discover.test.ts b/webapp/worker/test/discover.test.ts index 1566282..0bbbe8d 100644 --- a/webapp/worker/test/discover.test.ts +++ b/webapp/worker/test/discover.test.ts @@ -35,6 +35,12 @@ describe('parseFeed', () => { expect(items[2].free).toBe(false) expect(items[3].image).toBeNull() // only img.itch.zone covers are kept }) + + it('falls back to the title without its [tags] when there is no plainTitle', () => { + const feed = `Night Shift [Free] [Horror] [Windows] +https://studio.itch.io/night-shift$0.00` + expect(parseFeed(feed)[0]).toMatchObject({ title: 'Night Shift', genre: 'Horror', free: true }) + }) }) describe('pickFeed', () => { diff --git a/webapp/worker/text.test.ts b/webapp/worker/text.test.ts new file mode 100644 index 0000000..4fdce9b --- /dev/null +++ b/webapp/worker/text.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import { compareCodeUnits, decodeEntities, trimEnd } from './text' + +describe('decodeEntities', () => { + it('decodes decimal, hex and named references', () => { + expect(decodeEntities('ABC <b> Rock & Roll ')).toBe('ABC Rock & Roll') + }) +}) + +describe('trimEnd', () => { + it.each([ + ['/games/', '/', '/games'], + ['/games///', '/', '/games'], + ['/games', '/', '/games'], + ['///', '/', ''], + ['', '/', ''], + ['c2lnbg==', '=', 'c2lnbg'], + ])('%j without trailing %j', (text, char, expected) => { + expect(trimEnd(text, char)).toBe(expected) + }) + + it('takes linear time on a long run that is not at the end (a /\\/+$/ regex did not)', () => { + const path = `/${'/'.repeat(200_000)}x` + expect(trimEnd(path, '/')).toBe(path) + }) +}) + +describe('compareCodeUnits', () => { + it('orders like a bare sort(): code units, not the locale', () => { + expect(['game_info_010.json', 'alpha', 'game_info_002.json', 'Zeta', 'b', 'B'].sort(compareCodeUnits)).toEqual([ + 'B', + 'Zeta', + 'alpha', + 'b', + 'game_info_002.json', + 'game_info_010.json', + ]) + expect(compareCodeUnits('same', 'same')).toBe(0) + }) +}) diff --git a/webapp/worker/text.ts b/webapp/worker/text.ts index 4b08a9e..d449652 100644 --- a/webapp/worker/text.ts +++ b/webapp/worker/text.ts @@ -2,7 +2,7 @@ export function decodeEntities(text: string): string { return text .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n))) - .replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(parseInt(n, 16))) + .replace(/&#x([0-9a-f]+);/gi, (_, n: string) => String.fromCodePoint(Number.parseInt(n, 16))) .replace(/"/g, '"') .replace(/'/g, "'") .replace(/</g, '<') @@ -11,3 +11,19 @@ export function decodeEntities(text: string): string { .replace(/&/g, '&') .trim() } + +/** + * `text` without its trailing `char`s. A loop, not `/x+$/`: that regex retries + * from every character of a long run that is not at the end (quadratic). + */ +export function trimEnd(text: string, char: string): string { + let end = text.length + while (end > 0 && text[end - 1] === char) end-- + return text.slice(0, end) +} + +/** Code-unit order, like a bare `sort()` and, for these ASCII names, Python's `sorted()`; not locale-aware. */ +export function compareCodeUnits(a: string, b: string): number { + if (a === b) return 0 + return a < b ? -1 : 1 +}