Compare commits
189
Commits
No files matched your search
+12
-5
@@ -26,6 +26,7 @@ updates:
|
||||
- 'vite'
|
||||
- '@vitejs/*'
|
||||
- 'typescript'
|
||||
- '@typescript/*'
|
||||
- 'typescript-eslint'
|
||||
- 'eslint*'
|
||||
- '@eslint/*'
|
||||
@@ -36,12 +37,14 @@ updates:
|
||||
- 'wrangler'
|
||||
- '@cloudflare/*'
|
||||
ignore:
|
||||
# typescript-eslint supports TypeScript < 6.1 only.
|
||||
# `typescript` is an alias of @typescript/typescript6 (the 6.0 API):
|
||||
# typescript-eslint supports TypeScript < 6.1 only. TypeScript 7 itself
|
||||
# is `@typescript/native`. Never move the alias to a new major.
|
||||
- dependency-name: 'typescript'
|
||||
versions: ['>=6.1']
|
||||
# Match the Node 24 runtime.
|
||||
update-types: ['version-update:semver-major']
|
||||
# Match the Node 26 runtime.
|
||||
- dependency-name: '@types/node'
|
||||
versions: ['>=25']
|
||||
versions: ['>=27']
|
||||
|
||||
- package-ecosystem: pip
|
||||
directory: /
|
||||
@@ -63,10 +66,14 @@ updates:
|
||||
actions:
|
||||
patterns: ['*']
|
||||
|
||||
# Weekly on Mondays like npm, so tauri-rs and tauri-js bumps land together:
|
||||
# `tauri build` (release workflows only) refuses crate / npm versions on a
|
||||
# different major.minor; Webapp CI's check:tauri step catches a one-sided bump.
|
||||
- package-ecosystem: cargo
|
||||
directory: /webapp/src-tauri
|
||||
schedule:
|
||||
interval: monthly
|
||||
interval: weekly
|
||||
day: monday
|
||||
groups:
|
||||
tauri-rs:
|
||||
patterns: ['tauri*']
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.15
|
||||
@@ -1,7 +1,12 @@
|
||||
# Manual re-scrape of every scraper-owned field (repairs mojibake, backfills
|
||||
# fields after a scraper upgrade). Always preserves safe_virus / notes / nsfw.
|
||||
# Runs through refresh.yml in full mode: one URL, or a rotating batch of the
|
||||
# games whose last full re-scrape is oldest — dispatch again to continue.
|
||||
# Queues refresh.yml in full mode: one URL, or a rotating batch of the games
|
||||
# whose last full re-scrape is oldest. Dispatch again to continue: the runs
|
||||
# wait FIFO in refresh.yml's concurrency group and each batch starts from the
|
||||
# latest main, so several dispatches in a row cover consecutive batches.
|
||||
#
|
||||
# A dispatch rather than a reusable-workflow call: called through `uses:`,
|
||||
# refresh.yml failed at startup ("workflow file issue", no annotation).
|
||||
name: Force update
|
||||
|
||||
on:
|
||||
@@ -22,22 +27,24 @@ on:
|
||||
|
||||
permissions: {}
|
||||
|
||||
# FIFO (`queue: max`): with the default queue a newer pending dispatch would
|
||||
# replace an older pending one and silently drop its `url`.
|
||||
concurrency:
|
||||
group: force-update
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
rescrape:
|
||||
queue:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: write
|
||||
uses: ./.github/workflows/refresh.yml
|
||||
with:
|
||||
full: true
|
||||
url: ${{ inputs.url }}
|
||||
budget: ${{ inputs.budget }}
|
||||
allow_mass_removal: ${{ inputs.allow_mass_removal }}
|
||||
secrets:
|
||||
DISCORD_CI_WEBHOOK: ${{ secrets.DISCORD_CI_WEBHOOK }}
|
||||
actions: write # dispatch refresh.yml (a workflow_dispatch from GITHUB_TOKEN does start a run)
|
||||
steps:
|
||||
- name: Queue a full refresh
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
REF: ${{ github.ref_name }}
|
||||
URL: ${{ inputs.url }}
|
||||
BUDGET: ${{ inputs.budget }}
|
||||
ALLOW_MASS_REMOVAL: ${{ inputs.allow_mass_removal }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh workflow run refresh.yml --ref "$REF" \
|
||||
-f full=true -f budget="$BUDGET" -f url="$URL" -f allow_mass_removal="$ALLOW_MASS_REMOVAL"
|
||||
echo "Queued a full refresh on \`$REF\` (budget $BUDGET${URL:+, $URL}) — see the Refresh catalog runs." \
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,5 +1,5 @@
|
||||
# Lint + tests for the data pipeline, and schema validation of the catalog.
|
||||
# Pushes by GitHub Apps (browser extension, admin Worker) also trigger this,
|
||||
# Pushes by the admin Worker's GitHub App also trigger this,
|
||||
# so every externally written data change is validated.
|
||||
name: Python CI
|
||||
|
||||
@@ -11,7 +11,7 @@ on:
|
||||
- 'data_game/**'
|
||||
- 'requirements*.txt'
|
||||
- 'pyproject.toml'
|
||||
- '.python-version'
|
||||
- '.github/python-version'
|
||||
- '.github/workflows/python-ci.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
@@ -21,7 +21,7 @@ on:
|
||||
- 'data_game/**'
|
||||
- 'requirements*.txt'
|
||||
- 'pyproject.toml'
|
||||
- '.python-version'
|
||||
- '.github/python-version'
|
||||
- '.github/workflows/python-ci.yml'
|
||||
|
||||
permissions: {}
|
||||
@@ -30,9 +30,6 @@ concurrency:
|
||||
group: python-ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PYTHONUTF8: '1'
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -48,7 +45,9 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
# Not .python-version: Dependabot's pip updater reads that file and
|
||||
# supports Python <= 3.14 only (CLAUDE.md gotcha 39).
|
||||
python-version-file: .github/python-version
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
# (a run that would remove or strike an implausible share of games
|
||||
# — usually itch.io markup changes — withholds the removals).
|
||||
#
|
||||
# Also called by force_update.yml with `full: true` (re-scrape every field).
|
||||
# Also dispatched with `full: true` (by force_update.yml, or by hand): re-scrape
|
||||
# every scraper-owned field of one URL or of the next batch.
|
||||
name: Refresh catalog
|
||||
|
||||
on:
|
||||
@@ -29,31 +30,17 @@ on:
|
||||
description: 'Apply removals even if the mass-change guard trips (confirm on itch.io first)'
|
||||
type: boolean
|
||||
default: false
|
||||
workflow_call:
|
||||
inputs:
|
||||
budget:
|
||||
type: number
|
||||
default: 0
|
||||
url:
|
||||
type: string
|
||||
required: false
|
||||
full:
|
||||
description: 'Re-scrape every scraper-owned field (rotates on the last full re-scrape)'
|
||||
type: boolean
|
||||
default: false
|
||||
allow_mass_removal:
|
||||
type: boolean
|
||||
default: false
|
||||
secrets:
|
||||
DISCORD_CI_WEBHOOK:
|
||||
required: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
# One refresh pipeline (scan → apply) at a time, FIFO — a scheduled refresh, a
|
||||
# manual one and a force-update batch never start from the same state, so an
|
||||
# older removal can't undo a newer healthy check. The name differs from
|
||||
# force_update.yml's own group on purpose: a called workflow asking for its
|
||||
# caller's group is cancelled as a deadlock.
|
||||
# manual one and a force-update batch never start from the same state (the scan
|
||||
# checks out the branch as it is when the run starts), so an older removal can't
|
||||
# undo a newer healthy check, and queued full batches move on through the catalog.
|
||||
concurrency:
|
||||
group: refresh-pipeline
|
||||
cancel-in-progress: false
|
||||
@@ -61,7 +48,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: '1'
|
||||
PYTHONUTF8: '1'
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
@@ -73,15 +59,18 @@ jobs:
|
||||
rate_limited: ${{ steps.refresh.outputs.rate_limited }}
|
||||
mass_change: ${{ steps.refresh.outputs.mass_change }}
|
||||
steps:
|
||||
# The branch's latest commit, not the one it had when the run was queued:
|
||||
# a run waiting behind another must see that run's refresh state.
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
python-version-file: .github/python-version
|
||||
cache: pip
|
||||
cache-dependency-path: requirements.txt
|
||||
|
||||
@@ -105,7 +94,7 @@ jobs:
|
||||
|
||||
- name: Upload patch
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: refresh-patch
|
||||
path: ${{ runner.temp }}/patch.json
|
||||
@@ -136,7 +125,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
python-version-file: .github/python-version
|
||||
cache: pip
|
||||
cache-dependency-path: requirements.txt
|
||||
|
||||
@@ -144,7 +133,7 @@ jobs:
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
- name: Download patch
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: refresh-patch
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
|
||||
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '17'
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
# (it hard-errors otherwise). Release builds use no caches at all.
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version-file: webapp/.node-version
|
||||
# No dependency cache in release builds: a poisoned cache could end
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Set up Rust
|
||||
uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master
|
||||
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
||||
with:
|
||||
toolchain: stable
|
||||
targets: aarch64-linux-android
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
echo "apk_name=$OUT" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload APK as workflow artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: ${{ steps.sign.outputs.apk_name }}
|
||||
path: ${{ steps.sign.outputs.apk_path }}
|
||||
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version-file: webapp/.node-version
|
||||
# No dependency cache in release builds: a poisoned cache could end
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Set up Rust
|
||||
uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master
|
||||
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
|
||||
with:
|
||||
toolchain: stable
|
||||
targets: ${{ matrix.tauri_target }}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Ingest queued links from scripts/temp_link.json (written by the browser
|
||||
# extension, and later the admin Worker) plus an optional dispatched URL.
|
||||
# Ingest queued links from scripts/temp_link.json (written by the admin
|
||||
# Worker and its cron) plus an optional dispatched URL.
|
||||
#
|
||||
# Runs right away when temp_link.json changes on main (pushes made by GitHub
|
||||
# Apps trigger it; this workflow's own GITHUB_TOKEN push does not, so no loop),
|
||||
@@ -34,7 +34,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: '1'
|
||||
PYTHONUTF8: '1'
|
||||
|
||||
jobs:
|
||||
ingest:
|
||||
@@ -51,7 +50,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version-file: .python-version
|
||||
python-version-file: .github/python-version
|
||||
cache: pip
|
||||
cache-dependency-path: requirements.txt
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version-file: webapp/.node-version
|
||||
cache: npm
|
||||
@@ -39,6 +39,12 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
# The web build never runs `tauri build`, which refuses mismatched
|
||||
# crate / npm versions; without this a Dependabot bump on one side
|
||||
# surfaces only when the release workflows run on a tag.
|
||||
- name: Tauri crates match their npm packages
|
||||
run: npm run check:tauri
|
||||
|
||||
- name: Worker types are up to date
|
||||
run: npm run cf:types -- --check
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
3.14
|
||||
+5
-1
@@ -3,12 +3,16 @@
|
||||
All notable changes to this project, newest first. The notes of each release live in
|
||||
[`changelog/`](changelog/), one file per minor version: `changelog/<major>.x/ver<major>.<minor>.x.md`.
|
||||
|
||||
**Latest release: [4.1.1](changelog/4.x/ver4.1.x.md#411---2026-09-29-new-contact-addresses) (2026-09-29) — New contact addresses.**
|
||||
**Latest release: [4.4.1](changelog/4.x/ver4.4.x.md#441---2026-10-16-extension-and-bot-retired) (2026-10-16) — Extension and bot retired.**
|
||||
|
||||
## 4.x
|
||||
|
||||
| Version | Date | Highlights |
|
||||
|---|---|---|
|
||||
| [4.4.1](changelog/4.x/ver4.4.x.md#441---2026-10-16-extension-and-bot-retired) | 2026-10-16 | Extension and bot retired (archived repos), ingest API removed |
|
||||
| [4.4.0](changelog/4.x/ver4.4.x.md#440---2026-10-13-admin-automation-bulk-actions-smoother-charts) | 2026-10-13 | Admin automation, bulk actions, smoother charts |
|
||||
| [4.3.0](changelog/4.x/ver4.3.x.md#430---2026-10-10-ai-disclosure-accessibility-refreshed-catalog) | 2026-10-10 | AI disclosure, accessibility, refreshed catalog |
|
||||
| [4.2.0](changelog/4.x/ver4.2.x.md#420---2026-10-03-compact-catalog-data-themed-scrollbars) | 2026-10-03 | Compact catalog data, themed scrollbars |
|
||||
| [4.1.1](changelog/4.x/ver4.1.x.md#411---2026-09-29-new-contact-addresses) | 2026-09-29 | New contact addresses (@poli0981.dev) |
|
||||
| [4.1.0](changelog/4.x/ver4.1.x.md#410---2026-09-24-verification-check-readable-charts-changelog-folder) | 2026-09-24 | Verification check, readable charts, changelog folder |
|
||||
| <a name="400---2026-09-18-freeitchgameswin-on-cloudflare-read-only-site-review-queue-new-design"></a>[4.0.0](changelog/4.x/ver4.0.x.md#400---2026-09-18-freeitchgameswin-on-cloudflare-read-only-site-review-queue-new-design) | 2026-09-18 | freeitchgames.win on Cloudflare: read-only site, review queue, new design |
|
||||
|
||||
@@ -8,18 +8,18 @@ A curated, auto-updating catalog of free itch.io games, published at **https://f
|
||||
everyone; the old github.io URL only redirects). Three layers share one JSON catalog:
|
||||
|
||||
1. **Python pipeline** — the data layer (GitHub Actions).
|
||||
- `data_game/game_info_NNN.json` — chunked records, max 500 per file, 2,811 games on 2026-09-24. Serialization is fixed: `json.dumps(..., ensure_ascii=False, indent=4)`, no trailing newline (`json_io.dumps`) — the Worker port (`webapp/worker/catalog.ts`) emits identical bytes (`tests/fixtures/golden/`).
|
||||
- `data_game/game_info_NNN.json` — chunked records, max 500 per file, 3,251 games on 2026-10-10. Serialization is fixed: `json.dumps(..., ensure_ascii=False, indent=4)`, no trailing newline (`json_io.dumps`) — the Worker port (`webapp/worker/catalog.ts`) emits identical bytes (`tests/fixtures/golden/`).
|
||||
- `data_game/index.json` — chunk manifest (total + per-file counts). `data_game/count_history.json` — date-keyed `{date,total}` series ("games over time" chart). `scripts/deleted_games.json` — removal log (public "Removed" page). `scripts/temp_link.json` — ingest queue.
|
||||
- `scripts/` — scanners never write the catalog: `update_info.py` (queued URLs) and `refresh.py` (rotating 1/7-per-day check; `--full` re-scrape) emit a URL-keyed **patch** (`patch.py`); `apply_patch.py` applies it to the latest `main` and runs `validate.py`; `bash/commit_push.sh` does fetch → reset → apply → commit → push with retries. Shared: `scraper.py` (session, `Pacer` + 429 back-off, parsing), `data_store.py` (chunks stay in place, rebalance only past thresholds, write-if-changed), `json_io.py`, `canonical.py` (canonical itch.io URL; vectors in `tests/fixtures/url_vectors.json`, shared with the Worker). Pipeline bookkeeping lives in `scripts/state/` (not published). Tests: `pytest` (network blocked). Lint: `ruff check scripts tests webapp/scripts`, `ruff format`, `vulture` (config in root `pyproject.toml`). Local env: `python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt` (Windows: run with `PYTHONUTF8=1`).
|
||||
2. **Cloudflare Worker + static assets** — `webapp/worker/` + `webapp/wrangler.jsonc`, deployed by **Cloudflare Workers Builds** from `main` (root dir `webapp`, build `npm run build`, deploy `npx wrangler deploy`; no deploy workflow). The catalog JSON is bundled into the site at build time (`vite-plugins/catalog-data.ts` → `dist/data/*`, plus `data/urls.json` and `sitemap.xml`), so every data commit rebuilds the site.
|
||||
3. **Web app + Tauri apps** — `webapp/src/` (React 19.3 + TypeScript 6.0 + Vite 8/rolldown + Tailwind + shadcn/ui + React Router 8), and the **Tauri 2 desktop + Android** read-only viewers in `webapp/src-tauri/` (same code; they fetch `https://freeitchgames.win/data` and load covers straight from img.itch.zone). Plus `webapp/admin/` — the maintainer-only admin app (second Vite entry, served at `/admin/`).
|
||||
- `scripts/` — scanners never write the catalog: `update_info.py` (queued URLs) and `refresh.py` (rotating 1/7-per-day check; `--full` re-scrape) emit a URL-keyed **patch** (`patch.py`); `apply_patch.py` applies it to the latest `main` and runs `validate.py`; `bash/commit_push.sh` does fetch → reset → apply → commit → push with retries. Shared: `scraper.py` (session, `Pacer` + 429 back-off, parsing: the info panel incl. itch.io's 2025–26 rows — `AI Disclosure` ("AI Assisted", then what for) / `Content` linking `/games/tag-no-ai` → `ai_disclosure` + `ai_content`, `Accessibility` → `accessibility`; `description` = the creator's tagline (`og:description`), else the first sentence), `data_store.py` (chunks stay in place, rebalance only past thresholds, write-if-changed), `json_io.py`, `canonical.py` (canonical itch.io URL; vectors in `tests/fixtures/url_vectors.json`, shared with the Worker). Pipeline bookkeeping lives in `scripts/state/` (not published), including `ingest_outcomes.json` (how each queued link ended — added / duplicate / paid / dead / private / deleted / gave_up; patch key `ingest_outcomes`; the last 90 days, ≤ 5,000) that the admin's tracking and the cron's reconcile read. Tests: `pytest` (network blocked). Lint: `ruff check scripts tests webapp/scripts`, `ruff format`, `vulture` (config in root `pyproject.toml`). Local env: Python 3.15 (`.github/python-version`, gotcha 39; ruff `target-version = "py315"`), `python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt` (3.15 is in UTF-8 mode by default — no `PYTHONUTF8` needed, the workflows no longer set it).
|
||||
2. **Cloudflare Worker + static assets** — `webapp/worker/` + `webapp/wrangler.jsonc`, deployed by **Cloudflare Workers Builds** from `main` (root dir `webapp`, build `npm run build`, deploy `npx wrangler deploy`; no deploy workflow). The catalog is bundled into the site at build time (`vite-plugins/catalog-data.ts` → `dist/data/pack/` compact packs + the legacy readable JSON in `dist/data/`, plus `data/urls.json` and `sitemap.xml`), so every data commit rebuilds the site.
|
||||
3. **Web app + Tauri apps** — `webapp/src/` (React 19.3 + TypeScript 7.0 + Vite 8/rolldown + Tailwind + shadcn/ui + React Router 8), and the **Tauri 2 desktop + Android** read-only viewers in `webapp/src-tauri/` (same code; they fetch `https://freeitchgames.win/data/pack` and load covers straight from img.itch.zone). Plus `webapp/admin/` — the maintainer-only admin app (second Vite entry, served at `/admin/`).
|
||||
|
||||
Adding games: the public **Suggest** page (`/suggest`, Turnstile), the maintainer's browser extension (`POST /api/ingest`, Access service token; repo `poli0981/itch-f2p-extension`), and RSS discovery (Worker cron) all feed a **D1 review queue**; approving in `/admin` commits the URLs to `scripts/temp_link.json` as a GitHub App, which triggers `update.yml`.
|
||||
Adding games: the public **Suggest** page (`/suggest`, Turnstile) and RSS discovery (Worker cron) feed a **D1 review queue** (the browser extension's `POST /api/ingest` was removed in 4.4.1; the extension `poli0981/itch-f2p-extension` and the Telegram bot `poli0981/telegram-scraper-bot` are archived repos); approving in `/admin` commits the URLs to `scripts/temp_link.json` as a GitHub App, which triggers `update.yml`. Since 4.4 the cron also approves by rule (`/admin` → Settings, on by default: unflagged `rss` / `form` items nobody rejected within 24 h, only those found after the switch-on baseline `since`, ≤ 100 per run) and queues the new free games of **followed itch.io collections** directly (removed or flagged ones go to the queue) — the policies say so (ToS §5, Privacy §2.4, Disclaimer §4). The Add page takes any text, a file or a collection, **Check**s it first (dry run), sends batches of 200 and follows each game's ingest (`/api/admin/track`).
|
||||
|
||||
## How to work in webapp/
|
||||
|
||||
```sh
|
||||
cd webapp
|
||||
cd webapp # Node 26 (.node-version; engines >=26)
|
||||
npm ci
|
||||
npm run dev # Vite on http://localhost:5173 (proxies /api and /img to wrangler dev on :8787)
|
||||
npx wrangler dev # the Worker + built assets (run `npm run build` first); .dev.vars holds local secrets
|
||||
@@ -39,44 +39,46 @@ Local admin without Access: put `DEV_ADMIN_EMAIL=you@example.com` in `webapp/.de
|
||||
|
||||
## Worker (`webapp/worker/`)
|
||||
|
||||
- `index.ts` routes: `/img/*` (img.ts, behind the gate), `/api/health`, `/api/verify` (gate.ts), `/api/suggest` (suggest.ts), `/api/ingest` (ingest.ts), `/api/admin/*` + `/admin*` (admin.ts), everything else that misses an asset → `spa.ts`. `run_worker_first` also lists `/` (otherwise served straight from `dist/index.html`) so the gate sees it. `scheduled()` (cron `23 */4 * * *`): `discover.ts` (one itch.io RSS feed per run, 6 h back-off on 429) → `QueueStore.reconcile` (queued → ingested / failed after 3 days) → `purge` (idempotency 7 d, Suggest notes 180 d).
|
||||
- `img.ts`: `/img/<160|640>/<path on img.itch.zone>` — allow-listed against `data/urls.json`, edge cache → R2 `THUMBS` (`w{w}/<sha256>.webp`) → `cf.image` transform (stored once) → fallback streams the original with a 1-day cache (quota error 9422). Upstream errors → controlled 502.
|
||||
- `spa.ts`: `not_found_handling` is `"none"`, so misses reach the Worker: `/assets/*`, `/data/*` and file-like paths get a real 404 (`no-store`); app routes get the shell with 200; unknown paths and game slugs not in the catalog get the shell with **404**; with the gate on, a visitor without a pass gets the verification page instead (checked before the shell is fetched, so never a 304). Keep its `ROUTES` in sync with `src/App.tsx` and `src/hooks/useSeoHead.ts`.
|
||||
- `index.ts` routes: `/img/*` (img.ts, behind the gate), `/api/health`, `/api/verify` (gate.ts), `/api/suggest` (suggest.ts), `/api/admin/*` + `/admin*` (admin.ts), the readable `/data/*.json` (legacy-data.ts), everything else that misses an asset → `spa.ts`. `run_worker_first` also lists `/` (otherwise served straight from `dist/index.html`) so the gate sees it, and the five legacy catalog files one by one (never a `!` pattern — gotcha 26). `scheduled()` (cron `23 */4 * * *`), four steps each in its own try/catch: `discover.ts` (one itch.io RSS feed per run, 6 h back-off on 429) → `QueueStore.reconcile` (queued → ingested / failed with the pipeline's outcome, read at head through the GitHub App; else in the catalog → ingested, > 3 days → failed `timeout`) → `automation.ts` (auto-approve + followed collections, ≤ 30 itch.io pages a run, ONE `[auto]` commit, D1 written only after it lands; the run's summary in settings `automation_last_run`) → `purge` (Suggest notes 180 d). The account is on **Workers Paid** (30 s CPU): admin catalog edits parse and rewrite ~3.8 MB in the Worker (≈ 25–30 ms for 100 removals).
|
||||
- `img.ts`: `/img/<160|640>/<path on img.itch.zone>` — allow-listed against `data/urls.json`, edge cache → R2 `THUMBS` (`w{w}/<sha256>.webp`) → `cf.image` transform (stored once) → fallback streams the original with a 1-day cache (quota error 9422). The fallback original and a missing upstream image are also kept in `caches.default` for a day (GET only); widths are the exact strings `160` / `640` (a `Map`, no `Number()` aliases); background puts log failures. Upstream errors → controlled 502.
|
||||
- `spa.ts`: `not_found_handling` is `"none"`, so misses reach the Worker: `/assets/*`, `/data/*` (a pack from an older deploy included — `no-store`, CORS) and file-like paths get a real 404 (`no-store`); app routes get the shell with 200; unknown paths and game slugs not in the catalog get the shell with **404**; with the gate on, a visitor without a pass gets the verification page instead (checked before the shell is fetched, so never a 304). Keep its `ROUTES` in sync with `src/App.tsx` and `src/hooks/useSeoHead.ts`.
|
||||
- `gate.ts` + `gate-page.ts`: the **verification gate** (web only). App pages and `/img` need a pass: the `__Host-fig_gate` cookie (HMAC of its issue time, HttpOnly, `BOT_GATE_TTL_HOURS`, default 48) set by `POST /api/verify` after a Turnstile check (action `gate`, `RL_GATE`, same-origin); `GET /api/verify` reports `{enabled, valid, via, ttl}`. The page is self-contained (inline CSS/JS with a per-response CSP nonce, EN/VI, the site's og tags, 200 + noindex). Verified bots pass with `x-fig-known-bot: <GATE_BOT_TOKEN>` from a zone Transform Rule; clients that only claim to be a crawler get 503. `/assets`, `/data`, `/api/*`, `/admin*` and the other static files stay open. `turnstile.ts`: siteverify shared with Suggest (hostname + action checked; the test secrets skip both).
|
||||
- `access.ts`: verifies `Cf-Access-Jwt-Assertion` with jose (issuer + AUD). Maintainer = email in secret `ADMIN_EMAILS` + same-origin writes; ingest = service-token `common_name`.
|
||||
- `admin.ts`: API (queue list / approve / reject / add / edit the 3 fields / delete with reason + R2 cleanup / restore) and the `/admin` shell with its own CSP. The maintainer's email goes only to the D1 audit log, never into a commit (the repo is public).
|
||||
- `access.ts`: verifies `Cf-Access-Jwt-Assertion` with jose (issuer + AUD). Maintainer = email in secret `ADMIN_EMAILS` + same-origin writes (the only identity: the service-token ingest is gone since 4.4.1).
|
||||
- `admin.ts`: API (queue list / approve / reject / check (dry run) / add / track / edit one or ≤ 100 (`/games/bulk`) / remove one or ≤ 100 (`/games/remove`; after the commit the audit row and the R2 cleanup are best effort → `warnings`, never a 500) / restore / activity / settings / itch.io collections) and the `/admin` shell with its own CSP (theme-script hash included). Request limits shared with the admin app: `limits.ts`. `collections.ts` reads public itch.io collections (gotcha 36); `automation.ts` holds the cron's rules (`planCollectionItems`: only URLs D1 has never seen and without a flag are queued directly — no re-add loop for games the ingest dropped). The maintainer's email goes only to the D1 audit log, never into a commit (the repo is public).
|
||||
- `repo.ts` + `github.ts` + `catalog.ts`: writes as a **GitHub App** (installation token, cached per isolate) with GraphQL `createCommitOnBranch` + `expectedHeadOid` (verified commits; `commitWithRetry` rebuilds from the new head on a race). Reads use the raw media type (no base64/UTF-8 mojibake). `catalog.ts` ports `data_store.py` byte-for-byte (golden fixtures).
|
||||
- `queue.ts`: URL classification (catalog → blocklist → pending → previously deleted) + the D1 store (`worker/migrations/`, additive only — never edit an applied migration). `data.ts`: catalog facts read from the site's own `/data` via `env.ASSETS`, cached per isolate (loaded values only — see gotchas).
|
||||
- `legacy-data.ts`: the readable catalog JSON (`index.json`, `game_info_NNN.json`, `count_history.json`, `deleted_games.json`, `urls.json`) goes only to apps 4.0–4.1.1 — `Origin` `tauri://localhost` / `http(s)://tauri.localhost` → `env.ASSETS` passthrough + `Vary: Origin`; anyone else → **410** JSON with the repository links (`no-store`, CORS, a sampled `console.info` of the Origin). A nudge, not a security boundary: the data is public in the repo.
|
||||
- `queue.ts`: URL classification (catalog → blocklist → pending → previously deleted) + the D1 store (`worker/migrations/`, additive only — never edit an applied migration; `0002` added `candidates.outcome`, `settings` and `watched_collections`; the `idempotency` table and `'ext'` in the `candidates.source` CHECK are unused leftovers of the ingest API removed in 4.4.1; URL lists bind in `IN` slices of 90, gotcha 37). `data.ts`: catalog facts read from the site's own `/data` via `env.ASSETS`, cached per isolate (loaded values only — see gotchas).
|
||||
- Configuration: non-secret settings are `vars` in `wrangler.jsonc` (empty until setup); secrets (`GH_APP_PRIVATE_KEY` — the PEM GitHub downloads (PKCS#1) or PKCS#8; `github.ts` converts, `ADMIN_EMAILS`, `TURNSTILE_SECRET`, `GATE_SECRET`, `GATE_BOT_TOKEN`) via `wrangler secret put`, typed in `worker/env.ts` as optional. A feature whose settings are missing answers **503** (the gate just stays off); the public site needs none of them. `env.staging` = Worker `free-games-itchio-list-staging` on `staging.freeitchgames.win` (no cron, own R2/D1/rate-limit namespaces).
|
||||
|
||||
## Web app structure (`webapp/src/`)
|
||||
|
||||
- `routes/*.tsx` — one file per route (`/`, `/games`, `/games/:slug`, `/charts`, `/removed` (`/deleted` redirects), `/suggest`, `/settings`, `/about`, `/errors/:code`); `App.tsx` wires them; `components/app-router.tsx` picks **BrowserRouter on the web, HashRouter in Tauri**. `lib/legacy.ts` rewrites v3 `#/x` links to `/x` (crafted hashes go home) and deletes v3 PAT/GPG storage keys.
|
||||
- Data: `lib/config.ts` (`DATA_BASE` = `/data` on the web, `https://freeitchgames.win/data` in Tauri), `lib/data/*`, `hooks/useGames.ts`. react-query cache persisted to IndexedDB (`webapp.query-cache`, 7 days); `CACHE_BUSTER = ${APP.version}:data-v1`. **`gcTime` must stay >= persist `maxAge`.**
|
||||
- Data: `lib/config.ts` (`PACK_BASE` = `/data/pack` on the web, `https://freeitchgames.win/data/pack` in Tauri). **Packs** (since 4.2): `lib/data/pack-format.ts` (pure, shared with the build plugin: records by column, dictionaries for repeated strings and every string list; `manifest-v1.json` names per chunk a `main` pack — every field but `description` — and a `desc` pack, plus `deleted` / `history`; files are `<sha256 prefix>.bin`, deflate-raw, served `immutable`). `lib/data/catalog.ts` loads them (`['db','all']` →`{ games, index: {total_games, last_updated}, chunks }`; the manifest revalidated per load and shared 10 s; a main-pack 404 → manifest `cache: 'reload'` + one retry; an unchanged manifest returns the previous result). `lib/data/compression.ts`: `DecompressionStream('deflate-raw')`, else lazy `fflate`. `Game` has **no `description`**: `hooks/useDescriptions.ts` loads `['desc', <file>]` on demand (game page: its chunk; Games search: all, `filterGames(…, descriptions)`).
|
||||
- Cache: react-query persisted to IndexedDB (`webapp.query-cache`, 7 days) by `lib/data/persister.ts` — byte 0 = format (1 deflate-raw, 0 plain), then the JSON; a legacy string is dropped; writes throttled 1 s and skipped when no data object changed. Persisted: `db`, `deleted`, `count-history` and only the `desc` files the current catalog names (`shouldPersistQuery`); stale `desc` queries are removed after a `db` success. `CACHE_BUSTER = ${APP.version}:data-v2`. **`gcTime` must stay >= persist `maxAge`.**
|
||||
- Layout: `components/site/*` (sticky top nav + global search with the `/` shortcut + display menu + phone sheet menu; footer) — the window scrolls, not `<main>`. `/` is the Welcome page (`routes/welcome.tsx`).
|
||||
- `/games`: search, filters, sort and page live in the URL — `lib/game-filters.ts` (`parseQuery` / `toParams` drop defaults; params `q genre platform status tag lang input engine rating browser sort page`; unit-tested) — rendered by `components/games/*` (filter chips with facet counts that ignore their own filter, a desktop ARIA table / phone list chosen by `useIsMobile`, 100 per page; the detail page's breadcrumb links back to the same list via router state). Counts show the whole catalog ("2,729 of 2,811") because 18+ games are hidden by default.
|
||||
- `components/charts/` (16 charts, 4 lazy tabs, **no barrel `index.ts`**; every Recharts tooltip/cursor comes from `chart-theme.ts` + `chart-tooltip.tsx`), `components/game-thumb.tsx` (`lib/thumbnail.ts` → `/img/160|640/...` on the web, the original URL in Tauri), `components/legal-gate.tsx` (a non-dismissable Radix dialog **over** the page, so deep links land where they pointed; links flagged `inGate` in `lib/about.ts`; re-prompts when `LEGAL_VERSION` in `stores/prefs.ts` changes; no "Leave" button in Tauri).
|
||||
- Styling: **Tailwind v4** via `@tailwindcss/vite` (no PostCSS, no `tailwind.config`); tokens are hex CSS variables in `src/index.css` (`:root` / `.dark`, `@theme inline`; `--brand-soft`, `--thumb`, `--success` besides the shadcn set) and must match the approved mockup. Self-hosted Geist / Geist Mono (`@fontsource-variable/*`, CSP `font-src 'self'`). Numbers and dates go through `lib/format.ts` (`useFormat`: en-GB / vi-VN). `webapp/scripts/gen_assets.py` redraws favicon, PWA icons and `og.png` from the same tokens and the Geist fonts in `node_modules`.
|
||||
- `/games`: search, filters, sort and page live in the URL — `lib/game-filters.ts` (`parseQuery` / `toParams` drop defaults; params `q genre platform status tag lang input engine ai access rating browser sort page`; `ai` treats "N/A" / a missing field as its own "Not disclosed" value and lists its options in a fixed order (`aiRank`); unit-tested) — rendered by `components/games/*` (filter chips with facet counts that ignore their own filter, a desktop ARIA table / phone list chosen by `useIsMobile`, 100 per page; the detail page's breadcrumb links back to the same list via router state). Counts show the whole catalog ("2,729 of 2,811") because 18+ games are hidden by default.
|
||||
- `components/charts/` (19 charts, 4 lazy tabs loaded with `lib/lazy-preload.ts` — preloaded when idle or on hover/focus, rendered without a Suspense fallback once loaded — and kept mounted once shown (gotcha 33; skeletons in `chart-skeletons.tsx` match the cards), every series spreading `SERIES_ANIMATION` (300 ms, no delay; a test checks), **no barrel `index.ts`**; every Recharts tooltip/cursor comes from `chart-theme.ts` + `chart-tooltip.tsx`, every pie legend from `LEGEND_BASE` there; below `md` the charts get 12 px legends / 11 px ticks from a media block in `index.css`, the tab bar is one row of equal tabs, and Languages / Avg session switch to horizontal bars via `useIsMobile`), `components/game-thumb.tsx` (`lib/thumbnail.ts` → `/img/160|640/...` on the web, the original URL in Tauri), `components/legal-gate.tsx` (a non-dismissable Radix dialog **over** the page, so deep links land where they pointed; links flagged `inGate` in `lib/about.ts`; re-prompts when `LEGAL_VERSION` in `stores/prefs.ts` changes; no "Leave" button in Tauri).
|
||||
- Styling: **Tailwind v4** via `@tailwindcss/vite` (no PostCSS, no `tailwind.config`); tokens are hex CSS variables in `src/index.css` (`:root` / `.dark`, `@theme inline`; `--brand-soft`, `--thumb`, `--success`, `--scrollbar-thumb(-hover/-active)` besides the shadcn set) and must match the approved mockup. Themed scrollbars (`@layer base`, only under `(hover: hover) and (pointer: fine)`): `::-webkit-scrollbar*` for Chromium/Safari, `scrollbar-color` / `scrollbar-width: thin` for Firefox only (gotcha 28); "Hide scrollbars" = unlayered `html[data-scrollbars='hidden']` rules (`hooks/useScrollbarsEffect.ts`). On mouse devices `<html>` reserves the scrollbar gutter, released while a dialog locks scrolling (gotcha 35). The theme: `stores/theme.ts` (synced across tabs, site ↔ admin), `hooks/useThemeEffect.ts` (`.dark` + the `theme-color` metas), an inline pre-paint script in both HTML entries (gotcha 34), and the display menu's button shows `components/theme-icon.ts`'s icon for the theme in use. Self-hosted Geist / Geist Mono (`@fontsource-variable/*`, CSP `font-src 'self'`). Numbers and dates go through `lib/format.ts` (`useFormat`: en-GB / vi-VN). `webapp/scripts/gen_assets.py` redraws favicon, PWA icons and `og.png` from the same tokens and the Geist fonts in `node_modules`.
|
||||
- `lib/gate.ts` (web only): when a tab comes back into view or a cover fails and the verification pass may have run out (`webapp.gate` hint), it asks `GET /api/verify` and reloads into the verification page; `lib/reload.ts` is the shared at-most-every-30-s reload guard.
|
||||
- `hooks/useSeoHead.ts` sets canonical + robots per route (index.html has no static canonical/og:url on purpose).
|
||||
- `lib/i18n/` — `en.ts` is the source of truth; `vi.ts` is `Record<MessageKey, string>` and must never be static-imported (lazy chunk).
|
||||
- `stores/` — Zustand: `prefs` (`webapp.prefs`: language, density, legal acceptance…), `theme` (`webapp.theme`).
|
||||
- `stores/` — Zustand: `prefs` (`webapp.prefs`: language, density, `hideScrollbars`, legal acceptance…), `theme` (`webapp.theme`).
|
||||
- `lib/about.ts` — keep `THIRD_PARTY` up to date when adding npm deps; `LEGAL_LINKS`; `APP.version`.
|
||||
- Code split (`vite.config.ts` rolldown `codeSplitting.groups`): `vendor-react` (priority 100 — must claim React first), `vendor-query`, `vendor-ui`. There is deliberately **no vendor-charts group** (it pulled React/clsx in and put Recharts on the first load); Recharts stays in the lazy chart chunks. Initial load ≈ 173 KB gzip (JS + CSS) plus ~51 KB of Geist woff2. Web build `target` is Safari/iOS 15.4 (Radix class static blocks otherwise break iOS 15); Tauri uses the default.
|
||||
- Code split (`vite.config.ts` rolldown `codeSplitting.groups`): `vendor-react` (priority 100 — must claim React first), `vendor-query`, `vendor-ui`. There is deliberately **no vendor-charts group** (it pulled React/clsx in and put Recharts on the first load); Recharts stays in the lazy chart chunks. Initial load ≈ 179 KB gzip (JS + CSS, gzip -9; 4.2 added ~4.6 KB for the pack loader, persister and `useQueries`) plus ~51 KB of Geist woff2; catalog data on a first visit ≈ 264 KB (main packs + log + history), descriptions ≈ 170 KB on demand. `fflate` is a lazy chunk, never preloaded. Web build `target` is Safari/iOS 15.4 (Radix class static blocks otherwise break iOS 15); Tauri uses the default.
|
||||
|
||||
## Editable vs read-only fields
|
||||
|
||||
Each game has **20** fields (plus optional `added_at` / `updated_at`, appended by the pipeline — schema changes are additive only, installed v3 apps may still read the raw JSON). Only **3** are maintainer-editable: `safe_virus` (`?|Yes|No|Caution`), `notes`, `nsfw` (`Yes|No`) — via `/admin`. The other 17 are scraper output; no scrape (not even `--full`) overwrites the 3 editable fields.
|
||||
Each game has **20** fields (plus optional `added_at` / `updated_at`, appended by the pipeline, and since 4.3 the optional scraper fields `ai_disclosure` (`No AI|AI Assisted|N/A`) / `ai_content` / `accessibility`, which the daily refresh adds to older records — schema changes are additive only, installed v3 apps may still read the raw JSON). Only **3** are maintainer-editable: `safe_virus` (`?|Yes|No|Caution`), `notes`, `nsfw` (`Yes|No`) — via `/admin`. The other 17 are scraper output; no scrape (not even `--full`) overwrites the 3 editable fields.
|
||||
|
||||
## GitHub Actions
|
||||
|
||||
| Workflow | Trigger | Notes |
|
||||
|---|---|---|
|
||||
| `update.yml` | Push to `main` touching `scripts/temp_link.json` + daily 01:23 UTC + `workflow_dispatch` (`url`) | Ingest queue → patch → `commit_push.sh`. Workflow-level `concurrency: ingest` with **`queue: max`** (the default queue replaces a pending run, which would drop its dispatched `url`). A dispatched `url` is processed first and written to `temp_link.json` (`queue_add`) until it reaches a final outcome. Failed fetches count as an attempt only ≥ 6 h apart (`last_attempt`; 3 → given up), and a 429 counts as none. GitHub-App pushes (extension, admin Worker) trigger it; its own `GITHUB_TOKEN` push does not (no loop). Keep the filename. |
|
||||
| `refresh.yml` | Daily 02:47 UTC + `workflow_dispatch` (`budget`, `url`, `allow_mass_removal`) + `workflow_call` | scan (read-only, 55-min step timeout, patch artifact uploaded `if: always()`) → apply (`!cancelled()`: a failed / timed-out scan's checkpoint is applied, a cancelled run's is not) → notify. Workflow-level `concurrency: refresh-pipeline` + `queue: max` runs one whole scan → apply at a time (also for force-update's call; the group name must differ from the caller's or GitHub cancels it as a deadlock). Removal needs the same 404/410 or paid strike seen again ≥ 20 h after the first sighting (`strike_at`). Checkpoints never carry removals. **Mass-change guard:** a run whose removals or new strikes exceed max(10, 10 %) of the games it checked withholds all its removals and marks them — and, on a strike spike, all its strikes — `withheld_at`; later runs keep withholding those (they take at most half a run's budget) until a healthy check clears them, and alert (`mass_change`) — that pattern means itch.io markup changed; dispatch with `allow_mass_removal` only after checking on itch.io. `apply_patch.py` re-checks the same per-run limit and max(25, 5 %) of the catalog unless `ALLOW_MASS_REMOVAL=1`. |
|
||||
| `force_update.yml` | `workflow_dispatch` ONLY (`url`, `budget`, `allow_mass_removal`) | `queue: max`. Calls `refresh.yml` with `full: true`: re-scrapes every scraper field for one URL or the next batch (rotates on `full` timestamps). Always preserves `safe_virus` / `notes` / `nsfw`. Canonical tool for repairing mojibake. |
|
||||
| `update.yml` | Push to `main` touching `scripts/temp_link.json` + daily 01:23 UTC + `workflow_dispatch` (`url`) | Ingest queue → patch → `commit_push.sh`. Workflow-level `concurrency: ingest` with **`queue: max`** (the default queue replaces a pending run, which would drop its dispatched `url`). A dispatched `url` is processed first and written to `temp_link.json` (`queue_add`) until it reaches a final outcome. Failed fetches count as an attempt only ≥ 6 h apart (`last_attempt`; 3 → given up), and a 429 counts as none. GitHub-App pushes (the admin Worker and its cron) trigger it; its own `GITHUB_TOKEN` push does not (no loop). Keep the filename. |
|
||||
| `refresh.yml` | Daily 02:47 UTC + `workflow_dispatch` (`budget`, `url`, `allow_mass_removal`, `full`) | scan (read-only, 55-min step timeout, checks out the branch as it is when the job starts — not the queued SHA — so runs waiting in the queue see the previous run's state; patch artifact uploaded `if: always()`) → apply (`!cancelled()`: a failed / timed-out scan's checkpoint is applied, a cancelled run's is not) → notify. Workflow-level `concurrency: refresh-pipeline` + `queue: max` runs one whole scan → apply at a time (scheduled, manual and force-update runs alike). Removal needs the same 404/410, paid or `private` (itch.io's password form, `is_password_protected`; ingest drops such links as final) strike seen again ≥ 20 h after the first sighting (`strike_at`). Checkpoints never carry removals. **Mass-change guard:** a run whose removals or new strikes exceed max(10, 10 %) of the games it checked withholds all its removals and marks them — and, on a strike spike, all its strikes — `withheld_at`; later runs keep withholding those (they take at most half a run's budget) until a healthy check clears them, and alert (`mass_change`) — that pattern means itch.io markup changed; dispatch with `allow_mass_removal` only after checking on itch.io. `apply_patch.py` re-checks the same per-run limit and max(25, 5 %) of the catalog unless `ALLOW_MASS_REMOVAL=1`. |
|
||||
| `force_update.yml` | `workflow_dispatch` ONLY (`url`, `budget`, `allow_mass_removal`) | Dispatches `refresh.yml` with `full: true` (`actions: write`; it used to call it through `uses:`, which failed at startup with no annotation): re-scrapes every scraper field for one URL or the next batch (rotates on `full` timestamps). Dispatch it N times in a row to cover N consecutive batches (FIFO, each from the latest main). Always preserves `safe_virus` / `notes` / `nsfw`. Canonical tool for repairing mojibake. |
|
||||
| `notify-failure.yml` | `workflow_call` | Discord (`DISCORD_CI_WEBHOOK`) on failure / **cancelled** (job-level timeouts end as cancelled) / rate-limited / mass change / failed Cloudflare build. |
|
||||
| `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` / `npm run cf:types -- --check` + lint + knip + vitest + build + `wrangler deploy --dry-run --env=""`. |
|
||||
| `python-ci.yml` / `webapp-ci.yml` | PRs (+ pushes for Python) | ruff + format + vulture + pytest + `validate.py` / `npm run check:tauri` (gotcha 31) + `npm run cf:types -- --check` + lint + knip + vitest + build + `wrangler deploy --dry-run --env=""`. |
|
||||
| `deploy-status.yml` | Hourly + manual | Not a deploy. `bash/cf_build_status.sh` walks main's first-parent history to the latest finished `Workers Builds: free-games-itchio-list` check run (Cloudflare posts it only when a build ends; branch commits carry preview builds under the same name) and alerts once per failed build (cache key per check-run id, saved after the Discord post). Scheduled because GitHub never starts `check_run` workflows for `GITHUB_TOKEN` pushes (the data commits). |
|
||||
| `release_desktop.yml` | Tag `v*` push, or manual | `create-release` makes the single draft (id passed to tauri-action v1 `releaseId`); release builds use **no** npm/cargo caches (cache-poisoning). Win + macOS aarch64 + macOS x86_64 (cross-compile) + Linux (`ubuntu-22.04` for a low glibc floor). Uploads `.dmg` / `.app.tar.gz` / `.pkg` (macOS), `.msi` / `.exe` (Windows), `.deb` / `.rpm` / `.AppImage` (Linux). macOS `.pkg` `--identifier` reads `tauri.conf.json`. |
|
||||
| `release_android.yml` | Tag `v*` push, or manual | Tauri **Android** build → arm64-v8a APK, **post-signed** with `zipalign`+`apksigner` (gen/ stays gitignored). Tag → attaches to the same draft Release; manual dispatch → workflow artifact. Secrets: `ANDROID_KEYSTORE_BASE64` / `ANDROID_KEYSTORE_PASSWORD` / `ANDROID_KEY_ALIAS`. |
|
||||
@@ -96,7 +98,7 @@ git push origin vX.Y.Z
|
||||
gh release edit vX.Y.Z --draft=false
|
||||
```
|
||||
|
||||
Draft-first: **never** `gh release create` by hand before CI — tauri-action v1 refuses to upload into a published release, and announcements would go out before installers exist. Merge PRs into `main` with squash or a merge commit, never by fast-forward pushing a branch: its commits' preview-build check runs would then sit on main's own line and confuse `deploy-status.yml`. Tag from `main` only. Release notes go to `changelog/<major>.x/ver<major>.<minor>.x.md` plus a row in the `CHANGELOG.md` index (each row keeps the anchor of the old single-file heading as `<a name>`: published release notes link to them). Versions live in `webapp/src/lib/about.ts` (`APP.version`), `webapp/package.json`, `webapp/src-tauri/tauri.conf.json` and `Cargo.toml` (+ `Cargo.lock`) — keep them equal. GPG signing is wired through gpg-agent (`commit.gpgsign=true`, so `git tag -s` and `git commit` both sign; the pinentry must be answered — in unattended sessions commits fail rather than going unsigned). The historical key `03F965C2E2DB5C6B` was exposed in the browser extension's storage (Sept 2026) and is being replaced — use the maintainer's current key.
|
||||
Draft-first: **never** `gh release create` by hand before CI — tauri-action v1 refuses to upload into a published release, and announcements would go out before installers exist. Merge PRs into `main` with squash or a merge commit, never by fast-forward pushing a branch: its commits' preview-build check runs would then sit on main's own line and confuse `deploy-status.yml`. Tag from `main` only. Release notes go to `changelog/<major>.x/ver<major>.<minor>.x.md` plus a row in the `CHANGELOG.md` index (each row keeps the anchor of the old single-file heading as `<a name>`: published release notes link to them). Versions live in `webapp/src/lib/about.ts` (`APP.version`), `webapp/package.json`, `webapp/src-tauri/tauri.conf.json` and `Cargo.toml` (+ `Cargo.lock`) — keep them equal. GPG signing is wired through gpg-agent (`commit.gpgsign=true`, so `git tag -s` and `git commit` both sign; the pinentry must be answered — in unattended sessions commits fail rather than going unsigned). Signing key since 2026-10-10: `838884804847BE7AE5FCF0A96AD1AB0921B96825` (ed25519 certify-only primary, generated locally and never published before; signing subkey `ED0BFD3005662D5D`; `user.signingkey` = the primary fingerprint, global + repo). The old key `03F965C2E2DB5C6B` is **revoked** (compromised: it sat in the browser extension's storage, Sept 2026); tags up to v4.3.0 and older commits carry it. The owner's offline backup and revocation certificates live outside the repo.
|
||||
|
||||
## Licenses & policies
|
||||
|
||||
@@ -109,7 +111,7 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri
|
||||
3. **Tauri icons must exist before build.** `npm run tauri icon -- path/to/source.png`; the 1024×1024 source is gitignored, generated icons are committed.
|
||||
4. **Tauri doesn't ship `.pkg`.** Built with `pkgbuild` after tauri-action and uploaded via `gh release upload`; the step uses `if: ${{ !cancelled() && … }}` so a DMG flake doesn't drop the PKG while a cancel still stops it.
|
||||
5. **Vite v8 `base`** only accepts `'./'`, an absolute URL, or an empty string. Web uses `'/'` (BrowserRouter deep links), Tauri `'./'`.
|
||||
6. **TS 6** deprecates `baseUrl`. Use `paths` alone. Stay on TS 6.0.x until typescript-eslint supports newer.
|
||||
6. **TypeScript 7 has no JS API (expected in 7.1), and typescript-eslint needs one** (peer `typescript <6.1`). So in `webapp/package.json` `typescript` is an alias of `@typescript/typescript6` (the 6.0 API for typescript-eslint and `scripts/i18n-edit.mjs`; bin `tsc6`) and TS 7 is `@typescript/native` (`npm:typescript@^7`), whose `tsc` is what `npm run build` / `npx tsc -b` run — `npx tsc --version` must say 7.x. Drop the alias once typescript-eslint accepts TS 7; never let Dependabot move the alias to 7. `baseUrl` is gone since TS 6: use `paths` alone.
|
||||
7. **lucide-react v1+** removed brand icons (`Github`). Use `Library`, `Database`, etc.
|
||||
8. **`bundle.windows.nsis.license` does NOT exist in `@tauri-apps/cli` 2.11** — use top-level `bundle.licenseFile`, and it must be **RTF** (WiX `.msi` requires it; NSIS auto-detects). File: `webapp/src-tauri/installer/EULA.rtf`.
|
||||
9. **The bundle `identifier` must be alphanumeric** (`com.poli0981.freegamesitchio`): Android rejects `-`, Tauri's validator rejects `_`; one identifier serves every platform.
|
||||
@@ -120,7 +122,7 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri
|
||||
14. **With `run_worker_first` as an array, the asset layer's SPA fallback answers every miss before the Worker** — a stale hashed chunk got `index.html` + the year-long immutable cache. Hence `not_found_handling: "none"` + `spa.ts`.
|
||||
15. **Never share an in-flight promise across requests in a Worker** (cache only resolved values): a pending subrequest dies with the request that started it (client disconnect) and every later awaiter hangs.
|
||||
16. **itch.io cover URLs are signed per size**, so you can't rewrite `/original/` to a smaller variant — resizing goes through Image Transformations (`cf.image`) and R2.
|
||||
17. **GitHub Actions concurrency keeps at most one *pending* run per group by default** and cancels the older pending one — use `queue: max` where a queued run carries unique input. A called workflow must not request its caller's group (deadlock → cancelled).
|
||||
17. **GitHub Actions concurrency keeps at most one *pending* run per group by default** and cancels the older pending one — use `queue: max` where a queued run carries unique input. A called workflow must not request its caller's group (deadlock → cancelled). **A queued run checks out the SHA from when it was triggered** (`workflow_dispatch`: the branch head at dispatch time), so runs waiting behind each other read the same state unless the job checks out `ref: ${{ github.ref }}` (refresh.yml's scan does).
|
||||
18. **Pushes made with `GITHUB_TOKEN` trigger no workflows** (no ingest loop), and GitHub never starts `check_run` workflows for them (hence the scheduled `deploy-status.yml`). Workers Builds posts its check run only when a build ends.
|
||||
19. **`wrangler types` reads `.dev.vars`** — `npm run cf:types` points it at the empty `types.env` so the committed `worker-configuration.d.ts` never depends on local secrets. Don't use `secrets.required` until every environment has the secrets (deploys fail without them).
|
||||
20. **D1 migrations are applied by hand** (`npx wrangler d1 migrations apply freeitchgames --remote`), never in the build — and before deploying code that needs them. Production `freeitchgames` was created by command (APAC); its `database_id` is in `wrangler.jsonc` (not a secret). **R2 must be enabled on the account** (dashboard, payment method) before `wrangler r2 bucket create`; a deploy whose bucket is missing fails.
|
||||
@@ -128,7 +130,21 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri
|
||||
22. **`window.confirm` / `alert` / `prompt` do nothing in the macOS app** (wry's WebKit answers "Cancel"). Use in-app dialogs (`components/nsfw-dialog.tsx` + `requestNsfw()` in `lib/nsfw.ts`).
|
||||
23. **The design tokens are hex: write `var(--x)`, never `hsl(var(--x))`.** The invalid colour painted the Recharts hover band black and left tooltips transparent (fixed in 4.1.0).
|
||||
24. **Recharts 3 makes charts focusable** (`accessibilityLayer`, plus `tabindex` on inner `<g>`s): a mouse click focused the SVG, Chromium drew its black focus ring and the tooltip stuck on the first item. `ChartCard` keeps mouse clicks from moving focus; keyboard focus keeps the `--ring` outline.
|
||||
25. **The verification gate fails open, and its off switch is deleting `GATE_SECRET`** (`BOT_GATE` changed in the dashboard is undone by the next deploy). It also needs `GATE_BOT_TOKEN` + the zone Transform Rule — without them Google would get the challenge; a client that only claims to be a crawler gets a 503, never the noindex page. Don't gate `/data`: the apps and CC BY reuse read it.
|
||||
25. **The verification gate fails open, and its off switch is deleting `GATE_SECRET`** (`BOT_GATE` changed in the dashboard is undone by the next deploy). It also needs `GATE_BOT_TOKEN` + the zone Transform Rule — without them Google would get the challenge; a client that only claims to be a crawler gets a 503, never the noindex page. Don't gate `/data`: every app version reads it — `/data/pack/*` (4.2+), and the legacy JSON (4.0–4.1.1, Tauri origins only; browsers get 410). Keep those legacy files in `dist/`: the Worker also reads `urls.json` and `deleted_games.json` through `env.ASSETS`.
|
||||
26. **A negative `run_worker_first` pattern (`!/data/pack/*`) sends a missing file straight to the asset worker**, whose 404 still gets the `_headers` rules — for a pack, the year-long `immutable`, so a browser would cache the 404 of a name that can come back (rollback). List the Worker-first files positively; pack misses then reach `spa.ts` (`no-store`).
|
||||
27. **A `desc` pack matches its games by row, so only within one build.** Load the `desc` named in the same `db` result; on a 404 refetch `db` (`useDescriptions.ts`), never re-resolve a desc file against a newer manifest for games already on screen.
|
||||
28. **In Chromium 121+, `scrollbar-color` or `scrollbar-width` on an element switches its `::-webkit-scrollbar` styling off.** Keep the standard properties Firefox-only (`@supports not selector(::-webkit-scrollbar)`), except `scrollbar-width: none` for hiding.
|
||||
29. **Pack names are content hashes: the build must stay deterministic** (column and dictionary order, zlib level 9 — a Node/zlib upgrade only re-downloads once), **and no hash may reach the JS bundle** (data-only deploys keep JS byte-identical; the manifest is fetched, never imported). `_headers` joins a header set by two matching rules with a comma — give each path one `Cache-Control`.
|
||||
30. **itch.io shows the `Published` and `Updated` info rows only for a few weeks** after the event (every `Published` date in a 250-page sample of Oct 2026 was under four weeks old); an explicit `Release date` row stays. That is why `release_date` is N/A for most older games, and why a `--full` re-scrape keeps a known `release_date` / `thumbnail` instead of writing N/A (`KEEP_KNOWN_FIELDS` in `refresh.py`).
|
||||
31. **Tauri crates and their npm packages must share major.minor** (`tauri` ↔ `@tauri-apps/api`, `tauri-plugin-x` ↔ `@tauri-apps/plugin-x`): `tauri build` refuses to build otherwise, and only the release workflows run it — every v4.3.0 release build first failed on `tauri-plugin-opener` 2.6.0 vs `@tauri-apps/plugin-opener` 2.7.0 after Dependabot bumped the npm side alone (#119). Webapp CI's `npm run check:tauri` (`webapp/scripts/check-tauri-versions.mjs`, lock files only) now fails such a PR; Dependabot's cargo updates run weekly on Mondays like npm so both groups arrive together. Fix a one-sided bump with `cargo update -p <crate>` in `webapp/src-tauri/`.
|
||||
32. **Recharts hides category labels silently and styles legends its own way.** The default `interval="preserveEnd"` keeps a label only if its line box (font size × 1.5) plus `minTickGap` (5) fits its row, and the first one only if half of it fits above: the 15-row Genres chart showed 7 labels, without the two largest genres — category axes set `interval={0}`. A label wider than the axis wraps onto two lines, measured at the 16px body size unless `style` carries a size with a unit (`singleLineTick()` in `chart-theme.ts`). `<Legend>` sorts alphabetically (`itemSorter: 'value'`) and colours each label like its slice (amber/green text on white fails contrast) — pies spread `LEGEND_BASE`.
|
||||
33. **Recharts' ResponsiveContainer unmounts its chart when it measures 0×0** (`display: none`, e.g. Radix's `hidden` panel): a hidden tab must keep its size. The Charts page force-mounts visited tabs and hides the inactive ones with `absolute inset-x-0 top-0 h-0 overflow-hidden invisible` + `inert` (no scroll height, no focus); switching back otherwise rebuilt every chart and replayed its animation. Its panel elements are memoised, so a tab switch re-renders only the tab bar (a re-render restarts Recharts tweens).
|
||||
34. **The inline theme script's sha256 is in three CSPs**: `public/_headers`, `src-tauri/tauri.conf.json` and `THEME_SCRIPT_HASH` in `worker/admin.ts`. Change the script and `scripts/check-csp-hashes.test.mjs` fails with the hash to paste. A wrong hash only blocks the script (the old light flash). Keep `%` and `import(` out of it (Vite rewrites them in HTML).
|
||||
35. **`scrollbar-gutter: stable` vs dialogs**: fixed overlays stop at the reserved gutter, and react-remove-scroll-bar adds the scrollbar width as body margin on top. Hence `html:not(:has(> body[data-scroll-locked]))` — the gutter is released while a dialog locks scrolling, which is then exactly the old behaviour.
|
||||
36. **itch.io collection JSON** (`/c/<id>/<slug>?format=json&page=N`): `&page=1` redirects to the HTML page (page 1 has no `page`); a wrong or missing slug redirects to the canonical URL **without** the query string (re-request with the slug from `Location`); an unknown id answers 500; link attributes come in any order; past the last page `num_items` is 0. Zero cells while `num_items > 0` means changed markup: `collections.ts` reports it (422) and adds nothing.
|
||||
37. **D1 counts every statement of a batch** towards its 1,000 queries per invocation: bind URL lists in `IN (…)` slices of ≤ 90 (100 bound parameters at most), as `setStatus` / `reconcile` / `markIngested` do, never one statement per URL.
|
||||
38. **npm 11.20+ (Node 26) gates dependency install scripts on `allowScripts` in `webapp/package.json`.** esbuild and workerd are approved by name (their postinstall only checks the platform binary). A new dependency with an install script shows up in `npm install-scripts ls`; review it, then `npm install-scripts approve <pkg> --no-allow-scripts-pin` (name-only, so Dependabot bumps stay covered).
|
||||
39. **Dependabot's pip updater reads `.python-version` and aborts on a Python it doesn't support** (3.10–3.14 as of Oct 2026): a `3.15` there stopped pip version updates and the pip dependency graph ("Dependabot detected the following Python requirement … '3.15.*'"). So the CI pin lives in `.github/python-version`, which only `setup-python`'s `python-version-file` reads (python-ci, refresh ×2, update); move it back to `.python-version` once Dependabot lists 3.15.
|
||||
|
||||
## Safe-edit rules
|
||||
|
||||
@@ -137,10 +153,20 @@ Code **MIT** (`LICENSE`); catalog data **CC BY 4.0** for the maintainer's contri
|
||||
- Never commit secrets (GitHub App key, admin emails, Turnstile secret, tokens). Data workflows push with the job's `GITHUB_TOKEN`; the Worker writes as the GitHub App.
|
||||
- When adding an npm dep, also add it to `webapp/src/lib/about.ts` so the About page lists it.
|
||||
|
||||
## Current state (Sept 2026)
|
||||
## Current state (Oct 2026)
|
||||
|
||||
**Toolchain** (2026-10-22, on `main`, not released yet — goes into the next release's notes): TypeScript 7.0.2 (`@typescript/native`; `typescript` = the 6.0 API alias, gotcha 6), Node 26 (`webapp/.node-version`, `engines >=26`, `@types/node` 26, Workers Builds `NODE_VERSION` raised to 26 by the owner in the dashboard — `docs/operations.md` §1 step 1), Python 3.15 (`.github/python-version` — gotcha 39, ruff `py315`, `urllib3` declared since `scraper.py` imports it). No source change was needed: TS 7 reports 0 errors, the suites pass on Node 26.12 and Python 3.15.1.
|
||||
|
||||
**v4.4.1** (2026-10-16): the browser extension and the Telegram bot are retired — both repos (`poli0981/itch-f2p-extension`, `poli0981/telegram-scraper-bot`) were already public archives; an Announcements discussion says so. Removed: `POST /api/ingest` (`ingest.ts`, `requireService`, `RL_INGEST`, `ACCESS_AUD_INGEST`, the idempotency store — it never received a game: 0 `ext` rows in production D1), the admin's "Browser extension" auto-approve source and "Extension" label, the About page's Telegram numeric-ID note. The policies (Privacy, ToS, EULA + installer RTF, Disclaimer, Security; EN + VI) only lost the extension / ingest wording: "Last updated" 2026-10-16, `LEGAL_VERSION` deliberately unchanged (nothing new to accept, ToS §14). Unused repo secrets `GH_TOKEN`, `TELEGRAM_BOT_TOKEN`, `DISCUSSION_*` deleted. Ships Dependabot #131 (vite 8.3.4), #135 (tauri crates), #136 (serde_json), #137 (ruff).
|
||||
|
||||
**v4.4.0** (2026-10-13, PR #133): admin automation. Add page: any text / file / itch.io collection → **Check** (dry run) → batches of 200 → per-game ingest tracking (`/api/admin/track`, from the new `scripts/state/ingest_outcomes.json`). The cron runs the maintainer's automatic rules in one `[auto]` commit: auto-approve (on by default, `rss` + `form`, unflagged, after 24 h, only items found after the `since` baseline, ≤ 100 a run) and **followed collections** (≤ 30 pages a run; never-seen unflagged free games queued directly, removed or flagged ones to review). Bulk remove / edit ≤ 100 in one commit (preset reasons translated on the Removed page), New games, Activity, Settings; queue batches of 200. D1 migration `0002_automation.sql` applied to production before the merge. Policies rewritten for approval by rules (ToS §5, Privacy §2.3/§2.4, Disclaimer §4, EN + VI; `LEGAL_VERSION` 2026-10-13). Web: the theme button shows the theme in use (a v4 regression), a pre-paint theme script (no dark flash; hash in three CSPs), Charts tabs kept mounted (switching back reuses every chart, no replayed animation), 300 ms entrances, scrollbar gutter. Also ships #130 (2026-10-11, the Charts page on phones: one-row tab bar, 12 px legends / 11 px ticks, horizontal Languages / Avg session bars, all 15 Genre labels, data-order pie legends, translated removal reasons, locale axis numbers).
|
||||
|
||||
**v4.3.0** (2026-10-10, PR #117; released the same day): itch.io's newer info rows in the catalog — `ai_disclosure` / `ai_content` (generative-AI disclosure) and `accessibility` — on the game page, as `/games` filters (`ai`, `access`) and as charts (Overview: "Generative AI"; Discovery: "Accessibility Features", "What AI Was Used For"); `description` = the creator's tagline. The whole catalog was re-scraped the same day with `force_update.yml` (6 × 600 + 10, ≈14.5 games/min; an itch.io outage ~05:20–06:50Z only cost retries): 3,236 of 3,251 games carry the new fields — No AI 53 %, AI Assisted 11 %, not disclosed 36 %; accessibility on 14 % (Subtitles 257, One button 119); the other 15 are 14 dead-page strikes awaiting confirmation and one password-protected page (`everium.itch.io/underthethorns`; it failed every check until #129 taught the refresh to strike such pages). Fixed on the way, same day: #118 (`force_update.yml` had never worked in v4 — it now dispatches `refresh.yml`, whose scan reads the latest main so queued batches rotate), #119 (`tauri-plugin-opener` 2.7: every first v4.3.0 release build had failed; the tag was moved before publishing), #120 (the 5 open Dependabot alerts; `check:tauri` CI guard; weekly cargo Dependabot), then Dependabot #122–#127.
|
||||
|
||||
**v4.2.0** (2026-10-03, PR #103): catalog packs + lazy descriptions (first visit ≈ 264 KB instead of ≈ 520 KB), compressed IndexedDB cache, `/data/*.json` only for apps 4.0–4.1.1 (410 + repository links for everyone else), `/img` cache fixes, themed scrollbars + "Hide scrollbars", policies (`LEGAL_VERSION` 2026-10-03: data reuse goes through the repository). Checked on the live site the same day: TLS 1.2/1.3 only, HSTS preload header (domain not on the preload list), Cloudflare compression already on (zstd/br), Smart Tiered Cache on but irrelevant here (`docs/operations.md` §1 step 12).
|
||||
|
||||
**v4.1.1** (2026-09-29): contact addresses moved to `@poli0981.dev` (Google Workspace), About page links poli0981.dev + /links/, `LEGAL_VERSION` 2026-09-29.
|
||||
|
||||
**v4.1.0** (2026-09-24): the Turnstile verification gate for the web app, readable chart tooltips, the changelog split into `changelog/`, a docs refresh, updated Privacy/ToS/EULA (`LEGAL_VERSION` 2026-09-24). The gate ships switched on in `wrangler.jsonc` but stays inert until `GATE_BOT_TOKEN`, the Transform Rule and `GATE_SECRET` are in place (`docs/operations.md` §1 step 14) — `curl -s https://freeitchgames.win/api/verify` says `"enabled"`.
|
||||
|
||||
**v4.0.0** (2026-09-18) moved the site to the Cloudflare Worker at https://freeitchgames.win (PRs #81, #82, #88). The whole chain is live: Access for `/admin`, the GitHub App (verified commits to `scripts/temp_link.json` → `update.yml`), Turnstile, `ADMIN_EMAILS`; D1 `freeitchgames`, R2 `freeitchgames-thumbs`, Image Transformations (source `img.itch.zone`); Web Analytics is injected by the zone. `/api/ingest` stays unused — the browser extension is archived. **Still open (owner):** the legal-text review items; a new GPG signing key (commits are still signed with the exposed `03F965C2E2DB5C6B`); deleting the unused repo secrets `GH_TOKEN`, `TELEGRAM_BOT_TOKEN`, `DISCUSSION_*`.
|
||||
**v4.0.0** (2026-09-18) moved the site to the Cloudflare Worker at https://freeitchgames.win (PRs #81, #82, #88). The whole chain is live: Access for `/admin`, the GitHub App (verified commits to `scripts/temp_link.json` → `update.yml`), Turnstile, `ADMIN_EMAILS`; D1 `freeitchgames`, R2 `freeitchgames-thumbs`, Image Transformations (source `img.itch.zone`); Web Analytics is injected by the zone. **Still open (owner):** the legal-text review items; add the new GPG key's public half on GitHub (until then new signatures show Unverified) and remove the revoked `03F965C2E2DB5C6B` there; delete the Zero Trust Access app "Ingest" and its service token (unused since 4.4.1).
|
||||
+5
-5
@@ -12,16 +12,16 @@ Games go through the website; everything else goes through the GitHub issue/PR t
|
||||
- Optional: a short note (why it's good, content warnings…), up to 500 characters. Notes are deleted after 180 days. Please don't put personal data in it.
|
||||
- A Cloudflare Turnstile check and a rate limit keep bots out; if you get blocked, wait a bit and try again.
|
||||
- Only **free** itch.io games. Paid games are dropped automatically when the game page is scraped.
|
||||
- Nothing goes live instantly: every suggestion lands in a **review queue**, and nothing enters the catalog until the Maintainer approves it. Approved links are then scraped by the data pipeline and show up on the site after the next deploy.
|
||||
- Nothing goes live instantly: every suggestion lands in a **review queue**, where the Maintainer approves or rejects it, by hand or through automatic rules (a suggestion without a warning flag that nobody rejected within about a day may be approved automatically). Approved links are then scraped by the data pipeline and show up on the site after the next deploy.
|
||||
|
||||
Got a whole list? The **[Add Games (bulk list)]** issue template takes up to 15 links at once, and a PR that adds itch.io URLs to `scripts/temp_link.json` works too (use the [Add Games] PR template). Both go through the same review: the Maintainer checks every link before it is queued, and merging such a PR is the approval. Please don't edit `data_game/` by hand: the pipeline owns those files (see [section 7](#7-submit-code-changes-pr)).
|
||||
|
||||
#### 1b. Other ways games get in
|
||||
|
||||
- The Maintainer's companion browser extension ([itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension), separate repo, GPL-3.0) sends links through an authenticated API while browsing itch.io. It is the Maintainer's own tool, not a public submission channel.
|
||||
- The site reads itch.io's public RSS feeds of new and popular free games every 4 hours (one feed per run) and queues what it finds.
|
||||
- The site also reads the public itch.io collections the Maintainer follows and queues their new free games directly (one removed before, or with a warning flag, goes to the review queue instead).
|
||||
|
||||
Both land in the same review queue as the Suggest page. (The old Telegram bot path is retired.)
|
||||
The RSS feeds land in the same review queue as the Suggest page. (The Maintainer's browser extension and the old Telegram bot are retired; their repositories, [itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) and [telegram-scraper-bot](https://github.com/poli0981/telegram-scraper-bot), are archived.)
|
||||
|
||||
### 2. Remove Games or Fix Their Data
|
||||
|
||||
@@ -80,7 +80,7 @@ Details live in the [Terms of Use](docs/ToS.md) and the [Disclaimer](docs/DISCLA
|
||||
|
||||
Full instructions (Worker, local D1, secrets, Tauri, Android) are in **[`docs/dev_env.md`](docs/dev_env.md)**; the Tauri/Android build notes are in [`webapp/TAURI.md`](webapp/TAURI.md). The short version:
|
||||
|
||||
**Python pipeline** (Python 3.14, see `.python-version`):
|
||||
**Python pipeline** (Python 3.15, see `.github/python-version`):
|
||||
|
||||
```sh
|
||||
python -m venv .venv
|
||||
@@ -96,7 +96,7 @@ To try a real scrape locally: add a URL to `scripts/temp_link.json`, then run
|
||||
`python scripts/update_info.py --out patch.json` and `python scripts/apply_patch.py patch.json`
|
||||
(don't commit the result; see the PR rules above).
|
||||
|
||||
**Webapp + Worker** (Node.js 22.22+, CI uses the version in `webapp/.node-version`):
|
||||
**Webapp + Worker** (Node.js 26+, CI uses the version in `webapp/.node-version`):
|
||||
|
||||
```sh
|
||||
cd webapp
|
||||
|
||||
@@ -45,7 +45,7 @@ The process is described in the [Terms of Use](docs/ToS.md).
|
||||
| **Catalog data** | `data_game/`, `scripts/deleted_games.json`, and the files built from them under <https://freeitchgames.win/data> | CC BY 4.0, for the Maintainer's contribution only. Creator descriptions, cover images, game names and trademarks are **excluded** | [`data_game/LICENSE.md`](data_game/LICENSE.md), [`LICENSES/CC-BY-4.0.txt`](LICENSES/CC-BY-4.0.txt) |
|
||||
| **Documentation** | `*.md` files in the repository root and in `docs/` (README, policies, guides and their Vietnamese translations) | CC BY 4.0 | [`LICENSES/CC-BY-4.0.txt`](LICENSES/CC-BY-4.0.txt) |
|
||||
| **Third-party components** | Libraries, frameworks and tools the project uses | Their own licenses | [`docs/THIRD_PARTY.md`](docs/THIRD_PARTY.md) and the [About page](https://freeitchgames.win/about) |
|
||||
| **Browser extension** | [poli0981/itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension), a separate repository | GPL-3.0 | That repository |
|
||||
| **Browser extension** (archived) | [poli0981/itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension), a separate repository, archived and no longer used | GPL-3.0 | That repository |
|
||||
|
||||
When you reuse the documentation, credit poli0981 (SkullMute), link to this repository and to CC BY 4.0,
|
||||
and say if you changed it. For the data, use the credit line in
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Free Itch.io Games List
|
||||
|
||||
[](https://github.com/poli0981/free-games-itchio-list/releases/latest)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/releases/latest)
|
||||
[](https://freeitchgames.win)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/stargazers)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/network/members)
|
||||
@@ -35,7 +35,7 @@ read-only for everyone: no accounts, no sign-in, no comments, no ads, no payment
|
||||
Everything lives on the website, **<https://freeitchgames.win>**:
|
||||
|
||||
- **Games**: the whole catalog in one fast table (a compact list on phones), with filters for genre, platform,
|
||||
status and more, plus a detail page for every game.
|
||||
status, the creator's generative-AI disclosure, accessibility and more, plus a detail page for every game.
|
||||
- **Charts**: how the catalog breaks down, and how it has grown over time.
|
||||
- **Removed**: every game dropped from the catalog, with the reason
|
||||
(<https://freeitchgames.win/removed>).
|
||||
@@ -49,9 +49,10 @@ Everything lives on the website, **<https://freeitchgames.win>**:
|
||||
Every link goes to the game's own page on itch.io. Cover images on the website are resized copies served
|
||||
by freeitchgames.win, so your browser does not contact itch.io until you click a link.
|
||||
|
||||
Want the raw data? It is plain JSON, in [`data_game/`](data_game/) in this repo and at
|
||||
<https://freeitchgames.win/data/index.json> (plus the files it lists). Please use those files instead of
|
||||
scraping the website; see [License](#license) for how you may reuse them.
|
||||
Want the raw data? It is plain JSON in [`data_game/`](data_game/) (plus
|
||||
[`scripts/deleted_games.json`](scripts/deleted_games.json)) in this repo: clone it, download it, or read the
|
||||
raw files. Please use those instead of scraping the website, which serves its own compact format; see
|
||||
[License](#license) for how you may reuse them.
|
||||
|
||||
(The old address, `poli0981.github.io/free-games-itchio-list`, only redirects to the new site. The
|
||||
per-genre markdown tables under `lists/` and `deleted_games.txt` are gone: the catalog is web-only.)
|
||||
@@ -68,7 +69,7 @@ from the latest [GitHub Release](https://github.com/poli0981/free-games-itchio-l
|
||||
| Linux | `.deb`, `.rpm` or `.AppImage` |
|
||||
| Android 11+ (arm64-v8a) | `.apk` (sideloaded, see below) |
|
||||
|
||||
The apps download the catalog JSON from <https://freeitchgames.win/data> and load cover images directly
|
||||
The apps download the catalog data from <https://freeitchgames.win/data> and load cover images directly
|
||||
from itch.io's image CDN (`img.itch.zone`); clicking a game opens itch.io in your browser. No telemetry.
|
||||
Updates are manual: download a new release when one comes out. Installing an app means accepting the
|
||||
[EULA](docs/EULA.md).
|
||||
@@ -92,20 +93,26 @@ still gets security hardening and that we actually test on (≈87% of active dev
|
||||
|
||||
## Add a game
|
||||
|
||||
There are three ways a game gets into the catalog. **All three go through a review queue**: nothing is
|
||||
added until the Maintainer approves it.
|
||||
There are three ways a game gets into the catalog. Each is approved by the Maintainer, one by one in the
|
||||
review queue or through automatic rules the Maintainer sets.
|
||||
|
||||
1. **Suggest page**: <https://freeitchgames.win/suggest>. Paste an itch.io game link and, if you like, a
|
||||
short note (up to 500 characters; notes are deleted after 180 days). A Cloudflare Turnstile check and a
|
||||
rate limit keep spam out. Please don't put personal data in the note.
|
||||
2. **Browser extension**: the Maintainer's own
|
||||
[itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) (separate repo, GPL-3.0) sends
|
||||
links through an authenticated API while browsing itch.io.
|
||||
3. **Automatic discovery**: every 4 hours the site reads one of itch.io's public RSS feeds of new and
|
||||
2. **Automatic discovery**: every 4 hours the site reads one of itch.io's public RSS feeds of new and
|
||||
popular free games (one feed per run) and queues what it finds.
|
||||
3. **Followed collections**: the site also reads the public itch.io collections the Maintainer follows and
|
||||
queues their new free games directly (one removed before, or with a warning flag, goes to the review
|
||||
queue instead).
|
||||
|
||||
The Maintainer reviews the queue in a private admin area (protected by Cloudflare Access). Approved links
|
||||
are handed to the data pipeline, which scrapes each game page; paid games are dropped.
|
||||
The Maintainer's browser extension and the old Telegram bot are retired; their repositories
|
||||
([itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension),
|
||||
[telegram-scraper-bot](https://github.com/poli0981/telegram-scraper-bot)) are archived.
|
||||
|
||||
The Maintainer reviews the queue in a private admin area (protected by Cloudflare Access); an item
|
||||
without a warning flag that nobody rejected within a waiting time (24 hours by default) is approved
|
||||
automatically. Approved links are handed to the data pipeline, which scrapes each game page; paid games
|
||||
are dropped.
|
||||
|
||||
Prefer GitHub, or have a long list? The "Add Games (bulk list)" issue template works too; those links go
|
||||
through the same review. See [CONTRIBUTING.md](CONTRIBUTING.md#1-suggest-new-games-most-welcome).
|
||||
@@ -119,9 +126,9 @@ Details: [Terms of Use](docs/ToS.md) and [Disclaimer](docs/DISCLAIMER.md).
|
||||
## How the data stays fresh
|
||||
|
||||
```
|
||||
Suggest page ──┐
|
||||
Extension ──┼─→ review queue ─→ Maintainer approves ─→ scripts/temp_link.json
|
||||
RSS feeds ──┘ (admin) │
|
||||
Suggest page ──┬─→ review queue ─→ Maintainer approves ─→ scripts/temp_link.json
|
||||
RSS feeds ──┘ (admin, or auto rules) │
|
||||
Followed collections ─────────────────────────────────────────┤
|
||||
▼
|
||||
update.yml → update_info.py (new games) ──┐
|
||||
refresh.yml → refresh.py (1/7 per day) ──┴─→ patch ─→ apply_patch.py
|
||||
@@ -155,14 +162,14 @@ The pipeline is polite to itch.io: it identifies itself as
|
||||
- **Web app** ([`webapp/src/`](webapp/src/)): React + TypeScript + Vite + Tailwind CSS, TanStack
|
||||
Query, React Router, Zustand. English and Vietnamese UI.
|
||||
- **Cloudflare Worker** ([`webapp/worker/`](webapp/worker/)): serves the site and `/data`, the verification gate
|
||||
(Cloudflare Turnstile), the image proxy (`/img`, resized WebP covers stored on Cloudflare R2), the Suggest and ingest APIs, the review queue
|
||||
(Cloudflare Turnstile), the image proxy (`/img`, resized WebP covers stored on Cloudflare R2), the Suggest API, the review queue
|
||||
(Cloudflare D1), the admin API and the RSS discovery schedule. Config:
|
||||
[`webapp/wrangler.jsonc`](webapp/wrangler.jsonc).
|
||||
- **Admin app** ([`webapp/admin/`](webapp/admin/)): the Maintainer-only review screen at `/admin`, behind
|
||||
Cloudflare Access. It writes to this repo as a GitHub App.
|
||||
- **Native shell** ([`webapp/src-tauri/`](webapp/src-tauri/)): Tauri 2 for Windows, macOS, Linux and
|
||||
Android. See [`webapp/TAURI.md`](webapp/TAURI.md).
|
||||
- **Data pipeline** ([`scripts/`](scripts/), Python 3.14): scraper, refresh, patch/apply and validation,
|
||||
- **Data pipeline** ([`scripts/`](scripts/), Python 3.15): scraper, refresh, patch/apply and validation,
|
||||
run by GitHub Actions.
|
||||
|
||||
Deploys happen through Cloudflare Workers Builds (root dir `webapp`, `npm run build`, then
|
||||
@@ -224,9 +231,9 @@ docs/ # Policies, dev environment, third-party list (+ Vietnam
|
||||
| Job | Schedule | Purpose |
|
||||
|---|---|---|
|
||||
| Ingest (`update.yml`) | On queue change + daily 01:23 UTC | Scrape approved links from `temp_link.json`, skip paid / dead / removed |
|
||||
| Refresh (`refresh.yml`) | Daily 02:47 UTC | Re-check 1/7 of the catalog: dead links, now-paid games, rating, status |
|
||||
| Refresh (`refresh.yml`) | Daily 02:47 UTC | Re-check 1/7 of the catalog: dead links, now-paid or password-protected games, rating, status |
|
||||
| Force update (`force_update.yml`) | Manual | Re-scrape every field (one URL or the next batch); keeps `safe_virus` / `notes` / `nsfw` |
|
||||
| RSS discovery (Cloudflare Worker) | Every 4 hours | Poll one itch.io feed per run; new games go to the review queue |
|
||||
| RSS discovery and automatic rules (Cloudflare Worker) | Every 4 hours | Poll one itch.io feed per run into the review queue; auto-approve unflagged items after the waiting time; queue new free games of followed collections |
|
||||
| Python CI / Webapp CI | Pull requests (Python CI also on pushes to `main` that touch it) | Lint, tests, data validation / type-check, tests, build |
|
||||
| Deploy website | On push to `main` | Cloudflare Workers Builds (`webapp/wrangler.jsonc`) → freeitchgames.win |
|
||||
| Build monitor (`deploy-status.yml`) | Hourly | Posts each failed Workers Build of `main` to Discord, once |
|
||||
@@ -238,8 +245,9 @@ mass changes are reported to the project's Discord as automated notifications.
|
||||
|
||||
## Data fields
|
||||
|
||||
Each game in `data_game/game_info_NNN.json` has these 20 fields (plus optional `added_at` / `updated_at`
|
||||
timestamps added by the pipeline). Only `safe_virus`, `notes` and `nsfw` are maintained by the Maintainer;
|
||||
Each game in `data_game/game_info_NNN.json` has these 20 fields, plus optional `added_at` / `updated_at`
|
||||
timestamps added by the pipeline and, since 4.3, the optional `ai_disclosure` / `ai_content` /
|
||||
`accessibility` fields (filled in as each game is re-checked). Only `safe_virus`, `notes` and `nsfw` are maintained by the Maintainer;
|
||||
everything else is scraped from the game's public itch.io page, and no re-scrape ever overwrites those three.
|
||||
|
||||
| Field | Description |
|
||||
@@ -247,7 +255,7 @@ everything else is scraped from the game's public itch.io page, and no re-scrape
|
||||
| `url` | Game page URL |
|
||||
| `name` | Game title |
|
||||
| `dev` | Developer / author name(s) |
|
||||
| `description` | Short description (first sentence, max 200 chars) |
|
||||
| `description` | The creator's tagline on itch.io, else the description's first sentence (max 200 chars) |
|
||||
| `genre` | Genre(s) from itch.io |
|
||||
| `tags` | All tags |
|
||||
| `status` | Release status |
|
||||
@@ -264,6 +272,9 @@ everything else is scraped from the game's public itch.io page, and no re-scrape
|
||||
| `safe_virus` | Manual safety note: `?` (default), `Yes`, `No`, `Caution`. A note, not a guarantee: downloads are not scanned |
|
||||
| `notes` | Manual notes |
|
||||
| `thumbnail` | Cover image URL (on itch.io) |
|
||||
| `ai_disclosure` | The creator's generative-AI disclosure: `No AI`, `AI Assisted` (itch.io's wording) or `N/A` (not disclosed) |
|
||||
| `ai_content` | What AI was used for when assisted (`Graphics`, `Text`, `Code`, `Sounds`) |
|
||||
| `accessibility` | Accessibility features (`Subtitles`, `Configurable controls`, `Color-blind friendly`, …) |
|
||||
|
||||
Scraped fields default to `N/A` when not available on the game page.
|
||||
|
||||
@@ -323,8 +334,9 @@ Free itch.io games catalog by poli0981 (SkullMute) — https://freeitchgames.win
|
||||
```
|
||||
|
||||
Third-party components keep their own licenses: see [`docs/THIRD_PARTY.md`](docs/THIRD_PARTY.md) and the
|
||||
site's About page. The browser extension is a separate repo under GPL-3.0. Contributions are accepted under
|
||||
the same license as the part they change (inbound = outbound): MIT for code, CC BY 4.0 for data and docs.
|
||||
site's About page. The archived browser extension is a separate repo under GPL-3.0. Contributions are
|
||||
accepted under the same license as the part they change (inbound = outbound): MIT for code, CC BY 4.0 for
|
||||
data and docs.
|
||||
|
||||
**Not affiliated with itch.io.** This is an independent fan project. It is not affiliated with, endorsed by,
|
||||
or sponsored by itch.io, Leaf Corcoran or itch corp. "itch.io" and all game names, logos and trademarks
|
||||
|
||||
+23
-18
@@ -1,6 +1,6 @@
|
||||
# Danh sách Game Itch.io Miễn Phí
|
||||
|
||||
[](https://github.com/poli0981/free-games-itchio-list/releases/latest)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/releases/latest)
|
||||
[](https://freeitchgames.win)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/stargazers)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/network/members)
|
||||
@@ -34,7 +34,7 @@ Một danh mục được tuyển chọn và tự động cập nhật gồm **h
|
||||
|
||||
Mọi thứ nằm trên website **<https://freeitchgames.win>**:
|
||||
|
||||
- **Danh sách game**: toàn bộ danh mục trong một bảng chạy nhanh (dạng danh sách gọn trên điện thoại), lọc theo thể loại, nền tảng, trạng thái và nhiều thứ khác, cùng một trang chi tiết cho từng game.
|
||||
- **Danh sách game**: toàn bộ danh mục trong một bảng chạy nhanh (dạng danh sách gọn trên điện thoại), lọc theo thể loại, nền tảng, trạng thái, công bố của tác giả về AI tạo sinh, hỗ trợ tiếp cận và nhiều thứ khác, cùng một trang chi tiết cho từng game.
|
||||
- **Biểu đồ**: danh mục được chia ra sao, và đã lớn lên thế nào theo thời gian.
|
||||
- **Đã gỡ** (Removed): mọi game bị loại khỏi danh mục, kèm lý do (<https://freeitchgames.win/removed>).
|
||||
- **Kiểm tra nhanh trước.** Lần đầu vào, rồi khoảng mỗi 48 giờ, trang sẽ chạy một bước kiểm tra Cloudflare Turnstile ngắn để chặn bot. Thường nó tự qua rồi mở đúng trang bạn cần. Dữ liệu trong `/data` và các ứng dụng không cần bước này.
|
||||
@@ -42,7 +42,7 @@ Mọi thứ nằm trên website **<https://freeitchgames.win>**:
|
||||
|
||||
Mọi link đều dẫn tới trang riêng của game trên itch.io. Ảnh bìa trên website là bản thu nhỏ do freeitchgames.win phục vụ, nên trình duyệt của bạn không kết nối tới itch.io cho tới khi bạn bấm vào một link.
|
||||
|
||||
Muốn lấy dữ liệu gốc? Đó là JSON thuần, nằm trong [`data_game/`](data_game/) của repo này và tại <https://freeitchgames.win/data/index.json> (cùng các file mà nó liệt kê). Vui lòng dùng các file đó thay vì cào (scrape) website; xem [Giấy phép](#giấy-phép) để biết bạn được tái sử dụng chúng thế nào.
|
||||
Muốn lấy dữ liệu gốc? Đó là JSON thuần, nằm trong [`data_game/`](data_game/) (cùng [`scripts/deleted_games.json`](scripts/deleted_games.json)) của repo này: clone, tải về hoặc đọc file raw. Vui lòng dùng các file đó thay vì cào (scrape) website, vốn chỉ phục vụ định dạng thu gọn riêng của nó; xem [Giấy phép](#giấy-phép) để biết bạn được tái sử dụng chúng thế nào.
|
||||
|
||||
(Địa chỉ cũ `poli0981.github.io/free-games-itchio-list` giờ chỉ chuyển hướng sang website mới. Các bảng markdown theo thể loại trong `lists/` và file `deleted_games.txt` đã bị bỏ: danh mục giờ chỉ có trên web.)
|
||||
|
||||
@@ -57,7 +57,7 @@ Cùng một app cũng được phát hành dưới dạng **ứng dụng native
|
||||
| Linux | `.deb`, `.rpm` hoặc `.AppImage` |
|
||||
| Android 11+ (arm64-v8a) | `.apk` (cài tay, xem bên dưới) |
|
||||
|
||||
Các app tải JSON danh mục từ <https://freeitchgames.win/data> và tải ảnh bìa trực tiếp từ CDN ảnh của itch.io (`img.itch.zone`); bấm vào một game sẽ mở itch.io trong trình duyệt của bạn. Không có telemetry. Cập nhật thủ công: khi có bản mới thì tải bản mới về. Cài app đồng nghĩa với việc chấp nhận [EULA](docs/i18n/vi/EULA.md) ([EN](docs/EULA.md)).
|
||||
Các app tải dữ liệu danh mục từ <https://freeitchgames.win/data> và tải ảnh bìa trực tiếp từ CDN ảnh của itch.io (`img.itch.zone`); bấm vào một game sẽ mở itch.io trong trình duyệt của bạn. Không có telemetry. Cập nhật thủ công: khi có bản mới thì tải bản mới về. Cài app đồng nghĩa với việc chấp nhận [EULA](docs/i18n/vi/EULA.md) ([EN](docs/EULA.md)).
|
||||
|
||||
### Android (tải & cài đặt)
|
||||
|
||||
@@ -71,13 +71,15 @@ Lưu ý: **chỉ arm64-v8a** (mọi điện thoại từ ~2017; không hỗ tr
|
||||
|
||||
## Thêm game
|
||||
|
||||
Có ba cách để một game vào danh mục. **Cả ba đều đi qua hàng chờ duyệt**: không có gì được thêm cho tới khi Maintainer duyệt.
|
||||
Có ba cách để một game vào danh mục. Mỗi game đều được Maintainer duyệt, từng game một trong hàng chờ duyệt hoặc qua các quy tắc tự động do Maintainer đặt ra.
|
||||
|
||||
1. **Trang Suggest**: <https://freeitchgames.win/suggest>. Dán link game itch.io và, nếu muốn, một ghi chú ngắn (tối đa 500 ký tự; ghi chú bị xóa sau 180 ngày). Một bước kiểm tra Cloudflare Turnstile và giới hạn tần suất giúp chặn spam. Vui lòng không ghi dữ liệu cá nhân vào ghi chú.
|
||||
2. **Extension trình duyệt**: [itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) của chính Maintainer (repo riêng, GPL-3.0) gửi link qua một API có xác thực trong lúc duyệt itch.io.
|
||||
3. **Tự động phát hiện**: cứ 4 giờ một lần, website đọc một trong các RSS feed công khai của itch.io về game miễn phí mới và phổ biến (mỗi lượt một feed) rồi đưa những gì tìm được vào hàng chờ.
|
||||
2. **Tự động phát hiện**: cứ 4 giờ một lần, website đọc một trong các RSS feed công khai của itch.io về game miễn phí mới và phổ biến (mỗi lượt một feed) rồi đưa những gì tìm được vào hàng chờ.
|
||||
3. **Collection được theo dõi**: website cũng đọc các collection công khai trên itch.io mà Maintainer theo dõi và đưa thẳng game miễn phí mới của chúng vào hàng chờ ingest (game từng bị gỡ, hoặc có cờ cảnh báo, thì vào hàng chờ duyệt).
|
||||
|
||||
Maintainer duyệt hàng chờ trong một khu vực quản trị riêng (được bảo vệ bởi Cloudflare Access). Link được duyệt sẽ được chuyển cho pipeline dữ liệu, pipeline này cào từng trang game; game trả phí bị loại.
|
||||
Extension trình duyệt của Maintainer và Telegram bot cũ đã ngừng hoạt động; repo của chúng ([itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension), [telegram-scraper-bot](https://github.com/poli0981/telegram-scraper-bot)) đã được lưu trữ (archive).
|
||||
|
||||
Maintainer duyệt hàng chờ trong một khu vực quản trị riêng (được bảo vệ bởi Cloudflare Access); mục không có cờ cảnh báo mà không ai từ chối trong thời gian chờ (mặc định 24 giờ) được duyệt tự động. Link được duyệt sẽ được chuyển cho pipeline dữ liệu, pipeline này cào từng trang game; game trả phí bị loại.
|
||||
|
||||
Thích dùng GitHub hơn, hoặc có cả một danh sách dài? Issue template "Add Games (bulk list)" cũng dùng được; các link đó đi qua cùng quy trình duyệt. Xem [CONTRIBUTING](docs/i18n/vi/CONTRIBUTING.md#1-đề-xuất-game-mới-hoan-nghênh-nhất).
|
||||
|
||||
@@ -86,9 +88,9 @@ Muốn **gỡ** một game, hoặc sửa dữ liệu của nó (nhất là các
|
||||
## Dữ liệu được cập nhật thế nào
|
||||
|
||||
```
|
||||
Trang Suggest ─┐
|
||||
Extension ─┼─→ hàng chờ duyệt ─→ Maintainer duyệt ─→ scripts/temp_link.json
|
||||
RSS feed ─┘ (admin) │
|
||||
Trang Suggest ─┬─→ hàng chờ duyệt ─→ Maintainer duyệt ─→ scripts/temp_link.json
|
||||
RSS feed ─┘ (admin, hoặc quy tắc tự động) │
|
||||
Collection theo dõi ──────────────────────────────────────────┤
|
||||
▼
|
||||
update.yml → update_info.py (game mới) ──┐
|
||||
refresh.yml → refresh.py (1/7 mỗi ngày) ──┴─→ patch ─→ apply_patch.py
|
||||
@@ -108,10 +110,10 @@ Pipeline cư xử lịch sự với itch.io: nó tự xưng danh là `FreeItchGa
|
||||
## Bên trong dự án
|
||||
|
||||
- **Web app** ([`webapp/src/`](webapp/src/)): React + TypeScript + Vite + Tailwind CSS, TanStack Query, React Router, Zustand. Giao diện tiếng Anh và tiếng Việt.
|
||||
- **Cloudflare Worker** ([`webapp/worker/`](webapp/worker/)): phục vụ website và `/data`, cổng xác minh (Cloudflare Turnstile), proxy ảnh (`/img`, ảnh bìa WebP đã thu nhỏ lưu trên Cloudflare R2), API Suggest và ingest, hàng chờ duyệt (Cloudflare D1), API quản trị và lịch phát hiện qua RSS. Cấu hình: [`webapp/wrangler.jsonc`](webapp/wrangler.jsonc).
|
||||
- **Cloudflare Worker** ([`webapp/worker/`](webapp/worker/)): phục vụ website và `/data`, cổng xác minh (Cloudflare Turnstile), proxy ảnh (`/img`, ảnh bìa WebP đã thu nhỏ lưu trên Cloudflare R2), API Suggest, hàng chờ duyệt (Cloudflare D1), API quản trị và lịch phát hiện qua RSS. Cấu hình: [`webapp/wrangler.jsonc`](webapp/wrangler.jsonc).
|
||||
- **App quản trị** ([`webapp/admin/`](webapp/admin/)): màn hình duyệt chỉ dành cho Maintainer tại `/admin`, đứng sau Cloudflare Access. Nó ghi vào repo này dưới danh nghĩa một GitHub App.
|
||||
- **Vỏ native** ([`webapp/src-tauri/`](webapp/src-tauri/)): Tauri 2 cho Windows, macOS, Linux và Android. Xem [`webapp/TAURI.md`](webapp/TAURI.md).
|
||||
- **Pipeline dữ liệu** ([`scripts/`](scripts/), Python 3.14): scraper, refresh, patch/apply và validate, chạy bằng GitHub Actions.
|
||||
- **Pipeline dữ liệu** ([`scripts/`](scripts/), Python 3.15): scraper, refresh, patch/apply và validate, chạy bằng GitHub Actions.
|
||||
|
||||
Việc deploy do Cloudflare Workers Builds đảm nhận (thư mục gốc `webapp`, `npm run build`, rồi `npx wrangler deploy`): mỗi lần push vào `main` sẽ tự động lên site.
|
||||
|
||||
@@ -170,9 +172,9 @@ docs/ # Chính sách, môi trường dev, danh sách bên th
|
||||
| Tác vụ | Lịch | Mục đích |
|
||||
|---|---|---|
|
||||
| Ingest (`update.yml`) | Khi hàng chờ đổi + hằng ngày 01:23 UTC | Cào link đã duyệt từ `temp_link.json`, bỏ qua game trả phí / đã chết / từng bị gỡ |
|
||||
| Refresh (`refresh.yml`) | Hằng ngày 02:47 UTC | Kiểm tra lại 1/7 danh mục: link chết, game chuyển sang trả phí, rating, trạng thái |
|
||||
| Refresh (`refresh.yml`) | Hằng ngày 02:47 UTC | Kiểm tra lại 1/7 danh mục: link chết, game chuyển sang trả phí hoặc bị đặt mật khẩu, rating, trạng thái |
|
||||
| Force update (`force_update.yml`) | Thủ công | Cào lại mọi trường (một URL hoặc lô tiếp theo); giữ nguyên `safe_virus` / `notes` / `nsfw` |
|
||||
| Phát hiện qua RSS (Cloudflare Worker) | Mỗi 4 giờ | Đọc một feed itch.io mỗi lượt; game mới vào hàng chờ duyệt |
|
||||
| Phát hiện qua RSS và quy tắc tự động (Cloudflare Worker) | Mỗi 4 giờ | Đọc một feed itch.io mỗi lượt vào hàng chờ duyệt; tự duyệt mục không có cờ sau thời gian chờ; đưa game miễn phí mới của collection được theo dõi vào hàng chờ ingest |
|
||||
| Python CI / Webapp CI | Pull request (Python CI còn chạy khi push vào `main` chạm tới phần Python) | Lint, test, validate dữ liệu / type-check, test, build |
|
||||
| Deploy website | Khi push vào `main` | Cloudflare Workers Builds (`webapp/wrangler.jsonc`) → freeitchgames.win |
|
||||
| Theo dõi build (`deploy-status.yml`) | Mỗi giờ | Báo mỗi Workers Build lỗi của `main` lên Discord, một lần |
|
||||
@@ -182,14 +184,14 @@ Lỗi mạng được coi là tạm thời; game chỉ bị gỡ khi cùng lỗi
|
||||
|
||||
## Các trường dữ liệu
|
||||
|
||||
Mỗi game trong `data_game/game_info_NNN.json` có 20 trường sau (cộng thêm dấu thời gian tùy chọn `added_at` / `updated_at` do pipeline thêm vào). Chỉ `safe_virus`, `notes` và `nsfw` do Maintainer quản lý; mọi thứ khác được cào từ trang itch.io công khai của game, và không lần cào lại nào ghi đè ba trường đó.
|
||||
Mỗi game trong `data_game/game_info_NNN.json` có 20 trường sau, cộng thêm dấu thời gian tùy chọn `added_at` / `updated_at` do pipeline thêm vào và, từ bản 4.3, các trường tùy chọn `ai_disclosure` / `ai_content` / `accessibility` (được điền dần khi từng game được kiểm tra lại). Chỉ `safe_virus`, `notes` và `nsfw` do Maintainer quản lý; mọi thứ khác được cào từ trang itch.io công khai của game, và không lần cào lại nào ghi đè ba trường đó.
|
||||
|
||||
| Trường | Mô tả |
|
||||
|-------------------|------------------------------------------------------------|
|
||||
| `url` | URL trang game |
|
||||
| `name` | Tên game |
|
||||
| `dev` | Tên nhà phát triển / tác giả |
|
||||
| `description` | Mô tả ngắn (câu đầu tiên, tối đa 200 ký tự) |
|
||||
| `description` | Tagline của tác giả trên itch.io, nếu không có thì là câu đầu tiên của phần mô tả (tối đa 200 ký tự) |
|
||||
| `genre` | Thể loại theo itch.io |
|
||||
| `tags` | Tất cả tag |
|
||||
| `status` | Trạng thái phát hành |
|
||||
@@ -206,6 +208,9 @@ Mỗi game trong `data_game/game_info_NNN.json` có 20 trường sau (cộng th
|
||||
| `safe_virus` | Ghi chú an toàn thủ công: `?` (mặc định), `Yes`, `No`, `Caution`. Chỉ là ghi chú, không phải bảo đảm: file tải về không được quét |
|
||||
| `notes` | Ghi chú thủ công |
|
||||
| `thumbnail` | URL ảnh bìa (trên itch.io) |
|
||||
| `ai_disclosure` | Công bố của tác giả về AI tạo sinh: `No AI`, `AI Assisted` (cách ghi của itch.io) hoặc `N/A` (chưa công bố) |
|
||||
| `ai_content` | Phần dùng AI khi có dùng AI hỗ trợ (`Graphics`, `Text`, `Code`, `Sounds`) |
|
||||
| `accessibility` | Tính năng hỗ trợ tiếp cận (`Subtitles`, `Configurable controls`, `Color-blind friendly`, …) |
|
||||
|
||||
Các trường được cào mặc định là `N/A` khi trang game không có thông tin.
|
||||
|
||||
@@ -258,7 +263,7 @@ Khi tái sử dụng dữ liệu, vui lòng ghi công như sau:
|
||||
Free itch.io games catalog by poli0981 (SkullMute) — https://freeitchgames.win — CC BY 4.0
|
||||
```
|
||||
|
||||
Các thành phần bên thứ ba giữ giấy phép riêng của chúng: xem [`docs/THIRD_PARTY.md`](docs/THIRD_PARTY.md) và trang About của website. Extension trình duyệt là một repo riêng theo GPL-3.0. Đóng góp được nhận theo cùng giấy phép với phần mà nó thay đổi (inbound = outbound): MIT cho mã nguồn, CC BY 4.0 cho dữ liệu và tài liệu.
|
||||
Các thành phần bên thứ ba giữ giấy phép riêng của chúng: xem [`docs/THIRD_PARTY.md`](docs/THIRD_PARTY.md) và trang About của website. Extension trình duyệt (đã archive) là một repo riêng theo GPL-3.0. Đóng góp được nhận theo cùng giấy phép với phần mà nó thay đổi (inbound = outbound): MIT cho mã nguồn, CC BY 4.0 cho dữ liệu và tài liệu.
|
||||
|
||||
**Không liên kết với itch.io.** Đây là một dự án độc lập do người hâm mộ làm. Dự án không liên kết với, không được chứng thực hay tài trợ bởi itch.io, Leaf Corcoran hoặc itch corp. "itch.io" cùng mọi tên game, logo và nhãn hiệu thuộc về chủ sở hữu tương ứng.
|
||||
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
# Security Policy
|
||||
|
||||
Last updated: 2026-09-29
|
||||
Last updated: 2026-10-16
|
||||
|
||||
Applies from: the release of version 4.0.0.
|
||||
|
||||
@@ -24,7 +24,7 @@ A good report includes:
|
||||
|
||||
**In scope**
|
||||
|
||||
- The website https://freeitchgames.win and its Cloudflare Worker, including the image proxy (`/img`), the Suggest API (`/api/suggest`), the ingest API used by the Maintainer's browser extension (`/api/ingest`), and the admin app and API (`/admin`, `/api/admin`).
|
||||
- The website https://freeitchgames.win and its Cloudflare Worker, including the image proxy (`/img`), the Suggest API (`/api/suggest`), and the admin app and API (`/admin`, `/api/admin`).
|
||||
- The data pipeline (`scripts/`) and the GitHub Actions workflows (`.github/workflows/`).
|
||||
- The desktop apps and the Android APK published on [GitHub Releases](https://github.com/poli0981/free-games-itchio-list/releases).
|
||||
- The published data (`data_game/`, `scripts/deleted_games.json`, and https://freeitchgames.win/data) — for example, a way to get content into the catalog without the Maintainer's approval.
|
||||
@@ -37,7 +37,7 @@ A good report includes:
|
||||
- Volumetric denial-of-service (DoS/DDoS) and load testing.
|
||||
- Reports from automated scanners without a working proof of concept.
|
||||
|
||||
The browser extension (poli0981/itch-f2p-extension) lives in its own repository; report issues in the extension itself there. Issues in the ingest API it talks to are in scope here.
|
||||
The browser extension (poli0981/itch-f2p-extension) and the Telegram bot (poli0981/telegram-scraper-bot) are archived and no longer supported, so they are out of scope.
|
||||
|
||||
## 3. Supported versions
|
||||
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# Usage: bash bash/commit_push.sh PATCH.json "commit message"
|
||||
#
|
||||
# Every data writer (refresh, ingest, admin Worker, browser extension) pushes
|
||||
# Every data writer (refresh, ingest, the admin Worker and its cron) pushes
|
||||
# to main. Instead of rebasing whole-file JSON rewrites (which conflict), each
|
||||
# attempt starts from a fresh origin/main and re-applies the URL-keyed patch;
|
||||
# apply_patch.py is idempotent and validates before anything is committed.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# Changelog — 4.2.x
|
||||
|
||||
Release notes for 4.2.0. Every version: [CHANGELOG.md](../../CHANGELOG.md).
|
||||
|
||||
## [4.2.0] - 2026-10-03 (Compact catalog data, themed scrollbars)
|
||||
|
||||
Version 4.2 sends the catalog in a compact binary format that is about half the size on a first
|
||||
visit, keeps the offline copy compressed, points everyone who wants the raw data to the GitHub
|
||||
repository, and adds themed scrollbars with an option to hide them.
|
||||
|
||||
### Upgrade notes
|
||||
|
||||
- **Updated Privacy Policy, Terms of Use and EULA** (the data now comes from the repository; the
|
||||
compressed cache; the scrollbar setting): the site and the apps ask you to accept them again once.
|
||||
- **Website**: the first visit after the update downloads the catalog once in the new format; after
|
||||
that only the parts that changed are fetched.
|
||||
- **Desktop and Android apps 4.0–4.1.1 keep working**: they still get the older JSON files. Install
|
||||
4.2.0 over them as usual to get the smaller downloads.
|
||||
- **Reusing the data?** `https://freeitchgames.win/data/*.json` now answers 410 in a browser or a
|
||||
script. Get the same files from the repository: [`data_game/`](../../data_game/) and
|
||||
[`scripts/deleted_games.json`](../../scripts/deleted_games.json) (CC BY 4.0, as before).
|
||||
|
||||
### Added
|
||||
|
||||
- **Themed scrollbars**: slim, rounded, in the site's greys (the accent while you drag), in light and
|
||||
dark mode, on the page and in every scrolling panel. Phones keep their own overlay scrollbars.
|
||||
- **Hide scrollbars** (Settings → Appearance, and the display menu): hides every scrollbar, the page's
|
||||
included. Scrolling with the mouse wheel, touchpad, touch and keyboard still works.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Catalog packs.** The site and the apps read `/data/pack/manifest-v1.json` plus small compressed
|
||||
`.bin` packs: the catalog stored by column with dictionaries, then deflated at build time
|
||||
([`webapp/src/lib/data/pack-format.ts`](../../webapp/src/lib/data/pack-format.ts),
|
||||
[`webapp/vite-plugins/catalog-data.ts`](../../webapp/vite-plugins/catalog-data.ts)). A pack's name is a
|
||||
hash of its bytes, so it is cached for good and a returning visitor downloads only the packs that
|
||||
changed (usually 1–3 of 7 a day). First visit: about 264 KB instead of about 520 KB of JSON.
|
||||
- **Descriptions load when needed**: they are about 40 % of the catalog, so they live in their own
|
||||
packs. A game page loads the descriptions of its chunk (about 25 KB); a search loads all of them
|
||||
and adds the matches found in descriptions when they arrive ("Searching descriptions…").
|
||||
- **Offline copy compressed**: the catalog kept in IndexedDB (`webapp.query-cache`) is stored as
|
||||
compressed bytes instead of readable JSON, about 300 KB instead of 2 MB.
|
||||
- **Readable JSON for the old apps only**: `/data/index.json`, `game_info_NNN.json`,
|
||||
`count_history.json`, `deleted_games.json` and `urls.json` are answered by the Worker, which passes
|
||||
them to the desktop/Android apps 4.0–4.1.1 (recognised by their `Origin`) and gives everyone else a
|
||||
410 with links to the repository ([`webapp/worker/legacy-data.ts`](../../webapp/worker/legacy-data.ts)).
|
||||
- **Policies** (2026-10-03): programmatic access to the data goes through the repository; the
|
||||
compressed cache and the scrollbar setting are listed.
|
||||
- Dependencies: `fflate` added (only loaded where the browser can't decompress by itself: Safari and
|
||||
iOS before 16.4); `@tanstack/query-async-storage-persister` removed (replaced by a small binary
|
||||
persister, [`webapp/src/lib/data/persister.ts`](../../webapp/src/lib/data/persister.ts)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Cover images** (`/img`): other spellings of a width (`/img/0160/…`, `160.0`) no longer create
|
||||
duplicate cache entries; an original served while resizing was unavailable, and a missing image,
|
||||
are now kept at the edge for a day instead of being fetched again on every request; failed
|
||||
background writes to R2 or the edge cache are logged.
|
||||
- **Chart cards** with a list (recently removed, tag cloud): their scrollbar can be dragged and their
|
||||
text selected again; clicks on the charts themselves still don't move focus.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Changelog — 4.3.x
|
||||
|
||||
Release notes for 4.3.0. Every version: [CHANGELOG.md](../../CHANGELOG.md).
|
||||
|
||||
## [4.3.0] - 2026-10-10 (AI disclosure, accessibility, refreshed catalog)
|
||||
|
||||
Version 4.3 reads the rows itch.io has added to game pages (the creator's generative-AI disclosure and
|
||||
the accessibility features) into the catalog, its filters and its charts, uses the creator's own
|
||||
tagline as the description, and re-scrapes the whole catalog once.
|
||||
|
||||
### Upgrade notes
|
||||
|
||||
- **Nothing to accept**: the policies have not changed.
|
||||
- **The catalog is re-scraped after the release**, in a few batches over a few hours. Until a game has
|
||||
been checked again, its AI disclosure shows as "Not disclosed" and its accessibility list is empty.
|
||||
- **Reusing the data?** Every record gains three optional fields (`ai_disclosure`, `ai_content`,
|
||||
`accessibility`), and `description` now holds the creator's tagline when there is one. The 20
|
||||
existing fields keep their names and types, and apps 4.0–4.2 ignore the new ones.
|
||||
|
||||
### Added
|
||||
|
||||
- **Generative AI**: what the creator disclosed on itch.io. "No AI used" means the page's Content row
|
||||
says "No generative AI was used". "AI-assisted" comes with what the AI was used for (Graphics, Text,
|
||||
Code, Sounds). Otherwise it is "Not disclosed". It appears on the game page, as a filter on Games
|
||||
(`?ai=…`) and as a chart (Charts → Overview); a second chart shows what AI-assisted games used AI
|
||||
for (Charts → Discovery).
|
||||
- **Accessibility**: the features itch.io lists, such as Subtitles, Configurable controls, Color-blind
|
||||
friendly, High-contrast, One button and Interactive tutorial. They appear on the game page, as a
|
||||
filter (`?access=…`) and as a chart (Charts → Discovery).
|
||||
|
||||
### Changed
|
||||
|
||||
- **Descriptions are the creator's tagline** (itch.io's "short description or tagline", set for about
|
||||
87 % of games). Games without one keep the first sentence of their page's description, as before
|
||||
(200 characters at most). 18+ suggestions for newly added games still look at that first sentence
|
||||
too.
|
||||
- **The whole catalog is re-scraped once** (`force_update.yml`). Besides the new fields, this picks up
|
||||
the tags, platforms, engines, covers, names, authors and publishers that changed on itch.io since
|
||||
the games were added.
|
||||
- **The daily refresh keeps the AI disclosure and accessibility current**, as it already does for the
|
||||
status and rating.
|
||||
- **Re-scrapes keep a known release date and cover.** itch.io shows the "Published" row only for a few
|
||||
weeks after a game comes out, so a later re-scrape no longer replaces a known date with N/A.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Text from pages that break lines inside a title or link** is read with single spaces: a tag such
|
||||
as "Meaningful Choices" no longer carries an embedded line break.
|
||||
@@ -0,0 +1,112 @@
|
||||
# Changelog — 4.4.x
|
||||
|
||||
Release notes for 4.4.0 and 4.4.1, newest first. Every version: [CHANGELOG.md](../../CHANGELOG.md).
|
||||
|
||||
## [4.4.1] - 2026-10-16 (Extension and bot retired)
|
||||
|
||||
The maintainer's browser extension and the old Telegram bot were hardly used, and both repositories
|
||||
are now public archives (read-only):
|
||||
[itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) and
|
||||
[telegram-scraper-bot](https://github.com/poli0981/telegram-scraper-bot). This version removes what was
|
||||
left of them here. Games still come in through the Suggest page, itch.io's RSS feeds and the itch.io
|
||||
collections the maintainer follows.
|
||||
|
||||
### Upgrade notes
|
||||
|
||||
- **Nothing to accept again.** The Privacy Policy, Terms of Use, EULA, Disclaimer and Security Policy
|
||||
(EN + VI) only lose their mentions of the extension and its API; nothing new is collected, so the
|
||||
legal check does not ask again.
|
||||
- **Desktop and Android apps**: install 4.4.1 over 4.4.0 as usual.
|
||||
- **Running your own copy of the Worker?** The `RL_INGEST` rate limit and the `ACCESS_AUD_INGEST`
|
||||
variable are gone from `wrangler.jsonc`; delete the Access application that protected
|
||||
`/api/ingest` and its service token. No D1 migration: the `idempotency` table stays and is no longer
|
||||
used.
|
||||
|
||||
### Removed
|
||||
|
||||
- **The ingest API** (`POST /api/ingest`), which the browser extension used to send links to the review
|
||||
queue with an Access service token. It never received a game since 4.0; the address now answers 404
|
||||
like any unknown API path.
|
||||
- **Admin:** the "Browser extension" choice in the auto-approve settings and the "Extension" source in
|
||||
the review queue.
|
||||
- **About page:** the old note asking to send a Telegram numeric ID privately (that was for the bot).
|
||||
The Telegram link stays as a way to reach the maintainer.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Policies** (2026-10-16, EN + VI): the Privacy Policy no longer lists the extension as a source of
|
||||
games or its duplicate-submission records (7 days); the Terms of Use, the EULA (also in the
|
||||
installers) and the Security Policy no longer name the ingest API; the Disclaimer lists the three
|
||||
remaining ways games arrive. The extension and the bot are out of the Security Policy's scope.
|
||||
- **Docs**: the READMEs, contributing guide, notices, acknowledgements and operations guide describe
|
||||
three ways games get in, and mark the extension and the bot as archived.
|
||||
- **Dependencies**: Vite 8.3.4 (#131); Tauri 2.12.2 with `tauri-build` 2.7.2, `tauri-plugin-opener`
|
||||
2.7.1 and `tauri-plugin-single-instance` 2.5.3 (#135); `serde_json` 1.0.154 (#136); Ruff 0.17 for
|
||||
the pipeline's lint (#137).
|
||||
|
||||
## [4.4.0] - 2026-10-13 (Admin automation, bulk actions, smoother charts)
|
||||
|
||||
Version 4.4 lets the maintainer add whole lists of games, follow itch.io collections and approve the
|
||||
review queue by rule, remove or edit up to 100 games at once, and look back at everything the admin
|
||||
did. On the site, the theme button shows the theme in use again, dark mode no longer flashes white
|
||||
while a page loads, and the Charts tabs switch without rebuilding their charts.
|
||||
|
||||
### Upgrade notes
|
||||
|
||||
- **The policies changed: accept them once more.** The Terms of Use (§5), the Privacy Policy (§2.3,
|
||||
§2.4) and the Disclaimer (§4) now say that the maintainer approves games one by one **or through
|
||||
rules the maintainer sets**: a suggestion or a discovered game without a warning flag that nobody
|
||||
rejected within a waiting time (about a day), and new free games in public itch.io collections the
|
||||
maintainer follows. The website and the 4.4 apps ask you to accept the updated policies.
|
||||
- **Running your own copy of the Worker?** Apply the D1 migration `0002_automation.sql`
|
||||
(`npx wrangler d1 migrations apply <database> --remote`) before deploying.
|
||||
- **Reusing the data?** The catalog files are unchanged. The pipeline keeps one more bookkeeping file,
|
||||
`scripts/state/ingest_outcomes.json` (how each queued link ended); the website does not publish it.
|
||||
|
||||
### Added
|
||||
|
||||
- **Admin: add games from any list.** Paste text (plain links, Markdown, CSV, HTML or JSON) or load a
|
||||
file: every itch.io game link in it is found, de-duplicated and normalized. **Check** shows what Add
|
||||
would do with each one (new, waiting in the queue, already in the catalog, removed or rejected
|
||||
before, not a game) before anything is written; **Add** sends lists of any length in batches of 200.
|
||||
- **Admin: import an itch.io collection.** Paste a collection link: the admin reads all its pages
|
||||
(games with a price are skipped) and passes them through the same Check and Add.
|
||||
- **Admin: follow collections.** Every 4 hours the site reads the collections the maintainer follows
|
||||
(up to 30 pages a run, one a second) and queues their new free games directly. A game removed before,
|
||||
or one whose title raises a flag (18+, demo), goes to the review queue instead.
|
||||
- **Admin: auto-approve.** Pending suggestions and RSS discoveries without a flag that nobody rejected
|
||||
within 24 hours are approved automatically (at most 100 a run, and only items found after the rule
|
||||
was switched on). The sources and the waiting time are set in the admin's Settings.
|
||||
- **Admin: follow each game's ingest.** Batches queued from the admin show, game by game, whether the
|
||||
ingest added it, is still trying, or dropped it and why (paid, page gone, password-protected, failed
|
||||
three times). The pipeline now records these outcomes.
|
||||
- **Admin: remove or edit up to 100 games at once.** Select games (or paste their links under "Remove
|
||||
by list") and remove them with one public reason, or set their 18+ flag, virus check and notes, in
|
||||
one commit. The preset reasons have translated labels on the Removed page.
|
||||
- **Admin: New games, Activity, Settings.** The games added in the last 7 or 30 days, to review their
|
||||
18+ flag and virus check in bulk; every admin action and automatic run with its commit; the
|
||||
automatic rules, the followed collections and what the last automatic run did.
|
||||
|
||||
### Changed
|
||||
|
||||
- **The theme button shows the theme in use** (sun, moon or screen) instead of the same icon for every
|
||||
choice. The admin follows the theme chosen on the site, and a change in one tab applies to the others.
|
||||
- **Charts: switching tabs is instant.** A tab shown once stays as it is: switching back no longer
|
||||
rebuilds its charts or replays their entrance. Entrances are shorter (300 ms, no delay), the other
|
||||
tabs load in the background, and the placeholders have the height of the real cards.
|
||||
- **Pages no longer shift sideways** when they grow taller or shorter than the window: on computers the
|
||||
scrollbar's room stays reserved.
|
||||
- **The browser's toolbar colour** follows the theme chosen on the site, not only the system's.
|
||||
- **Charts on phones** (#130, live on the website since 2026-10-11, now in the apps too): one row of
|
||||
four equal tabs, smaller legends and ticks, horizontal bars for Languages and Average session, the
|
||||
reason under each recent removal, and the total spanning the row. At every size: all 15 genre labels,
|
||||
pie legends in data order with readable text, translated removal reasons and locale numbers on the
|
||||
axes.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Dark mode no longer flashes white** while a page loads.
|
||||
- **Admin:** "Select all" on a long review queue no longer fails (selections go out in batches of 200);
|
||||
removing a game no longer reports an error when only the cleanup of its resized covers failed; every
|
||||
error has a readable message; confirmations are in-app dialogs.
|
||||
- **Queue commits** name the number of games really queued.
|
||||
@@ -1,6 +1,6 @@
|
||||
# Catalog data license
|
||||
|
||||
Last updated: 2026-09-18
|
||||
Last updated: 2026-10-03
|
||||
|
||||
Applies from: the release of version 4.0.0.
|
||||
|
||||
@@ -18,8 +18,8 @@ Applies from: the release of version 4.0.0.
|
||||
`index.json` (the chunk manifest) and `count_history.json` (the games-over-time series);
|
||||
- [`scripts/deleted_games.json`](../scripts/deleted_games.json), the public log of removed games and the
|
||||
reason each one was removed;
|
||||
- the files built from them and published under <https://freeitchgames.win/data> (for example
|
||||
`https://freeitchgames.win/data/index.json`).
|
||||
- the files built from them and published under <https://freeitchgames.win/data> (the compact packs the
|
||||
website and the apps read).
|
||||
|
||||
"The Maintainer" means poli0981 (SkullMute), the author of this collection and the licensor.
|
||||
|
||||
@@ -94,10 +94,11 @@ to the game's itch.io page.
|
||||
|
||||
## 5. Where to get the Data
|
||||
|
||||
- **The published JSON**: <https://freeitchgames.win/data> (`index.json` lists the `game_info_NNN.json`
|
||||
chunks).
|
||||
- **This repository**: clone or download [`data_game/`](./) and
|
||||
[`scripts/deleted_games.json`](../scripts/deleted_games.json). Git history holds past versions.
|
||||
- **This repository**: clone or download [`data_game/`](./) (`index.json` lists the `game_info_NNN.json`
|
||||
chunks) and [`scripts/deleted_games.json`](../scripts/deleted_games.json), or read the raw files. Git
|
||||
history holds past versions.
|
||||
- The files under <https://freeitchgames.win/data> are the website's own compact format, not an interface
|
||||
for reuse (the readable JSON there is served only to older versions of the apps).
|
||||
|
||||
Please use these files rather than scraping the website page by page. The [Terms of Use](../docs/ToS.md)
|
||||
do not allow automated bulk scraping of the website, and the files are cheaper for everyone.
|
||||
|
||||
Generated
+84
@@ -370,5 +370,89 @@
|
||||
{
|
||||
"date": "2026-09-28",
|
||||
"total": 2952
|
||||
},
|
||||
{
|
||||
"date": "2026-09-30",
|
||||
"total": 3016
|
||||
},
|
||||
{
|
||||
"date": "2026-10-01",
|
||||
"total": 3039
|
||||
},
|
||||
{
|
||||
"date": "2026-10-02",
|
||||
"total": 3038
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"total": 3101
|
||||
},
|
||||
{
|
||||
"date": "2026-10-05",
|
||||
"total": 3140
|
||||
},
|
||||
{
|
||||
"date": "2026-10-06",
|
||||
"total": 3149
|
||||
},
|
||||
{
|
||||
"date": "2026-10-07",
|
||||
"total": 3171
|
||||
},
|
||||
{
|
||||
"date": "2026-10-08",
|
||||
"total": 3194
|
||||
},
|
||||
{
|
||||
"date": "2026-10-09",
|
||||
"total": 3230
|
||||
},
|
||||
{
|
||||
"date": "2026-10-10",
|
||||
"total": 3262
|
||||
},
|
||||
{
|
||||
"date": "2026-10-11",
|
||||
"total": 3254
|
||||
},
|
||||
{
|
||||
"date": "2026-10-12",
|
||||
"total": 3276
|
||||
},
|
||||
{
|
||||
"date": "2026-10-14",
|
||||
"total": 3358
|
||||
},
|
||||
{
|
||||
"date": "2026-10-15",
|
||||
"total": 3369
|
||||
},
|
||||
{
|
||||
"date": "2026-10-16",
|
||||
"total": 3406
|
||||
},
|
||||
{
|
||||
"date": "2026-10-17",
|
||||
"total": 3415
|
||||
},
|
||||
{
|
||||
"date": "2026-10-18",
|
||||
"total": 3430
|
||||
},
|
||||
{
|
||||
"date": "2026-10-19",
|
||||
"total": 3451
|
||||
},
|
||||
{
|
||||
"date": "2026-10-20",
|
||||
"total": 3483
|
||||
},
|
||||
{
|
||||
"date": "2026-10-21",
|
||||
"total": 3507
|
||||
},
|
||||
{
|
||||
"date": "2026-10-22",
|
||||
"total": 3509
|
||||
}
|
||||
]
|
||||
Generated
+3566
-1288
File diff suppressed because it is too large.
Load diff
Generated
+4111
-1855
File diff suppressed because it is too large.
Load diff
Generated
+4002
-2297
File diff suppressed because it is too large.
Load diff
Generated
+4645
-2125
File diff suppressed because it is too large.
Load diff
Generated
+4663
-2747
File diff suppressed because it is too large.
Load diff
Generated
+6188
-3430
File diff suppressed because it is too large.
Load diff
Generated
+20536
File diff suppressed because it is too large.
Load diff
Generated
+378
@@ -0,0 +1,378 @@
|
||||
[
|
||||
{
|
||||
"url": "https://yuhaolin2005.itch.io/qingxuan-volume1",
|
||||
"name": "Qingxuan: The Alchemy Hall Cold Case",
|
||||
"dev": "YuhaoLin2005",
|
||||
"description": "A complete story-driven cultivation idle game. Twelve chapters, eight endings. Volume 1 public test.",
|
||||
"genre": "Role Playing",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "20 October 2026 @ 13:25 UTC",
|
||||
"rating": "N/A",
|
||||
"rating_count": "N/A",
|
||||
"average_session": "N/A",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMxMTI0NzYyLnBuZw==/original/m88d8C.png",
|
||||
"tags": [
|
||||
"cultivation",
|
||||
"Fantasy",
|
||||
"Idle",
|
||||
"Incremental",
|
||||
"Mystery",
|
||||
"Singleplayer",
|
||||
"Story Rich",
|
||||
"Text based",
|
||||
"xianxia"
|
||||
],
|
||||
"platforms": [
|
||||
"HTML5"
|
||||
],
|
||||
"languages": [],
|
||||
"inputs": [],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "AI Assisted",
|
||||
"ai_content": [
|
||||
"Code",
|
||||
"Graphics",
|
||||
"Text"
|
||||
],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"added_at": "2026-10-21T08:24:10Z"
|
||||
},
|
||||
{
|
||||
"url": "https://cineeeekk.itch.io/castle-builddefend",
|
||||
"name": "Castle: Build&Defend",
|
||||
"dev": "Cineeeekk",
|
||||
"description": "Build by day. Hold until dawn.",
|
||||
"genre": "Strategy",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "14 October 2026 @ 17:36 UTC",
|
||||
"rating": "4.8",
|
||||
"rating_count": "5",
|
||||
"average_session": "A few minutes",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMwOTE4MDI3LmpwZw==/original/NJs7zi.jpg",
|
||||
"tags": [
|
||||
"3D",
|
||||
"base-building",
|
||||
"building",
|
||||
"castle",
|
||||
"defense",
|
||||
"Isometric",
|
||||
"Medieval",
|
||||
"Tower Defense"
|
||||
],
|
||||
"platforms": [
|
||||
"HTML5"
|
||||
],
|
||||
"languages": [
|
||||
"German",
|
||||
"English",
|
||||
"French",
|
||||
"Polish",
|
||||
"Chinese (Simplified)"
|
||||
],
|
||||
"inputs": [
|
||||
"Keyboard",
|
||||
"Mouse",
|
||||
"Touchscreen"
|
||||
],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "AI Assisted",
|
||||
"ai_content": [
|
||||
"Code",
|
||||
"Graphics",
|
||||
"Sounds",
|
||||
"Text"
|
||||
],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"updated_at": "18 October 2026 @ 20:39 UTC",
|
||||
"added_at": "2026-10-21T12:23:36Z"
|
||||
},
|
||||
{
|
||||
"url": "https://cosmicadventuresquad.itch.io/project-hail-meowy",
|
||||
"name": "Project Hail Meowy",
|
||||
"dev": "Cosmic Adventure Squad",
|
||||
"description": "Punch the card. Save the tuna.",
|
||||
"genre": "Strategy",
|
||||
"status": "Prototype",
|
||||
"publisher": "N/A",
|
||||
"release_date": "19 October 2026 @ 15:20 UTC",
|
||||
"rating": "5.0",
|
||||
"rating_count": "3",
|
||||
"average_session": "About a half-hour",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMxMDk2MzkwLmdpZg==/original/u1oxzb.gif",
|
||||
"tags": [
|
||||
"ascii",
|
||||
"Cats",
|
||||
"Ludum Dare 60",
|
||||
"Mouse only",
|
||||
"Retro",
|
||||
"Roguelike",
|
||||
"Space Sim",
|
||||
"Turn-based"
|
||||
],
|
||||
"platforms": [
|
||||
"HTML5",
|
||||
"Windows"
|
||||
],
|
||||
"languages": [
|
||||
"English"
|
||||
],
|
||||
"inputs": [
|
||||
"Keyboard",
|
||||
"Mouse",
|
||||
"Touchscreen"
|
||||
],
|
||||
"made_with": [
|
||||
"Godot"
|
||||
],
|
||||
"ai_disclosure": "AI Assisted",
|
||||
"ai_content": [
|
||||
"Code",
|
||||
"Graphics",
|
||||
"Text"
|
||||
],
|
||||
"accessibility": [
|
||||
"Subtitles"
|
||||
],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"added_at": "2026-10-21T12:23:43Z"
|
||||
},
|
||||
{
|
||||
"url": "https://doctorhummer.itch.io/irule",
|
||||
"name": "I.RULE",
|
||||
"dev": "DoctorHummer, toncho_",
|
||||
"description": "The Binding of Isaac fangame, heavily inspired by Plants vs. Zombies",
|
||||
"genre": "Strategy",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "N/A",
|
||||
"rating": "4.9",
|
||||
"rating_count": "500",
|
||||
"average_session": "N/A",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzgwNDQwODUucG5n/original/H%2Bqt0z.png",
|
||||
"tags": [
|
||||
"Fangame",
|
||||
"Roguelike",
|
||||
"Roguelite",
|
||||
"Tower Defense"
|
||||
],
|
||||
"platforms": [
|
||||
"Windows",
|
||||
"Android"
|
||||
],
|
||||
"languages": [],
|
||||
"inputs": [],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "N/A",
|
||||
"ai_content": [],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"added_at": "2026-10-21T12:23:49Z"
|
||||
},
|
||||
{
|
||||
"url": "https://pickpawket.itch.io/a-trick-of-his-light",
|
||||
"name": "A Trick of His Light",
|
||||
"dev": "pickpawket",
|
||||
"description": "For EGDC's \"Trick\" game jam :) Artwork by L_T Written by lilaofthelake Programmed by pickpawket",
|
||||
"genre": "Visual Novel",
|
||||
"status": "Prototype",
|
||||
"publisher": "N/A",
|
||||
"release_date": "20 October 2026 @ 20:56 UTC",
|
||||
"rating": "N/A",
|
||||
"rating_count": "N/A",
|
||||
"average_session": "N/A",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMxMTI4MDAzLnBuZw==/original/btWi2q.png",
|
||||
"tags": [],
|
||||
"platforms": [
|
||||
"Windows",
|
||||
"macOS",
|
||||
"Linux"
|
||||
],
|
||||
"languages": [],
|
||||
"inputs": [],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "No AI",
|
||||
"ai_content": [],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"added_at": "2026-10-21T16:23:38Z"
|
||||
},
|
||||
{
|
||||
"url": "https://sakiness.itch.io/everlune",
|
||||
"name": "EverLune BETA",
|
||||
"dev": "SakiNess",
|
||||
"description": "A romance visual novel where everything goes wrong... the moment it pulls you into the game.",
|
||||
"genre": "Visual Novel",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "21 October 2026 @ 03:58 UTC",
|
||||
"rating": "5.0",
|
||||
"rating_count": "1",
|
||||
"average_session": "About a half-hour",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMxMTM4MDg5LnBuZw==/original/mksM5w.png",
|
||||
"tags": [
|
||||
"2D",
|
||||
"Anime",
|
||||
"Destruction",
|
||||
"Horror",
|
||||
"No AI",
|
||||
"Psychological Horror",
|
||||
"Ren'Py",
|
||||
"Romance",
|
||||
"somethings-wrong"
|
||||
],
|
||||
"platforms": [
|
||||
"Windows",
|
||||
"macOS",
|
||||
"Android"
|
||||
],
|
||||
"languages": [],
|
||||
"inputs": [
|
||||
"Keyboard",
|
||||
"Mouse",
|
||||
"Touchscreen"
|
||||
],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "No AI",
|
||||
"ai_content": [],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"added_at": "2026-10-21T16:23:41Z"
|
||||
},
|
||||
{
|
||||
"url": "https://vhan-5.itch.io/u-dont-get-it",
|
||||
"name": "U DON'T GET IT",
|
||||
"dev": "Vhan || 반",
|
||||
"description": "\"I knew you wouldn't get it... or did you?\" A dark psychological visual novel about obsession, expectations, and uncovered truths....",
|
||||
"genre": "Visual Novel",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "20 October 2026 @ 21:36 UTC",
|
||||
"rating": "5.0",
|
||||
"rating_count": "1",
|
||||
"average_session": "N/A",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMxMTQwNjIxLnBuZw==/original/WplE0T.png",
|
||||
"tags": [
|
||||
"Creepy",
|
||||
"Indie",
|
||||
"kuudere",
|
||||
"LGBTQIA",
|
||||
"Psychological Horror",
|
||||
"Ren'Py",
|
||||
"Romance",
|
||||
"Story Rich",
|
||||
"Yandere"
|
||||
],
|
||||
"platforms": [
|
||||
"Android"
|
||||
],
|
||||
"languages": [],
|
||||
"inputs": [],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "No AI",
|
||||
"ai_content": [],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"added_at": "2026-10-21T16:23:44Z"
|
||||
},
|
||||
{
|
||||
"url": "https://alchemysheep.itch.io/game-is-full-of-bugs",
|
||||
"name": "Game is Full of Bugs",
|
||||
"dev": "AlchemySheep",
|
||||
"description": "Demo",
|
||||
"genre": "Action",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "08 October 2026 @ 14:00 UTC",
|
||||
"rating": "N/A",
|
||||
"rating_count": "N/A",
|
||||
"average_session": "N/A",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMwMDcxNzM3LnBuZw==/original/aqgn1X.png",
|
||||
"tags": [
|
||||
"2D",
|
||||
"bullet-heaven",
|
||||
"Dungeon Crawler",
|
||||
"Indie",
|
||||
"Pixel Art",
|
||||
"Roguelike",
|
||||
"Roguelite",
|
||||
"Singleplayer",
|
||||
"Top-Down",
|
||||
"Twin Stick Shooter"
|
||||
],
|
||||
"platforms": [],
|
||||
"languages": [],
|
||||
"inputs": [],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "No AI",
|
||||
"ai_content": [],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"updated_at": "19 October 2026 @ 14:22 UTC",
|
||||
"added_at": "2026-10-22T00:23:31Z"
|
||||
},
|
||||
{
|
||||
"url": "https://traumalab.itch.io/trauma-lab",
|
||||
"name": "Trauma lab",
|
||||
"dev": "TraumaLab",
|
||||
"description": "An anatomy and physics sandbox. Every bullet is tracked through skin, muscle, bone and organ.",
|
||||
"genre": "Simulation",
|
||||
"status": "In development",
|
||||
"publisher": "N/A",
|
||||
"release_date": "20 October 2026 @ 15:06 UTC",
|
||||
"rating": "4.7",
|
||||
"rating_count": "6",
|
||||
"average_session": "N/A",
|
||||
"nsfw": "No",
|
||||
"thumbnail": "https://img.itch.zone/aW1nLzMxMTI5MDA2LnBuZw==/original/wLPm93.png",
|
||||
"tags": [
|
||||
"anatomy",
|
||||
"browser",
|
||||
"Gore",
|
||||
"medical",
|
||||
"Physics",
|
||||
"ragdoll",
|
||||
"Sandbox",
|
||||
"Singleplayer",
|
||||
"Violent"
|
||||
],
|
||||
"platforms": [
|
||||
"HTML5",
|
||||
"Windows",
|
||||
"Android"
|
||||
],
|
||||
"languages": [],
|
||||
"inputs": [],
|
||||
"made_with": [],
|
||||
"ai_disclosure": "AI Assisted",
|
||||
"ai_content": [
|
||||
"Code",
|
||||
"Graphics",
|
||||
"Sounds"
|
||||
],
|
||||
"accessibility": [],
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
"updated_at": "21 October 2026 @ 22:32 UTC",
|
||||
"added_at": "2026-10-22T00:23:35Z"
|
||||
}
|
||||
]
|
||||
Generated
+16
-8
@@ -1,31 +1,39 @@
|
||||
{
|
||||
"total_games": 2952,
|
||||
"total_games": 3509,
|
||||
"max_per_file": 500,
|
||||
"last_updated": "2026-09-29T03:26:31Z",
|
||||
"last_updated": "2026-10-22T00:23:36Z",
|
||||
"files": [
|
||||
{
|
||||
"name": "game_info_001.json",
|
||||
"count": 490
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_002.json",
|
||||
"count": 491
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_003.json",
|
||||
"count": 497
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_004.json",
|
||||
"count": 488
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_005.json",
|
||||
"count": 495
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_006.json",
|
||||
"count": 491
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_007.json",
|
||||
"count": 500
|
||||
},
|
||||
{
|
||||
"name": "game_info_008.json",
|
||||
"count": 9
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -19,7 +19,7 @@ This repo wouldn't exist without a lot of help — mostly from tools and platfor
|
||||
|
||||
The website, the Worker and the desktop / Android apps build on a stack of open-source libraries. The full list with licenses lives in [`THIRD_PARTY.md`](THIRD_PARTY.md), and the runtime libraries are also on the [About page](https://freeitchgames.win/about) (from the `THIRD_PARTY` array in [`webapp/src/lib/about.ts`](../webapp/src/lib/about.ts)). Highlights:
|
||||
|
||||
- **Core**: React 19, TypeScript 6, Vite 8, React Router 8, Zustand 5
|
||||
- **Core**: React 19, TypeScript 7, Vite 8, React Router 8, Zustand 5
|
||||
- **UI**: Tailwind CSS, tw-animate-css, shadcn/ui (pattern), Radix UI primitives, lucide-react, class-variance-authority, tailwind-merge, clsx, Recharts, the Geist and Geist Mono typefaces (via Fontsource)
|
||||
- **Data**: TanStack Query, idb-keyval
|
||||
- **Worker**: jose, Wrangler
|
||||
@@ -27,9 +27,9 @@ The website, the Worker and the desktop / Android apps build on a stack of open-
|
||||
|
||||
When you add a new dep to `webapp/package.json`, also add it to the `THIRD_PARTY` array so the About page lists it, and to [`THIRD_PARTY.md`](THIRD_PARTY.md). House rules are in [CONTRIBUTING.md](../CONTRIBUTING.md).
|
||||
|
||||
## Browser extension — the other way in
|
||||
## Retired helpers
|
||||
|
||||
[poli0981/itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) (GPL-3.0, separate repo) lets me queue itch.io games straight from the browser. It sends them through an authenticated API into the same review queue as the public [Suggest page](https://freeitchgames.win/suggest) and the RSS discovery. Nothing goes live until I approve it. (The old Telegram bot did this job in v3 and is now retired.)
|
||||
Two side projects used to feed the catalog: the browser extension [poli0981/itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) (GPL-3.0) and, in v3, the Telegram bot [poli0981/telegram-scraper-bot](https://github.com/poli0981/telegram-scraper-bot) (MIT). Both were hardly used, so they are archived now (October 2026). Games come in through the public [Suggest page](https://freeitchgames.win/suggest), the RSS discovery and the itch.io collections I follow, and nothing goes live until it is approved.
|
||||
|
||||
## Find me / Support me
|
||||
Real channels exist now (still introvert max level, replies slow):
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Disclaimer
|
||||
|
||||
Last updated: 2026-09-29
|
||||
Last updated: 2026-10-16
|
||||
|
||||
Applies from: the release of version 4.0.0.
|
||||
|
||||
@@ -31,7 +31,7 @@ The Project is independent. It is **not affiliated with, endorsed by, or sponsor
|
||||
|
||||
The Project **does not make, host, distribute, sell or endorse** the games it lists. Every game links to its own page on itch.io, and any download happens there, under itch.io's terms and the creator's own terms or license.
|
||||
|
||||
Games reach the catalog through the public Suggest page, the Maintainer's browser extension, or itch.io's public RSS feeds, and nothing is added until the Maintainer approves it. That approval is **not** a security audit, a quality check or an endorsement.
|
||||
Games reach the catalog through the public Suggest page, itch.io's public RSS feeds, or public itch.io collections the Maintainer follows. Each one is approved by the Maintainer, one by one or through automatic rules the Maintainer sets. That approval is **not** a security audit, a quality check or an endorsement.
|
||||
|
||||
## 5. No warranty as to the games
|
||||
|
||||
|
||||
+5
-5
@@ -1,8 +1,8 @@
|
||||
# End-User License Agreement (EULA)
|
||||
|
||||
Last updated: 2026-09-29
|
||||
Last updated: 2026-10-16
|
||||
|
||||
Applies from: the release of version 4.1.1.
|
||||
Applies from: the release of version 4.2.0.
|
||||
|
||||
This EULA applies **only to the desktop and Android apps** of Itch.io Free Games DB: the installers (`.msi`, `.exe`, `.dmg`, `.pkg`, `.app.tar.gz`, `.deb`, `.rpm`, `.AppImage`) and the Android `.apk` published on GitHub Releases. It does **not** cover:
|
||||
|
||||
@@ -46,12 +46,12 @@ Nothing in this EULA takes away a permission the MIT License gives You over the
|
||||
|
||||
The App has **no telemetry**: no analytics, no crash reporting, no tracking. It connects to only two places:
|
||||
|
||||
- **freeitchgames.win**, to download the Catalog JSON from <https://freeitchgames.win/data>. The Website runs on Cloudflare, which processes ordinary request data (such as Your IP address and user agent) to deliver it, as described in the [Privacy Policy](PrivacyPolicy.md).
|
||||
- **freeitchgames.win**, to download the Catalog data from <https://freeitchgames.win/data>. The Website runs on Cloudflare, which processes ordinary request data (such as Your IP address and user agent) to deliver it, as described in the [Privacy Policy](PrivacyPolicy.md).
|
||||
- **img.itch.zone**, itch.io's image server, to load cover images directly. itch.io receives ordinary request data for those images, as with any image loaded from the web (see the [itch.io privacy policy](https://itch.io/docs/legal/privacy-policy)).
|
||||
|
||||
When You click a game or another outside link, it opens in Your default browser (itch.io, GitHub or the Website). From then on, that site's own terms and privacy policy apply.
|
||||
|
||||
The App keeps Your preferences (language, theme, layout density, the 18+ choice, and which version of the legal terms You accepted) and a cached copy of the Catalog (kept up to 7 days) on Your device only. None of it is sent to the Maintainer.
|
||||
The App keeps Your preferences (language, theme, layout density, whether scrollbars are hidden, the 18+ choice, and which version of the legal terms You accepted) and a compressed cached copy of the Catalog (kept up to 7 days) on Your device only. None of it is sent to the Maintainer.
|
||||
|
||||
## 5. Installation, updates and removal
|
||||
|
||||
@@ -82,7 +82,7 @@ You agree not to:
|
||||
|
||||
- Present a modified build as an official release, for example by distributing it under the project's name and icon, or with the official app identifier `com.poli0981.freegamesitchio`, without clearly marking it as unofficial. Forks are welcome under the MIT License; just make it obvious they are Yours.
|
||||
- Use the Maintainer's name, the GitHub handle `poli0981`, or the alias "SkullMute" to endorse or promote derivative works without prior written permission.
|
||||
- Use the App, or a modified version of it, in a way that breaks the Website's [Terms of Use](ToS.md), for example automated bulk requests beyond the published `/data` files, or attempts to reach the admin area or the ingest API without authorization.
|
||||
- Use the App, or a modified version of it, in a way that breaks the Website's [Terms of Use](ToS.md), for example automated bulk requests to the Website (the data is in the Repository), or attempts to reach the admin area without authorization.
|
||||
- Use the App or its installers to host, distribute or facilitate malware, phishing, or content that violates applicable law.
|
||||
|
||||
These rules are about how You use the App and the project's name and services. They sit alongside the MIT License and do not limit the rights it grants over the source code.
|
||||
|
||||
+12
-13
@@ -1,8 +1,8 @@
|
||||
# Privacy Policy
|
||||
|
||||
Last updated: 2026-09-29
|
||||
Last updated: 2026-10-16
|
||||
|
||||
Applies from: 2026-09-29 on the Website, and from version 4.1.1 of the Apps.
|
||||
Applies from: 2026-10-13 on the Website, and from version 4.4.0 of the Apps.
|
||||
|
||||
This Privacy Policy explains what personal data the `free-games-itchio-list` project processes, why, who helps process it, how long it is kept, and what rights you have. It covers the website **https://freeitchgames.win**, the desktop and Android apps, the public catalog data and the repository. The short version: there are no accounts, no ads and no tracking cookies. The Project keeps as little as it can, and most of what exists stays in your own browser.
|
||||
|
||||
@@ -36,7 +36,7 @@ The controller of the personal data described in this Policy is **the Maintainer
|
||||
|
||||
### 1.3 What this Policy does not cover
|
||||
|
||||
This Policy does not cover services that handle data on their own account: itch.io (every game link and every download goes there), GitHub for your GitHub account and your activity on github.com, Cloudflare's use of data for its own purposes as described in its privacy policy, the games themselves, or the Maintainer's browser extension, which is a separate project with its own documentation. Their own policies apply (links in section 4).
|
||||
This Policy does not cover services that handle data on their own account: itch.io (every game link and every download goes there), GitHub for your GitHub account and your activity on github.com, Cloudflare's use of data for its own purposes as described in its privacy policy, or the games themselves. Their own policies apply (links in section 4).
|
||||
|
||||
## 2. What is processed and why
|
||||
|
||||
@@ -53,7 +53,7 @@ This Policy does not cover services that handle data on their own account: itch.
|
||||
### 2.2 Using the Apps
|
||||
|
||||
- The Apps are read-only viewers with **no telemetry, no analytics and no accounts**.
|
||||
- They download the Catalog JSON from https://freeitchgames.win/data (Cloudflare processes the request data as described in section 2.1) and load cover images **directly from itch.io's image server (img.itch.zone)**, so itch.io receives your IP address and user agent when covers load.
|
||||
- They download the Catalog data from https://freeitchgames.win/data (Cloudflare processes the request data as described in section 2.1; versions 4.0–4.1.1 still get the older JSON files, recognised by the `Origin` header their window sends) and load cover images **directly from itch.io's image server (img.itch.zone)**, so itch.io receives your IP address and user agent when covers load.
|
||||
- Clicking a game opens its itch.io page in your browser.
|
||||
- Updates are manual: the Apps do not check for updates on their own. When you download an installer or the APK from GitHub Releases, GitHub processes that request.
|
||||
- The Apps keep the same local storage as the Website (section 3.1), on your device only. Uninstalling an App removes it.
|
||||
@@ -65,14 +65,14 @@ When you use the Suggest page (https://freeitchgames.win/suggest):
|
||||
- **What is stored**: the game URL, your optional note (up to 500 characters) and the time of submission, in the review queue (section 2.4). No name, email address or account is asked for or stored.
|
||||
- **Anti-spam**: Cloudflare Turnstile processes signals about your device and browser to tell people from bots. To check the result, the Website sends the Turnstile token and your IP address to Cloudflare's verification service.
|
||||
- **Rate limiting**: your IP address is used transiently to count requests. The Project does not store it.
|
||||
- **Purpose**: to review your suggestion. If it is approved, the game URL becomes part of the public Catalog. Your note is never published and is **deleted automatically after 180 days**.
|
||||
- **Purpose**: to review your suggestion, by the Maintainer or by the automatic rules the Maintainer sets (section 2.4). If it is approved, the game URL becomes part of the public Catalog. Your note is never published and is **deleted automatically after 180 days**.
|
||||
- **Please don't put personal data in the note**, yours or anyone else's. The form is for game links only.
|
||||
|
||||
### 2.4 The review queue and the admin area
|
||||
|
||||
- **How games arrive**: suggestions from the Suggest page; submissions from the Maintainer's browser extension through an authenticated API (the game URL, an optional title and note, and the extension's service-token ID; the records that prevent duplicate submissions are kept for 7 days); and automatic discovery from itch.io's public RSS feeds.
|
||||
- **The review queue** (Cloudflare D1) holds candidate game URLs with their source, the title, cover-image URL and genre hint taken from itch.io, automatic flags, notes, timestamps and the Maintainer's decision. Nothing enters the Catalog until the Maintainer approves it. Rejected URLs are kept indefinitely as a blocklist; only the URL is needed to avoid reviewing the same game again.
|
||||
- **The admin area** (https://freeitchgames.win/admin) is for the Maintainer only. It is protected by Cloudflare Access (GitHub login or a one-time PIN sent by email), which sets a `CF_Authorization` session cookie in the Maintainer's browser. Admin actions are recorded in an audit log together with the Maintainer's email address. The audit log is never published.
|
||||
- **How games arrive**: suggestions from the Suggest page; automatic discovery from itch.io's public RSS feeds; and new free games in public itch.io collections the Maintainer follows (the Website reads those public pages; nothing about you is involved).
|
||||
- **The review queue** (Cloudflare D1) holds candidate game URLs with their source, the title, cover-image URL and genre hint taken from itch.io, automatic flags, notes, timestamps and the Maintainer's decision. Games enter the Catalog when the Maintainer approves them: one by one, or through automatic rules the Maintainer sets and can change at any time (items without a warning flag that nobody rejected within a waiting time; new free games in collections the Maintainer follows). Automatic approvals are recorded in the audit log like the Maintainer's own. Rejected URLs are kept indefinitely as a blocklist; only the URL is needed to avoid reviewing the same game again.
|
||||
- **The admin area** (https://freeitchgames.win/admin) is for the Maintainer only. It is protected by Cloudflare Access (GitHub login or a one-time PIN sent by email), which sets a `CF_Authorization` session cookie in the Maintainer's browser. Admin actions, including changes to the automatic rules, are recorded in an audit log together with the Maintainer's email address. The audit log is never published.
|
||||
|
||||
### 2.5 Public game data in the Catalog
|
||||
|
||||
@@ -106,10 +106,10 @@ The Website and the Apps keep the following items on your device only. None of t
|
||||
|
||||
| Storage | Key | Contents | Kept until |
|
||||
|---|---|---|---|
|
||||
| `localStorage` | `webapp.prefs` | Interface preferences such as language and layout density, your 18+ (NSFW) content choice, and the version of the legal documents you accepted | You clear it |
|
||||
| `localStorage` | `webapp.prefs` | Interface preferences such as language, layout density and whether scrollbars are hidden, your 18+ (NSFW) content choice, and the version of the legal documents you accepted | You clear it |
|
||||
| `localStorage` | `webapp.theme` | Theme: `light`, `dark` or `system` | You clear it |
|
||||
| `localStorage` | `webapp.gate` | When your current verification pass (section 3.2) runs out, so an open tab can check again in time | Replaced by the next check; you can clear it |
|
||||
| IndexedDB (via `idb-keyval`) | `webapp.query-cache` | A cache of the public Catalog JSON, for fast loading and limited offline use | At most 7 days, then refreshed or discarded |
|
||||
| IndexedDB (via `idb-keyval`) | `webapp.query-cache` | A compressed copy of the public Catalog data (the game list, the removal log, the games-over-time series and the descriptions you opened or searched), for fast loading and limited offline use | At most 7 days, then refreshed or discarded |
|
||||
| `sessionStorage` | `reloaded-after-deploy` | A timestamp, so the page reloads at most once after a site update | The tab is closed |
|
||||
|
||||
On first load, the Website and the Apps also delete the entries that the old v3 app left in browser storage (an encrypted GitHub access token and commit-signing key data). v4 does not use them.
|
||||
@@ -145,7 +145,6 @@ Cloudflare and Google may process data anywhere on their global networks (see se
|
||||
| IP address used for rate limiting and Turnstile | Used transiently; never stored by the Project |
|
||||
| Suggest note | Deleted automatically after **180 days** |
|
||||
| Suggested or discovered game URLs, with source and timestamps | In the review queue while needed to avoid reviewing the same game twice. Approved URLs become part of the public Catalog; rejected URLs are kept indefinitely as a blocklist |
|
||||
| Duplicate-submission records from the browser extension | **7 days** |
|
||||
| Admin audit log (with the Maintainer's email address) | Kept as the record of admin changes; never published |
|
||||
| Catalog entries (including developer and publisher names) | While the game is listed. Removal records stay in the public removed-games list, and the Repository's Git history keeps past versions (see section 7.3) |
|
||||
| Emails, including removal and copyright requests | As long as needed to handle the request |
|
||||
@@ -219,8 +218,8 @@ Under the GDPR and the UK GDPR, you can also ask for processing to be restricted
|
||||
|
||||
- The Website is served only over HTTPS.
|
||||
- Automated traffic has to pass a Cloudflare Turnstile check before the Website's pages and cover images load (section 2.1).
|
||||
- Data minimization: no accounts, no IP addresses stored by the Project's code, Suggest notes deleted after 180 days, duplicate-submission records after 7 days.
|
||||
- The admin area and the ingest API are protected by Cloudflare Access, and the Website's server code verifies the Access token itself on every protected request.
|
||||
- Data minimization: no accounts, no IP addresses stored by the Project's code, Suggest notes deleted after 180 days.
|
||||
- The admin area is protected by Cloudflare Access, and the Website's server code verifies the Access token itself on every protected request.
|
||||
- Secrets (such as the GitHub App's private key) are kept in Cloudflare's secret storage, never in the Repository. The admin area writes to the Repository as a GitHub App, with verified commits.
|
||||
- A strict Content Security Policy limits scripts to the Website itself plus Cloudflare Web Analytics and Turnstile.
|
||||
- GitHub Actions are pinned to exact commit SHAs with least-privilege permissions, and Dependabot keeps dependencies up to date.
|
||||
|
||||
+11
-8
@@ -1,6 +1,6 @@
|
||||
# Third-party components
|
||||
|
||||
Last updated: 2026-09-29
|
||||
Last updated: 2026-10-03
|
||||
|
||||
Applies from: the release of version 4.0.0.
|
||||
|
||||
@@ -30,8 +30,9 @@ shared UI components).
|
||||
| [React](https://react.dev) (`react`, `react-dom`) | 19.3 | UI framework | MIT |
|
||||
| [React Router](https://reactrouter.com) (`react-router`) | 8.4 | Routing | MIT |
|
||||
| [Zustand](https://github.com/pmndrs/zustand) | 5.0 | Preference and theme stores | MIT |
|
||||
| [TanStack Query](https://tanstack.com/query) (`@tanstack/react-query`, `react-query-persist-client`, `query-async-storage-persister`) | 5.103 | Data fetching and the local catalog cache | MIT |
|
||||
| [idb-keyval](https://github.com/jakearchibald/idb-keyval) | 6.3 | IndexedDB storage for the catalog cache | Apache-2.0 |
|
||||
| [TanStack Query](https://tanstack.com/query) (`@tanstack/react-query`, `react-query-persist-client`) | 5.103 | Data fetching and the local catalog cache | MIT |
|
||||
| [idb-keyval](https://github.com/jakearchibald/idb-keyval) | 6.3 | IndexedDB storage for the compressed catalog cache | Apache-2.0 |
|
||||
| [fflate](https://github.com/101arrowz/fflate) | 0.8 | Decompressing the catalog where the browser has no Compression Streams (older Safari / iOS) | MIT |
|
||||
| [Recharts](https://recharts.org) | 3.10 | Charts | MIT |
|
||||
| [Radix UI](https://www.radix-ui.com) primitives (`@radix-ui/react-checkbox`, `-dialog`, `-label`, `-popover`, `-separator`, `-slot`, `-switch`, `-tabs`) | 1.x / 2.x | Accessible UI building blocks | MIT |
|
||||
| [shadcn/ui](https://ui.shadcn.com) | pattern | Component patterns copied into `webapp/src/components/ui/` | MIT |
|
||||
@@ -73,15 +74,16 @@ components. Section 7 lists every service the project uses.
|
||||
The apps render with the system web view that the operating system provides: WebView2 on Windows,
|
||||
WKWebView on macOS (Safari/WebKit 16.4 or later), WebKitGTK on Linux and Android System WebView on Android.
|
||||
|
||||
## 4. Data pipeline (Python 3.14)
|
||||
## 4. Data pipeline (Python 3.15)
|
||||
|
||||
| Package | Version | Used for | License |
|
||||
|---|---|---|---|
|
||||
| [requests](https://requests.readthedocs.io) | 2.34+ | HTTP requests to public itch.io pages | Apache-2.0 |
|
||||
| [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/) | 4.15+ | Parsing itch.io pages | MIT |
|
||||
| [urllib3](https://urllib3.readthedocs.io) | 2.8+ | Retry policy of the scraper's HTTP session | MIT |
|
||||
|
||||
These pull in `urllib3` (MIT), `certifi` (MPL-2.0), `charset-normalizer` (MIT), `idna` (BSD-3-Clause) and
|
||||
`soupsieve` (MIT).
|
||||
These pull in `certifi` (MPL-2.0), `charset-normalizer` (MIT), `idna` (BSD-3-Clause) and `soupsieve`
|
||||
(MIT).
|
||||
|
||||
## 5. Development and test tooling (never shipped)
|
||||
|
||||
@@ -103,8 +105,9 @@ These tools never end up in a build, but they keep the code honest.
|
||||
- **[ESLint](https://eslint.org)** with `@eslint/js`, **[typescript-eslint](https://typescript-eslint.io)**,
|
||||
`eslint-plugin-react-hooks`, `eslint-plugin-react-refresh` and `globals` (all MIT): webapp linting.
|
||||
Config: [`webapp/eslint.config.js`](../webapp/eslint.config.js). Run: `cd webapp && npm run lint`
|
||||
- **[TypeScript](https://www.typescriptlang.org)** 6.0 (Apache-2.0): `noUnusedLocals` and
|
||||
`noUnusedParameters` reject unused code at build time. Run: `cd webapp && npx tsc -b`
|
||||
- **[TypeScript](https://www.typescriptlang.org)** 7.0 (Apache-2.0): `noUnusedLocals` and
|
||||
`noUnusedParameters` reject unused code at build time. Run: `cd webapp && npx tsc -b`. The 6.0 API
|
||||
(`@typescript/typescript6`, installed as `typescript`) stays for typescript-eslint, which needs it
|
||||
|
||||
### Build, test and deploy
|
||||
|
||||
|
||||
+7
-7
@@ -1,12 +1,12 @@
|
||||
# Terms of Use
|
||||
|
||||
Last updated: 2026-09-29
|
||||
Last updated: 2026-10-16
|
||||
|
||||
Applies from: 2026-09-29 on the Website, and from version 4.1.1 of the Apps.
|
||||
Applies from: 2026-10-13 on the Website, and from version 4.4.0 of the Apps.
|
||||
|
||||
These Terms of Use ("Terms") govern your use of the website **https://freeitchgames.win**, the desktop and Android apps, and the public catalog data of `free-games-itchio-list`. They are written in plain language, but they are a real agreement. They replace the older Terms and also cover what the old web-app "EULA" used to say; the [EULA](EULA.md) now covers only the app installers and the Android APK.
|
||||
|
||||
> **TL;DR**: Browse, share, fork the code (MIT) and reuse the data with credit (CC BY 4.0). Don't bulk-scrape the site (use the `/data` files), don't hotlink `/img`, don't spam the Suggest form, don't poke at `/admin`, and don't pretend to be itch.io or the Maintainer. Game creators can get a game removed or corrected. Adult content stays hidden unless you opt in.
|
||||
> **TL;DR**: Browse, share, fork the code (MIT) and reuse the data with credit (CC BY 4.0). Don't bulk-scrape the site (the data is in the GitHub repo), don't hotlink `/img`, don't spam the Suggest form, don't poke at `/admin`, and don't pretend to be itch.io or the Maintainer. Game creators can get a game removed or corrected. Adult content stays hidden unless you opt in.
|
||||
|
||||
## 1. Definitions
|
||||
|
||||
@@ -37,7 +37,7 @@ These Terms of Use ("Terms") govern your use of the website **https://freeitchga
|
||||
|
||||
- Browse the Website and the Apps, and follow links to itch.io.
|
||||
- Share links to the Website and to game pages.
|
||||
- Download and reuse the Catalog data under **CC BY 4.0**, with credit (see section 11). For programmatic access, use the published JSON files under https://freeitchgames.win/data or the Repository; please cache them instead of downloading them again on every request.
|
||||
- Download and reuse the Catalog data under **CC BY 4.0**, with credit (see section 11). Get it from the Repository (`data_game/` and `scripts/deleted_games.json`, also as raw files), and please cache it instead of downloading it again on every request. The files under https://freeitchgames.win/data are the Website's and the Apps' own compact format, not a public interface; the readable JSON there is served only to older versions of the Apps.
|
||||
- Use, fork, modify and redistribute the code under the **MIT License**.
|
||||
- Suggest a free itch.io game on the Suggest page (see section 5).
|
||||
- Report bugs, give feedback and open pull requests through the Repository's templates.
|
||||
@@ -47,7 +47,7 @@ These Terms of Use ("Terms") govern your use of the website **https://freeitchga
|
||||
|
||||
- **Suggest page** (https://freeitchgames.win/suggest): you can submit an itch.io game link and an optional note of up to 500 characters. The page uses Cloudflare Turnstile against spam and is rate limited.
|
||||
- The note is seen only by the Maintainer and is deleted after 180 days. **Do not put personal data in it.** See the [Privacy Policy](PrivacyPolicy.md).
|
||||
- Every suggestion goes into a **review queue**. Nothing enters the Catalog until the Maintainer approves it, and the Maintainer has no obligation to accept a suggestion or to reply. Approved links are then scraped by the pipeline, and games that turn out to be paid are dropped.
|
||||
- Every suggestion goes into a **review queue**. The Maintainer approves or rejects it there, one by one or through automatic rules the Maintainer sets: a suggestion that raises no warning flag and that nobody rejected within the waiting time the Maintainer chose (currently about a day) may be approved automatically. The Maintainer has no obligation to accept a suggestion or to reply. Approved links are then scraped by the pipeline, and games that turn out to be paid are dropped.
|
||||
- Only genuinely **free-to-play** games on itch.io qualify. Demos of paid games, "name your own price" titles with a minimum above zero, and time-limited free promotions do not.
|
||||
- By submitting anything to the Project (a suggestion, an issue, a pull request or a comment), you confirm that:
|
||||
- it is your own work, or you have the right to share it;
|
||||
@@ -60,9 +60,9 @@ These Terms of Use ("Terms") govern your use of the website **https://freeitchga
|
||||
|
||||
You agree **not** to:
|
||||
|
||||
1. **Bulk-scrape** the Website or its API beyond the published `/data` files. Use the `/data` JSON files or the Repository instead; it is cheaper for everyone.
|
||||
1. **Bulk-scrape** the Website or its API, including the files under `/data`. Get the data from the Repository instead; it is cheaper for everyone.
|
||||
2. **Abuse the image proxy** (`/img`): it exists only to show cover images on the Website. Don't hotlink `/img` URLs from other sites or apps, and don't use it to fetch images in bulk.
|
||||
3. **Access restricted areas without authorization**: `/admin`, the admin API (`/api/admin/…`) and the ingest API (`/api/ingest`) are for the Maintainer only. Good-faith security research is welcome under the rules of the [Security Policy](../SECURITY.md).
|
||||
3. **Access restricted areas without authorization**: `/admin` and the admin API (`/api/admin/…`) are for the Maintainer only. Good-faith security research is welcome under the rules of the [Security Policy](../SECURITY.md).
|
||||
4. **Spam the Suggest form**, or circumvent the Website's verification check, Turnstile or the rate limits (for example with scripts, shared verification cookies or rotating IP addresses).
|
||||
5. **Submit links to malware**, scams, phishing, or illegal content, anywhere in the Project.
|
||||
6. **Misrepresent affiliation**: don't claim to be the Maintainer or the Project, don't present a fork or copy as the official Project, don't claim that the Project is run or endorsed by itch.io, and don't use the names "poli0981" or "SkullMute" to endorse your own work without prior written permission.
|
||||
|
||||
+14
-8
@@ -15,12 +15,17 @@ JetBrains 2026.x (paid lineup):
|
||||
VS Code, Sublime, vim, etc. all work fine; nothing JetBrains-specific is
|
||||
checked into the repo.
|
||||
|
||||
TypeScript 7 (the `tsc` that `npm run build` runs) is a native binary without `tsserver`. In VS Code,
|
||||
install the TypeScript Native Preview extension (`TypeScriptTeam.native-preview`) for the TS 7
|
||||
language server. In WebStorm, use its TypeScript 7 support or point *Languages & Frameworks →
|
||||
TypeScript* at `webapp/node_modules/@typescript/old` (TypeScript 6.0, which reports the same errors).
|
||||
|
||||
## Language toolchains
|
||||
|
||||
| Stack | Required | Used for |
|
||||
|-------|----------|----------|
|
||||
| Python | 3.14 (`.python-version`) | Data pipeline (`scripts/`) and its tests |
|
||||
| Node.js | 22.22+ (CI uses `webapp/.node-version`) | Web app, admin app, Worker, Tauri front end |
|
||||
| Python | 3.15 (`.github/python-version`) | Data pipeline (`scripts/`) and its tests |
|
||||
| Node.js | 26+ (CI uses `webapp/.node-version`) | Web app, admin app, Worker, Tauri front end |
|
||||
| npm | ships with Node.js | Dependencies (`npm ci` from `package-lock.json`) |
|
||||
| Wrangler | project dependency (`npx wrangler`) | Local Worker, local D1, deploys |
|
||||
| Rust | stable (via `rustup`) | Tauri desktop / Android builds only |
|
||||
@@ -39,8 +44,8 @@ python -m venv .venv
|
||||
pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
On Windows, set `PYTHONUTF8=1` (PowerShell: `$env:PYTHONUTF8 = "1"`) so Python reads and writes
|
||||
the catalog files as UTF-8.
|
||||
Python 3.15 reads and writes files as UTF-8 by default (UTF-8 mode, PEP 686), so Windows no longer
|
||||
needs `PYTHONUTF8=1`.
|
||||
|
||||
### Checks (same as `python-ci.yml`)
|
||||
|
||||
@@ -76,8 +81,8 @@ catalog. See [`CLAUDE.md`](../CLAUDE.md) for the data-layer overview.
|
||||
```sh
|
||||
cd webapp
|
||||
npm ci
|
||||
npm run dev # public app on http://localhost:5173 with HMR; /data is served from ../data_game
|
||||
npm run build # tsc -b + vite build → webapp/dist/ (app, /admin/, /data, sitemap.xml)
|
||||
npm run dev # public app on http://localhost:5173 with HMR; /data (packs + JSON) is built from ../data_game
|
||||
npm run build # tsc -b + vite build → webapp/dist/ (app, /admin/, /data/pack + legacy JSON, sitemap.xml)
|
||||
npm run preview # serve the production build (without the Worker)
|
||||
```
|
||||
|
||||
@@ -180,8 +185,9 @@ npm run tauri:dev # native window + HMR
|
||||
npm run tauri:build # native installers
|
||||
```
|
||||
|
||||
The apps read the live catalog from https://freeitchgames.win/data and load covers from
|
||||
`img.itch.zone`, so they need internet access but not the local Worker. Platform prerequisites
|
||||
The apps read the live catalog packs from https://freeitchgames.win/data/pack and load covers from
|
||||
`img.itch.zone`, so they need internet access but not the local Worker (and code that changes the pack
|
||||
format only works in `tauri:dev` once the site with that format is deployed). Platform prerequisites
|
||||
(WebView2 on Windows 10, Xcode CLT on macOS, `libwebkit2gtk-4.1-dev` on Debian/Ubuntu) and the
|
||||
Android build are documented in [`webapp/TAURI.md`](../webapp/TAURI.md).
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Hướng dẫn đóng góp
|
||||
|
||||
> Bản dịch từ bản tiếng Anh cập nhật ngày 2026-09-24 (English source revision 2026-09-24). Nếu có khác biệt, bản tiếng Anh được ưu tiên.
|
||||
> Bản dịch từ bản tiếng Anh cập nhật ngày 2026-10-13 (English source revision 2026-10-13). Nếu có khác biệt, bản tiếng Anh được ưu tiên.
|
||||
>
|
||||
> Bản tiếng Anh tại [`CONTRIBUTING.md`](../../../CONTRIBUTING.md) là bản chính thức.
|
||||
|
||||
@@ -16,16 +16,16 @@ Game thì gửi qua website; mọi thứ khác thì qua các template issue/PR t
|
||||
- Tùy chọn: một ghi chú ngắn (vì sao game hay, cảnh báo nội dung…), tối đa 500 ký tự. Ghi chú bị xóa sau 180 ngày. Vui lòng đừng ghi dữ liệu cá nhân vào đó.
|
||||
- Một bước kiểm tra Cloudflare Turnstile và giới hạn tần suất giúp chặn bot; nếu bị chặn, đợi một lúc rồi thử lại.
|
||||
- Chỉ game itch.io **miễn phí**. Game trả phí sẽ tự động bị loại khi trang game được cào.
|
||||
- Không có gì lên site ngay lập tức: mọi đề xuất đều vào **hàng chờ duyệt**, và không có gì vào danh mục cho tới khi Maintainer duyệt. Link được duyệt sau đó được pipeline dữ liệu cào và xuất hiện trên site sau lần deploy kế tiếp.
|
||||
- Không có gì lên site ngay lập tức: mọi đề xuất đều vào **hàng chờ duyệt**, nơi Maintainer duyệt hoặc từ chối, bằng tay hoặc qua quy tắc tự động (một đề xuất không có cờ cảnh báo mà không ai từ chối trong khoảng một ngày có thể được duyệt tự động). Link được duyệt sau đó được pipeline dữ liệu cào và xuất hiện trên site sau lần deploy kế tiếp.
|
||||
|
||||
Có cả một danh sách? Issue template **[Add Games (bulk list)]** nhận tối đa 15 link một lần, và một PR thêm URL itch.io vào `scripts/temp_link.json` cũng được (dùng PR template [Add Games]). Cả hai đều đi qua cùng quy trình duyệt: Maintainer kiểm tra từng link trước khi đưa vào hàng chờ, và việc merge một PR như vậy chính là sự phê duyệt. Vui lòng đừng sửa tay `data_game/`: các file đó do pipeline quản lý (xem [mục 7](#7-gửi-thay-đổi-code-pr)).
|
||||
|
||||
#### 1b. Những cách khác để game được thêm vào
|
||||
|
||||
- Extension trình duyệt đi kèm của Maintainer ([itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension), repo riêng, GPL-3.0) gửi link qua một API có xác thực trong lúc duyệt itch.io. Đây là công cụ riêng của Maintainer, không phải kênh gửi công khai.
|
||||
- Website đọc các RSS feed công khai của itch.io về game miễn phí mới và phổ biến mỗi 4 giờ (mỗi lượt một feed) và đưa những gì tìm được vào hàng chờ.
|
||||
- Website cũng đọc các collection công khai trên itch.io mà Maintainer theo dõi và đưa thẳng game miễn phí mới của chúng vào hàng chờ ingest (game từng bị gỡ, hoặc có cờ cảnh báo, thì vào hàng chờ duyệt).
|
||||
|
||||
Cả hai đều vào cùng hàng chờ duyệt với trang Suggest. (Đường thêm game qua Telegram bot cũ đã ngừng hoạt động.)
|
||||
RSS feed vào cùng hàng chờ duyệt với trang Suggest. (Extension trình duyệt của Maintainer và Telegram bot cũ đã ngừng hoạt động; repo của chúng, [itch-f2p-extension](https://github.com/poli0981/itch-f2p-extension) và [telegram-scraper-bot](https://github.com/poli0981/telegram-scraper-bot), đã được lưu trữ (archive).)
|
||||
|
||||
### 2. Gỡ game hoặc sửa dữ liệu game
|
||||
|
||||
@@ -84,7 +84,7 @@ Chi tiết có trong [Điều khoản sử dụng](ToS.md) và [Tuyên bố mi
|
||||
|
||||
Hướng dẫn đầy đủ (Worker, D1 cục bộ, secret, Tauri, Android) nằm trong **[`docs/dev_env.md`](../../dev_env.md)** ([VI](dev_env.md)); ghi chú build Tauri/Android nằm trong [`webapp/TAURI.md`](../../../webapp/TAURI.md). Bản ngắn gọn:
|
||||
|
||||
**Pipeline Python** (Python 3.14, xem `.python-version`):
|
||||
**Pipeline Python** (Python 3.15, xem `.github/python-version`):
|
||||
|
||||
```sh
|
||||
python -m venv .venv
|
||||
@@ -100,7 +100,7 @@ Muốn thử cào thật trên máy: thêm một URL vào `scripts/temp_link.jso
|
||||
`python scripts/update_info.py --out patch.json` và `python scripts/apply_patch.py patch.json`
|
||||
(đừng commit kết quả; xem quy tắc PR ở trên).
|
||||
|
||||
**Webapp + Worker** (Node.js 22.22+, CI dùng phiên bản trong `webapp/.node-version`):
|
||||
**Webapp + Worker** (Node.js 26+, CI dùng phiên bản trong `webapp/.node-version`):
|
||||
|
||||
```sh
|
||||
cd webapp
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Tuyên bố miễn trừ trách nhiệm (Disclaimer)
|
||||
|
||||
Cập nhật lần cuối: 2026-09-29
|
||||
Cập nhật lần cuối: 2026-10-16
|
||||
|
||||
Áp dụng từ: khi phát hành phiên bản 4.0.0.
|
||||
|
||||
> Bản dịch từ bản tiếng Anh cập nhật ngày 2026-09-18 (English source revision 2026-09-18). Nếu có khác biệt, bản tiếng Anh được ưu tiên. Bản gốc: [`docs/DISCLAIMER.md`](../../DISCLAIMER.md).
|
||||
> Bản dịch từ bản tiếng Anh cập nhật ngày 2026-10-13 (English source revision 2026-10-13). Nếu có khác biệt, bản tiếng Anh được ưu tiên. Bản gốc: [`docs/DISCLAIMER.md`](../../DISCLAIMER.md).
|
||||
|
||||
Dự án này — repository `free-games-itchio-list` và website **[freeitchgames.win](https://freeitchgames.win)** — là một dự án sở thích: một danh mục được tuyển chọn và tự động cập nhật gồm các game miễn phí trên [itch.io](https://itch.io). Dự án do một dev người Việt đang thất nghiệp, poli0981 (SkullMute), vận hành — dưới đây gọi là "Người duy trì" — với các trợ lý AI gánh phần lớn việc nặng. Làm ra vì vibes, để khám phá game indie và giải khuây. Không hứa hẹn gì lớn lao — nhưng các điều khoản mang tính pháp lý dưới đây là thật.
|
||||
|
||||
@@ -33,7 +33,7 @@ Dự án hoạt động độc lập. Dự án **không liên kết với, khôn
|
||||
|
||||
Dự án **không làm, không lưu trữ, không phân phối, không bán và không chứng thực** các game được liệt kê. Mỗi game liên kết tới trang riêng của nó trên itch.io, và mọi lượt tải về đều diễn ra tại đó, theo điều khoản của itch.io và điều khoản hoặc giấy phép riêng của người làm game.
|
||||
|
||||
Game được đưa vào catalog qua trang Suggest công khai, tiện ích trình duyệt của Người duy trì, hoặc các RSS feed công khai của itch.io, và không có gì được thêm vào cho đến khi Người duy trì duyệt. Việc duyệt đó **không phải** là kiểm tra bảo mật, đánh giá chất lượng hay sự chứng thực.
|
||||
Game được đưa vào catalog qua trang Suggest công khai, các RSS feed công khai của itch.io, hoặc các collection công khai trên itch.io mà Người duy trì theo dõi. Mỗi game đều được Người duy trì duyệt, từng game một hoặc qua các quy tắc tự động do Người duy trì đặt ra. Việc duyệt đó **không phải** là kiểm tra bảo mật, đánh giá chất lượng hay sự chứng thực.
|
||||
|
||||
## 5. Không bảo đảm về các game
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Thỏa thuận cấp phép cho người dùng cuối (EULA)
|
||||
|
||||
Cập nhật lần cuối: 2026-09-29
|
||||
Cập nhật lần cuối: 2026-10-16
|
||||
|
||||
Áp dụng từ: khi phát hành phiên bản 4.1.1.
|
||||
Áp dụng từ: khi phát hành phiên bản 4.2.0.
|
||||
|
||||
Bản dịch từ bản tiếng Anh cập nhật ngày 2026-09-24 (English source revision 2026-09-24). Nếu có khác biệt, bản tiếng Anh được ưu tiên.
|
||||
Bản dịch từ bản tiếng Anh cập nhật ngày 2026-10-03 (English source revision 2026-10-03). Nếu có khác biệt, bản tiếng Anh được ưu tiên.
|
||||
|
||||
> Bản tiếng Anh: [`docs/EULA.md`](../../EULA.md).
|
||||
|
||||
@@ -50,12 +50,12 @@ Không điều gì trong EULA này lấy đi quyền mà Giấy phép MIT trao c
|
||||
|
||||
Ứng dụng **không có telemetry**: không analytics, không báo cáo lỗi (crash reporting), không theo dõi. Ứng dụng chỉ kết nối tới hai nơi:
|
||||
|
||||
- **freeitchgames.win**, để tải file JSON của Catalog từ <https://freeitchgames.win/data>. Website chạy trên Cloudflare; Cloudflare xử lý dữ liệu yêu cầu thông thường (như địa chỉ IP và user agent của Bạn) để phân phối nội dung, như mô tả trong [Chính sách quyền riêng tư (Privacy Policy)](PrivacyPolicy.md).
|
||||
- **freeitchgames.win**, để tải dữ liệu Catalog từ <https://freeitchgames.win/data>. Website chạy trên Cloudflare; Cloudflare xử lý dữ liệu yêu cầu thông thường (như địa chỉ IP và user agent của Bạn) để phân phối nội dung, như mô tả trong [Chính sách quyền riêng tư (Privacy Policy)](PrivacyPolicy.md).
|
||||
- **img.itch.zone**, máy chủ ảnh của itch.io, để tải trực tiếp ảnh bìa. itch.io nhận dữ liệu yêu cầu thông thường cho các ảnh đó, như với bất kỳ ảnh nào được tải từ web (xem [chính sách quyền riêng tư của itch.io](https://itch.io/docs/legal/privacy-policy)).
|
||||
|
||||
Khi Bạn bấm vào một game hoặc một link bên ngoài khác, link đó sẽ mở trong trình duyệt mặc định của Bạn (itch.io, GitHub hoặc Website). Từ lúc đó, điều khoản và chính sách quyền riêng tư của chính trang đó được áp dụng.
|
||||
|
||||
Ứng dụng lưu các tùy chọn của Bạn (ngôn ngữ, giao diện sáng/tối, mật độ bố cục, lựa chọn 18+, và phiên bản điều khoản pháp lý Bạn đã chấp nhận) cùng một bản sao lưu đệm (cache) của Catalog (giữ tối đa 7 ngày) chỉ trên thiết bị của Bạn. Không dữ liệu nào trong số đó được gửi cho Người duy trì.
|
||||
Ứng dụng lưu các tùy chọn của Bạn (ngôn ngữ, giao diện sáng/tối, mật độ bố cục, việc ẩn thanh cuộn, lựa chọn 18+, và phiên bản điều khoản pháp lý Bạn đã chấp nhận) cùng một bản sao lưu đệm (cache) đã nén của Catalog (giữ tối đa 7 ngày) chỉ trên thiết bị của Bạn. Không dữ liệu nào trong số đó được gửi cho Người duy trì.
|
||||
|
||||
## 5. Cài đặt ứng dụng, cập nhật và gỡ bỏ
|
||||
|
||||
@@ -86,7 +86,7 @@ Bạn đồng ý không:
|
||||
|
||||
- Trình bày một bản build đã sửa đổi như một bản phát hành chính thức, ví dụ phân phối nó dưới tên và biểu tượng của dự án, hoặc với app identifier chính thức `com.poli0981.freegamesitchio`, mà không ghi rõ đó là bản không chính thức. Fork luôn được chào đón theo Giấy phép MIT; chỉ cần làm rõ rằng đó là bản của Bạn.
|
||||
- Sử dụng tên Người duy trì, GitHub handle `poli0981`, hoặc alias "SkullMute" để chứng thực hoặc quảng bá tác phẩm phái sinh mà không có sự cho phép trước bằng văn bản.
|
||||
- Sử dụng Ứng dụng, hoặc một phiên bản đã sửa đổi của nó, theo cách vi phạm [Điều khoản sử dụng](ToS.md) của Website, ví dụ gửi yêu cầu tự động hàng loạt vượt ngoài các file `/data` đã công bố, hoặc cố truy cập khu vực quản trị (admin) hay ingest API khi không được phép.
|
||||
- Sử dụng Ứng dụng, hoặc một phiên bản đã sửa đổi của nó, theo cách vi phạm [Điều khoản sử dụng](ToS.md) của Website, ví dụ gửi yêu cầu tự động hàng loạt tới Website (dữ liệu có sẵn trong Repository), hoặc cố truy cập khu vực quản trị (admin) khi không được phép.
|
||||
- Sử dụng Ứng dụng hoặc các bộ cài của nó để lưu trữ, phân phối hoặc tạo điều kiện phân phối malware, trang phishing, hoặc nội dung vi phạm pháp luật hiện hành.
|
||||
|
||||
Các quy tắc này là về cách Bạn sử dụng Ứng dụng cũng như tên và dịch vụ của dự án. Chúng tồn tại song song với Giấy phép MIT và không hạn chế các quyền mà giấy phép đó trao đối với mã nguồn.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Chính sách quyền riêng tư (Privacy Policy)
|
||||
|
||||
Cập nhật lần cuối: 2026-09-29
|
||||
Cập nhật lần cuối: 2026-10-16
|
||||
|
||||
Áp dụng từ: ngày 2026-09-29 đối với Website, và từ phiên bản 4.1.1 đối với các Ứng dụng.
|
||||
Áp dụng từ: ngày 2026-10-13 đối với Website, và từ phiên bản 4.4.0 đối với các Ứng dụng.
|
||||
|
||||
> Bản dịch từ bản tiếng Anh cập nhật ngày 2026-09-24 (English source revision 2026-09-24). Nếu có khác biệt, bản tiếng Anh được ưu tiên. Bản gốc: [`docs/PrivacyPolicy.md`](../../PrivacyPolicy.md).
|
||||
> Bản dịch từ bản tiếng Anh cập nhật ngày 2026-10-13 (English source revision 2026-10-13). Nếu có khác biệt, bản tiếng Anh được ưu tiên. Bản gốc: [`docs/PrivacyPolicy.md`](../../PrivacyPolicy.md).
|
||||
|
||||
Chính sách quyền riêng tư này giải thích dự án `free-games-itchio-list` xử lý những dữ liệu cá nhân nào, vì sao, ai hỗ trợ xử lý, dữ liệu được lưu trong bao lâu và bạn có những quyền gì. Chính sách áp dụng cho website **https://freeitchgames.win**, các ứng dụng desktop và Android, dữ liệu danh mục công khai và repository. Nói ngắn gọn: không có tài khoản, không quảng cáo, không cookie theo dõi. Dự án giữ ít dữ liệu nhất có thể, và phần lớn những gì tồn tại chỉ nằm trong trình duyệt của chính bạn.
|
||||
|
||||
@@ -36,7 +36,7 @@ Bên kiểm soát dữ liệu cá nhân được mô tả trong Chính sách nà
|
||||
|
||||
### 1.3 Những gì Chính sách này không áp dụng
|
||||
|
||||
Chính sách này không áp dụng cho các dịch vụ tự xử lý dữ liệu với tư cách của chính họ: itch.io (mọi link game và mọi lượt tải đều dẫn tới đó), GitHub đối với tài khoản GitHub và hoạt động của bạn trên github.com, việc Cloudflare dùng dữ liệu cho mục đích riêng của họ như mô tả trong chính sách quyền riêng tư của Cloudflare, bản thân các game, và tiện ích trình duyệt của Người duy trì (một dự án riêng, có tài liệu riêng). Chính sách của từng bên sẽ được áp dụng (link ở mục 4).
|
||||
Chính sách này không áp dụng cho các dịch vụ tự xử lý dữ liệu với tư cách của chính họ: itch.io (mọi link game và mọi lượt tải đều dẫn tới đó), GitHub đối với tài khoản GitHub và hoạt động của bạn trên github.com, việc Cloudflare dùng dữ liệu cho mục đích riêng của họ như mô tả trong chính sách quyền riêng tư của Cloudflare, và bản thân các game. Chính sách của từng bên sẽ được áp dụng (link ở mục 4).
|
||||
|
||||
## 2. Dữ liệu nào được xử lý và vì sao
|
||||
|
||||
@@ -53,7 +53,7 @@ Chính sách này không áp dụng cho các dịch vụ tự xử lý dữ li
|
||||
### 2.2 Khi dùng các Ứng dụng
|
||||
|
||||
- Các Ứng dụng là trình xem chỉ đọc, **không telemetry, không analytics và không tài khoản**.
|
||||
- Chúng tải JSON của Danh mục từ https://freeitchgames.win/data (Cloudflare xử lý dữ liệu request như mô tả ở mục 2.1) và tải ảnh bìa **trực tiếp từ máy chủ ảnh của itch.io (img.itch.zone)**, vì vậy itch.io nhận được địa chỉ IP và user agent của bạn khi ảnh bìa được tải.
|
||||
- Chúng tải dữ liệu Danh mục từ https://freeitchgames.win/data (Cloudflare xử lý dữ liệu request như mô tả ở mục 2.1; các phiên bản 4.0–4.1.1 vẫn nhận các file JSON cũ, được nhận biết qua header `Origin` mà cửa sổ ứng dụng gửi kèm) và tải ảnh bìa **trực tiếp từ máy chủ ảnh của itch.io (img.itch.zone)**, vì vậy itch.io nhận được địa chỉ IP và user agent của bạn khi ảnh bìa được tải.
|
||||
- Bấm vào một game sẽ mở trang itch.io của game đó trong trình duyệt của bạn.
|
||||
- Việc cập nhật là thủ công: các Ứng dụng không tự kiểm tra bản cập nhật. Khi bạn tải bộ cài hoặc file APK từ GitHub Releases, GitHub xử lý request đó.
|
||||
- Các Ứng dụng dùng cùng kiểu lưu trữ cục bộ như Website (mục 3.1), chỉ trên thiết bị của bạn. Gỡ cài đặt Ứng dụng sẽ xóa dữ liệu này.
|
||||
@@ -65,14 +65,14 @@ Khi bạn dùng trang Suggest (https://freeitchgames.win/suggest):
|
||||
- **Những gì được lưu**: URL game, ghi chú tùy chọn của bạn (tối đa 500 ký tự) và thời điểm gửi, trong hàng chờ duyệt (mục 2.4). Trang không yêu cầu và không lưu tên, địa chỉ email hay tài khoản nào.
|
||||
- **Chống spam**: Cloudflare Turnstile xử lý các tín hiệu về thiết bị và trình duyệt của bạn để phân biệt người thật với bot. Để kiểm tra kết quả, Website gửi token Turnstile và địa chỉ IP của bạn tới dịch vụ xác minh của Cloudflare.
|
||||
- **Giới hạn tần suất**: địa chỉ IP của bạn chỉ được dùng tạm thời để đếm số request. Dự án không lưu nó.
|
||||
- **Mục đích**: để xem xét đề xuất của bạn. Nếu được duyệt, URL game trở thành một phần của Danh mục công khai. Ghi chú của bạn không bao giờ được công bố và **tự động bị xóa sau 180 ngày**.
|
||||
- **Mục đích**: để xem xét đề xuất của bạn, do Người duy trì hoặc các quy tắc tự động do Người duy trì đặt ra (mục 2.4). Nếu được duyệt, URL game trở thành một phần của Danh mục công khai. Ghi chú của bạn không bao giờ được công bố và **tự động bị xóa sau 180 ngày**.
|
||||
- **Vui lòng không đưa dữ liệu cá nhân vào ghi chú**, dù là của bạn hay của người khác. Biểu mẫu này chỉ dành cho link game.
|
||||
|
||||
### 2.4 Hàng chờ duyệt và khu vực quản trị
|
||||
|
||||
- **Game đến từ đâu**: đề xuất từ trang Suggest; dữ liệu gửi từ tiện ích trình duyệt của Người duy trì qua một API có xác thực (URL game, tiêu đề và ghi chú tùy chọn, và ID service token của tiện ích; các bản ghi chống gửi trùng được giữ 7 ngày); và tự động phát hiện từ các RSS feed công khai của itch.io.
|
||||
- **Hàng chờ duyệt** (Cloudflare D1) chứa các URL game ứng viên cùng nguồn gửi, tiêu đề, URL ảnh bìa và gợi ý thể loại lấy từ itch.io, các cờ tự động, ghi chú, mốc thời gian và quyết định của Người duy trì. Không game nào được đưa vào Danh mục cho tới khi Người duy trì duyệt. Các URL bị từ chối được giữ vô thời hạn làm danh sách chặn; chỉ cần URL là đủ để không phải xét lại cùng một game.
|
||||
- **Khu vực quản trị** (https://freeitchgames.win/admin) chỉ dành cho Người duy trì. Khu vực này được bảo vệ bằng Cloudflare Access (đăng nhập bằng GitHub hoặc mã PIN dùng một lần gửi qua email); Cloudflare Access đặt cookie phiên `CF_Authorization` trong trình duyệt của Người duy trì. Các thao tác quản trị được ghi vào nhật ký kiểm tra (audit log) kèm địa chỉ email của Người duy trì. Nhật ký này không bao giờ được công bố.
|
||||
- **Game đến từ đâu**: đề xuất từ trang Suggest; tự động phát hiện từ các RSS feed công khai của itch.io; và game miễn phí mới trong các collection công khai trên itch.io mà Người duy trì theo dõi (Website đọc các trang công khai đó; không liên quan gì tới dữ liệu của bạn).
|
||||
- **Hàng chờ duyệt** (Cloudflare D1) chứa các URL game ứng viên cùng nguồn gửi, tiêu đề, URL ảnh bìa và gợi ý thể loại lấy từ itch.io, các cờ tự động, ghi chú, mốc thời gian và quyết định của Người duy trì. Game được đưa vào Danh mục khi Người duy trì duyệt: từng game một, hoặc qua các quy tắc tự động do Người duy trì đặt ra và có thể thay đổi bất cứ lúc nào (mục không có cờ cảnh báo mà không ai từ chối trong thời gian chờ; game miễn phí mới trong các collection Người duy trì theo dõi). Các lượt duyệt tự động được ghi vào nhật ký kiểm tra như thao tác của chính Người duy trì. Các URL bị từ chối được giữ vô thời hạn làm danh sách chặn; chỉ cần URL là đủ để không phải xét lại cùng một game.
|
||||
- **Khu vực quản trị** (https://freeitchgames.win/admin) chỉ dành cho Người duy trì. Khu vực này được bảo vệ bằng Cloudflare Access (đăng nhập bằng GitHub hoặc mã PIN dùng một lần gửi qua email); Cloudflare Access đặt cookie phiên `CF_Authorization` trong trình duyệt của Người duy trì. Các thao tác quản trị, kể cả thay đổi quy tắc tự động, được ghi vào nhật ký kiểm tra (audit log) kèm địa chỉ email của Người duy trì. Nhật ký này không bao giờ được công bố.
|
||||
|
||||
### 2.5 Dữ liệu game công khai trong Danh mục
|
||||
|
||||
@@ -106,10 +106,10 @@ Website và các Ứng dụng chỉ lưu các mục sau trên thiết bị của
|
||||
|
||||
| Lưu trữ | Khóa | Nội dung | Giữ đến khi |
|
||||
|---|---|---|---|
|
||||
| `localStorage` | `webapp.prefs` | Tùy chọn giao diện như ngôn ngữ và mật độ hiển thị, lựa chọn nội dung 18+ (NSFW) của bạn, và phiên bản các văn bản pháp lý bạn đã chấp nhận | Bạn xóa nó |
|
||||
| `localStorage` | `webapp.prefs` | Tùy chọn giao diện như ngôn ngữ, mật độ hiển thị và việc ẩn thanh cuộn, lựa chọn nội dung 18+ (NSFW) của bạn, và phiên bản các văn bản pháp lý bạn đã chấp nhận | Bạn xóa nó |
|
||||
| `localStorage` | `webapp.theme` | Giao diện: `light`, `dark` hoặc `system` | Bạn xóa nó |
|
||||
| `localStorage` | `webapp.gate` | Thời điểm lượt xác minh hiện tại (mục 3.2) hết hạn, để tab đang mở kịp kiểm tra lại | Bị thay bởi lần kiểm tra sau; bạn có thể xóa nó |
|
||||
| IndexedDB (qua `idb-keyval`) | `webapp.query-cache` | Bản cache của JSON Danh mục công khai, để tải nhanh và dùng offline ở mức hạn chế | Tối đa 7 ngày, sau đó được làm mới hoặc bị bỏ đi |
|
||||
| IndexedDB (qua `idb-keyval`) | `webapp.query-cache` | Bản sao đã nén của dữ liệu Danh mục công khai (danh sách game, nhật ký game bị gỡ, chuỗi số lượng game theo thời gian và mô tả của những game bạn đã mở hoặc tìm), để tải nhanh và dùng offline ở mức hạn chế | Tối đa 7 ngày, sau đó được làm mới hoặc bị bỏ đi |
|
||||
| `sessionStorage` | `reloaded-after-deploy` | Một mốc thời gian, để trang chỉ tự tải lại tối đa một lần sau khi site được cập nhật | Tab được đóng |
|
||||
|
||||
Ở lần tải đầu tiên, Website và các Ứng dụng cũng xóa các mục mà ứng dụng v3 cũ để lại trong bộ nhớ trình duyệt (một GitHub access token đã mã hóa và dữ liệu khóa ký commit). v4 không dùng tới các mục này.
|
||||
@@ -145,7 +145,6 @@ Cloudflare và Google có thể xử lý dữ liệu ở bất kỳ đâu trên
|
||||
| Địa chỉ IP dùng cho giới hạn tần suất và Turnstile | Chỉ dùng tạm thời; Dự án không bao giờ lưu |
|
||||
| Ghi chú trên trang Suggest | Tự động xóa sau **180 ngày** |
|
||||
| URL game được đề xuất hoặc được phát hiện, kèm nguồn và mốc thời gian | Nằm trong hàng chờ duyệt chừng nào còn cần để không phải xét cùng một game hai lần. URL được duyệt trở thành một phần của Danh mục công khai; URL bị từ chối được giữ vô thời hạn làm danh sách chặn |
|
||||
| Bản ghi chống gửi trùng từ tiện ích trình duyệt | **7 ngày** |
|
||||
| Nhật ký kiểm tra quản trị (kèm địa chỉ email của Người duy trì) | Được giữ làm hồ sơ các thay đổi quản trị; không bao giờ công bố |
|
||||
| Mục trong Danh mục (bao gồm tên nhà phát triển và nhà phát hành) | Khi game còn được liệt kê. Bản ghi gỡ bỏ vẫn nằm trong danh sách game đã gỡ công khai, và lịch sử Git của Repository giữ các phiên bản trước (xem mục 7.3) |
|
||||
| Email, bao gồm yêu cầu gỡ game và yêu cầu liên quan đến bản quyền | Trong thời gian cần thiết để xử lý yêu cầu |
|
||||
@@ -219,8 +218,8 @@ Theo GDPR và UK GDPR, bạn cũng có thể yêu cầu hạn chế việc xử
|
||||
|
||||
- Website chỉ được phục vụ qua HTTPS.
|
||||
- Truy cập tự động phải qua bước kiểm tra Cloudflare Turnstile trước khi các trang và ảnh bìa của Website được tải (mục 2.1).
|
||||
- Tối thiểu hóa dữ liệu: không tài khoản, mã nguồn của Dự án không lưu địa chỉ IP, ghi chú Suggest bị xóa sau 180 ngày, bản ghi chống gửi trùng bị xóa sau 7 ngày.
|
||||
- Khu vực quản trị và API ingest được bảo vệ bằng Cloudflare Access, và mã máy chủ của Website tự xác minh token Access ở mọi request được bảo vệ.
|
||||
- Tối thiểu hóa dữ liệu: không tài khoản, mã nguồn của Dự án không lưu địa chỉ IP, ghi chú Suggest bị xóa sau 180 ngày.
|
||||
- Khu vực quản trị được bảo vệ bằng Cloudflare Access, và mã máy chủ của Website tự xác minh token Access ở mọi request được bảo vệ.
|
||||
- Các thông tin bí mật (như private key của GitHub App) được giữ trong kho lưu trữ bí mật của Cloudflare, không bao giờ nằm trong Repository. Khu vực quản trị ghi vào Repository dưới danh nghĩa một GitHub App, với các commit đã được xác minh (verified).
|
||||
- Content Security Policy chặt chẽ chỉ cho phép script từ chính Website, cộng với Cloudflare Web Analytics và Turnstile.
|
||||
- Các GitHub Actions được ghim theo commit SHA cụ thể với quyền tối thiểu cần thiết, và Dependabot giữ cho các dependency luôn được cập nhật.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Chính sách bảo mật (Security Policy)
|
||||
|
||||
Cập nhật lần cuối: 2026-09-29
|
||||
Cập nhật lần cuối: 2026-10-16
|
||||
|
||||
Áp dụng từ: khi phát hành phiên bản 4.0.0.
|
||||
|
||||
@@ -26,7 +26,7 @@ Một báo cáo tốt nên có:
|
||||
|
||||
**Trong phạm vi**
|
||||
|
||||
- Website https://freeitchgames.win và Cloudflare Worker của nó, bao gồm proxy ảnh (`/img`), API Suggest (`/api/suggest`), API ingest mà tiện ích trình duyệt của Người duy trì sử dụng (`/api/ingest`), cùng ứng dụng và API quản trị (`/admin`, `/api/admin`).
|
||||
- Website https://freeitchgames.win và Cloudflare Worker của nó, bao gồm proxy ảnh (`/img`), API Suggest (`/api/suggest`), cùng ứng dụng và API quản trị (`/admin`, `/api/admin`).
|
||||
- Pipeline dữ liệu (`scripts/`) và các workflow GitHub Actions (`.github/workflows/`).
|
||||
- Các ứng dụng desktop và file APK Android được phát hành trên [GitHub Releases](https://github.com/poli0981/free-games-itchio-list/releases).
|
||||
- Dữ liệu được công bố (`data_game/`, `scripts/deleted_games.json` và https://freeitchgames.win/data) — ví dụ: một cách đưa nội dung vào catalog mà không cần Người duy trì phê duyệt.
|
||||
@@ -39,7 +39,7 @@ Một báo cáo tốt nên có:
|
||||
- Tấn công từ chối dịch vụ theo lưu lượng (DoS/DDoS) và kiểm thử tải.
|
||||
- Báo cáo từ công cụ quét tự động mà không có proof of concept chạy được.
|
||||
|
||||
Tiện ích trình duyệt (poli0981/itch-f2p-extension) nằm trong repository riêng; lỗi của chính tiện ích xin báo ở đó. Lỗi trong API ingest mà tiện ích kết nối tới thì thuộc phạm vi ở đây.
|
||||
Tiện ích trình duyệt (poli0981/itch-f2p-extension) và Telegram bot (poli0981/telegram-scraper-bot) đã được lưu trữ (archive) và không còn được hỗ trợ, nên nằm ngoài phạm vi.
|
||||
|
||||
## 3. Phiên bản được hỗ trợ
|
||||
|
||||
|
||||
+8
-8
@@ -1,16 +1,16 @@
|
||||
# Điều khoản sử dụng (Terms of Use)
|
||||
|
||||
Cập nhật lần cuối: 2026-09-29
|
||||
Cập nhật lần cuối: 2026-10-16
|
||||
|
||||
Áp dụng từ: ngày 2026-09-29 đối với Website, và từ phiên bản 4.1.1 đối với các Ứng dụng.
|
||||
Áp dụng từ: ngày 2026-10-13 đối với Website, và từ phiên bản 4.4.0 đối với các Ứng dụng.
|
||||
|
||||
Bản dịch từ bản tiếng Anh cập nhật ngày 2026-09-24 (English source revision 2026-09-24). Nếu có khác biệt, bản tiếng Anh được ưu tiên.
|
||||
Bản dịch từ bản tiếng Anh cập nhật ngày 2026-10-13 (English source revision 2026-10-13). Nếu có khác biệt, bản tiếng Anh được ưu tiên.
|
||||
|
||||
Bản tiếng Anh: [`docs/ToS.md`](../../ToS.md).
|
||||
|
||||
Các Điều khoản sử dụng này ("Điều khoản") điều chỉnh việc bạn sử dụng website **https://freeitchgames.win**, các ứng dụng desktop và Android, và dữ liệu catalog công khai của `free-games-itchio-list`. Văn bản được viết bằng ngôn ngữ dễ hiểu, nhưng đây là một thỏa thuận thực sự. Điều khoản này thay thế bản Điều khoản cũ và bao gồm cả những gì trước đây nằm trong "EULA" của web app cũ; [EULA](EULA.md) giờ chỉ áp dụng cho các bộ cài ứng dụng và file APK Android.
|
||||
|
||||
> **Tóm tắt**: Cứ thoải mái duyệt, chia sẻ, fork code (MIT) và dùng lại dữ liệu kèm ghi nguồn (CC BY 4.0). Đừng scrape hàng loạt website (hãy dùng các file `/data`), đừng hotlink `/img`, đừng spam form Suggest, đừng mò vào `/admin`, và đừng giả danh itch.io hay Người duy trì. Người làm game có thể yêu cầu gỡ hoặc sửa thông tin game của mình. Nội dung người lớn luôn bị ẩn trừ khi bạn chủ động bật.
|
||||
> **Tóm tắt**: Cứ thoải mái duyệt, chia sẻ, fork code (MIT) và dùng lại dữ liệu kèm ghi nguồn (CC BY 4.0). Đừng scrape hàng loạt website (dữ liệu có sẵn trong repo GitHub), đừng hotlink `/img`, đừng spam form Suggest, đừng mò vào `/admin`, và đừng giả danh itch.io hay Người duy trì. Người làm game có thể yêu cầu gỡ hoặc sửa thông tin game của mình. Nội dung người lớn luôn bị ẩn trừ khi bạn chủ động bật.
|
||||
|
||||
## 1. Định nghĩa
|
||||
|
||||
@@ -41,7 +41,7 @@ Các Điều khoản sử dụng này ("Điều khoản") điều chỉnh việc
|
||||
|
||||
- Duyệt Website và Ứng dụng, và mở các liên kết tới itch.io.
|
||||
- Chia sẻ liên kết tới Website và tới trang game.
|
||||
- Tải về và dùng lại dữ liệu Danh mục theo **CC BY 4.0**, kèm ghi nguồn (xem mục 11). Nếu truy cập bằng chương trình, hãy dùng các file JSON được công bố tại https://freeitchgames.win/data hoặc Repository; vui lòng cache chúng thay vì tải lại mỗi lần có request.
|
||||
- Tải về và dùng lại dữ liệu Danh mục theo **CC BY 4.0**, kèm ghi nguồn (xem mục 11). Hãy lấy dữ liệu từ Repository (`data_game/` và `scripts/deleted_games.json`, kể cả dạng file raw), và vui lòng cache lại thay vì tải lại mỗi lần có request. Các file tại https://freeitchgames.win/data là định dạng thu gọn riêng của Website và các Ứng dụng, không phải một giao diện công khai; bản JSON đọc được ở đó chỉ được trả cho các phiên bản Ứng dụng cũ.
|
||||
- Sử dụng, fork, sửa đổi và phân phối lại code theo **Giấy phép MIT**.
|
||||
- Đề xuất một game itch.io miễn phí trên trang Suggest (xem mục 5).
|
||||
- Báo bug, gửi góp ý và mở pull request qua các template của Repository.
|
||||
@@ -51,7 +51,7 @@ Các Điều khoản sử dụng này ("Điều khoản") điều chỉnh việc
|
||||
|
||||
- **Trang Suggest** (https://freeitchgames.win/suggest): bạn có thể gửi một liên kết game itch.io và một ghi chú tùy chọn dài tối đa 500 ký tự. Trang này dùng Cloudflare Turnstile để chống spam và có giới hạn tần suất (rate limit).
|
||||
- Ghi chú chỉ Người duy trì xem được và sẽ bị xóa sau 180 ngày. **Đừng đưa dữ liệu cá nhân vào ghi chú.** Xem [Chính sách quyền riêng tư](PrivacyPolicy.md).
|
||||
- Mọi đề xuất đều vào một **hàng đợi duyệt**. Không có gì được đưa vào Danh mục cho tới khi Người duy trì phê duyệt, và Người duy trì không có nghĩa vụ phải chấp nhận hay phản hồi một đề xuất. Sau khi được duyệt, liên kết sẽ được pipeline scrape, và game nào hóa ra là game trả phí sẽ bị loại.
|
||||
- Mọi đề xuất đều vào một **hàng đợi duyệt**. Người duy trì duyệt hoặc từ chối đề xuất tại đó, từng cái một hoặc qua các quy tắc tự động do Người duy trì đặt ra: một đề xuất không có cờ cảnh báo và không bị ai từ chối trong thời gian chờ mà Người duy trì chọn (hiện khoảng một ngày) có thể được duyệt tự động. Người duy trì không có nghĩa vụ phải chấp nhận hay phản hồi một đề xuất. Sau khi được duyệt, liên kết sẽ được pipeline scrape, và game nào hóa ra là game trả phí sẽ bị loại.
|
||||
- Chỉ những game **thực sự miễn phí** trên itch.io mới đủ điều kiện. Bản demo của game trả phí, game "name your own price" có mức giá tối thiểu lớn hơn 0, và game chỉ miễn phí trong thời gian khuyến mãi thì không.
|
||||
- Khi gửi bất cứ thứ gì cho Dự án (đề xuất, issue, pull request hay comment), bạn xác nhận rằng:
|
||||
- đó là sản phẩm của chính bạn, hoặc bạn có quyền chia sẻ nó;
|
||||
@@ -64,9 +64,9 @@ Các Điều khoản sử dụng này ("Điều khoản") điều chỉnh việc
|
||||
|
||||
Bạn đồng ý **không**:
|
||||
|
||||
1. **Scrape hàng loạt** Website hoặc API của nó, ngoài các file `/data` đã được công bố. Hãy dùng các file JSON trong `/data` hoặc Repository; cách đó tốn ít tài nguyên hơn cho tất cả mọi người.
|
||||
1. **Scrape hàng loạt** Website hoặc API của nó, kể cả các file trong `/data`. Hãy lấy dữ liệu từ Repository; cách đó tốn ít tài nguyên hơn cho tất cả mọi người.
|
||||
2. **Lạm dụng image proxy** (`/img`): proxy này chỉ để hiển thị ảnh bìa trên Website. Đừng hotlink các URL `/img` từ website hay ứng dụng khác, và đừng dùng nó để tải ảnh hàng loạt.
|
||||
3. **Truy cập trái phép các khu vực bị hạn chế**: `/admin`, API quản trị (`/api/admin/…`) và API ingest (`/api/ingest`) chỉ dành cho Người duy trì. Nghiên cứu bảo mật thiện chí luôn được hoan nghênh, theo các quy tắc trong [Security Policy](SECURITY.md).
|
||||
3. **Truy cập trái phép các khu vực bị hạn chế**: `/admin` và API quản trị (`/api/admin/…`) chỉ dành cho Người duy trì. Nghiên cứu bảo mật thiện chí luôn được hoan nghênh, theo các quy tắc trong [Security Policy](SECURITY.md).
|
||||
4. **Spam form Suggest**, hoặc tìm cách vượt qua bước xác minh của Website, Turnstile hay giới hạn tần suất (ví dụ bằng script, dùng chung cookie xác minh hoặc xoay vòng địa chỉ IP).
|
||||
5. **Gửi liên kết tới malware**, lừa đảo, phishing hoặc nội dung bất hợp pháp, ở bất kỳ đâu trong Dự án.
|
||||
6. **Mạo nhận liên kết**: đừng tự nhận là Người duy trì hay là Dự án, đừng giới thiệu một bản fork hay bản sao như thể đó là Dự án chính thức, đừng nói rằng Dự án do itch.io vận hành hoặc bảo trợ, và đừng dùng tên "poli0981" hay "SkullMute" để quảng bá sản phẩm của bạn khi chưa có sự cho phép trước bằng văn bản.
|
||||
|
||||
+14
-8
@@ -17,12 +17,17 @@ JetBrains 2026.x (dòng trả phí):
|
||||
|
||||
VS Code, Sublime, vim, v.v. đều dùng tốt; repo không chứa gì đặc thù của JetBrains.
|
||||
|
||||
TypeScript 7 (bản `tsc` mà `npm run build` chạy) là binary native, không có `tsserver`. Trong VS Code,
|
||||
cài extension TypeScript Native Preview (`TypeScriptTeam.native-preview`) để dùng language server của
|
||||
TS 7. Trong WebStorm, dùng chế độ hỗ trợ TypeScript 7 hoặc trỏ *Languages & Frameworks → TypeScript*
|
||||
tới `webapp/node_modules/@typescript/old` (TypeScript 6.0, báo cùng các lỗi).
|
||||
|
||||
## Toolchain ngôn ngữ
|
||||
|
||||
| Stack | Yêu cầu | Dùng cho |
|
||||
|-------|---------|----------|
|
||||
| Python | 3.14 (`.python-version`) | Pipeline dữ liệu (`scripts/`) và test của nó |
|
||||
| Node.js | 22.22+ (CI dùng `webapp/.node-version`) | Web app, admin app, Worker, phần front end của Tauri |
|
||||
| Python | 3.15 (`.github/python-version`) | Pipeline dữ liệu (`scripts/`) và test của nó |
|
||||
| Node.js | 26+ (CI dùng `webapp/.node-version`) | Web app, admin app, Worker, phần front end của Tauri |
|
||||
| npm | đi kèm Node.js | Dependency (`npm ci` theo `package-lock.json`) |
|
||||
| Wrangler | dependency của dự án (`npx wrangler`) | Worker chạy local, D1 local, deploy |
|
||||
| Rust | stable (qua `rustup`) | Chỉ để build Tauri desktop / Android |
|
||||
@@ -41,8 +46,8 @@ python -m venv .venv
|
||||
pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
Trên Windows, đặt `PYTHONUTF8=1` (PowerShell: `$env:PYTHONUTF8 = "1"`) để Python đọc và ghi các
|
||||
file catalog dưới dạng UTF-8.
|
||||
Python 3.15 mặc định đọc và ghi file dưới dạng UTF-8 (UTF-8 mode, PEP 686), nên trên Windows không
|
||||
cần đặt `PYTHONUTF8=1` nữa.
|
||||
|
||||
### Kiểm tra (giống `python-ci.yml`)
|
||||
|
||||
@@ -79,8 +84,8 @@ nắm tổng quan tầng dữ liệu.
|
||||
```sh
|
||||
cd webapp
|
||||
npm ci
|
||||
npm run dev # app công khai ở http://localhost:5173 có HMR; /data lấy từ ../data_game
|
||||
npm run build # tsc -b + vite build → webapp/dist/ (app, /admin/, /data, sitemap.xml)
|
||||
npm run dev # app công khai ở http://localhost:5173 có HMR; /data (gói + JSON) dựng từ ../data_game
|
||||
npm run build # tsc -b + vite build → webapp/dist/ (app, /admin/, /data/pack + JSON cũ, sitemap.xml)
|
||||
npm run preview # chạy bản build production (không có Worker)
|
||||
```
|
||||
|
||||
@@ -182,8 +187,9 @@ npm run tauri:dev # cửa sổ native + HMR
|
||||
npm run tauri:build # bộ cài native
|
||||
```
|
||||
|
||||
Các app đọc catalog trực tiếp từ https://freeitchgames.win/data và tải ảnh bìa từ `img.itch.zone`,
|
||||
nên cần internet nhưng không cần Worker local. Yêu cầu theo nền tảng (WebView2 trên Windows 10, Xcode
|
||||
Các app đọc các gói catalog trực tiếp từ https://freeitchgames.win/data/pack và tải ảnh bìa từ
|
||||
`img.itch.zone`, nên cần internet nhưng không cần Worker local (code đổi định dạng gói chỉ chạy được trong
|
||||
`tauri:dev` khi website có định dạng đó đã được deploy). Yêu cầu theo nền tảng (WebView2 trên Windows 10, Xcode
|
||||
CLT trên macOS, `libwebkit2gtk-4.1-dev` trên Debian/Ubuntu) và cách build Android có trong
|
||||
[`webapp/TAURI.md`](../../../webapp/TAURI.md).
|
||||
|
||||
|
||||
+61
-17
@@ -8,11 +8,11 @@ secret: secrets are only ever typed into Cloudflare (`wrangler secret put`) or a
|
||||
|
||||
| Piece | Where | What it does |
|
||||
|---|---|---|
|
||||
| Worker `free-games-itchio-list` | Cloudflare Workers, custom domain `freeitchgames.win` | Serves the built app + `/data` (static assets), `/img` (resized covers), the verification gate (`/api/verify`), `/api/suggest`, `/api/ingest`, `/api/admin/*`, `/admin`, and runs the RSS cron. Config: [`webapp/wrangler.jsonc`](../webapp/wrangler.jsonc). |
|
||||
| Worker `free-games-itchio-list` | Cloudflare Workers, custom domain `freeitchgames.win` | Serves the built app + the catalog packs `/data/pack/*` (static assets), the readable `/data/*.json` to apps 4.0–4.1.1 only ([`legacy-data.ts`](../webapp/worker/legacy-data.ts)), `/img` (resized covers), the verification gate (`/api/verify`), `/api/suggest`, `/api/admin/*`, `/admin`, and runs the RSS cron. Config: [`webapp/wrangler.jsonc`](../webapp/wrangler.jsonc). |
|
||||
| Workers Builds | Cloudflare dashboard → the Worker → Settings → Build | Builds and deploys `main` on every push (data commits included). |
|
||||
| R2 `freeitchgames-thumbs` | Cloudflare R2 | Resized WebP covers (`w160/…`, `w640/…`). |
|
||||
| D1 `freeitchgames` | Cloudflare D1 | Review queue, feed state, audit log, idempotency keys ([`webapp/worker/migrations/`](../webapp/worker/migrations)). |
|
||||
| Access apps "Admin" / "Ingest" | Cloudflare Zero Trust | Login for `/admin` + `/api/admin/*`; service token for `/api/ingest` (browser extension). |
|
||||
| D1 `freeitchgames` | Cloudflare D1 | Review queue, feed state, audit log, automation settings, followed collections ([`webapp/worker/migrations/`](../webapp/worker/migrations); the `idempotency` table of the retired ingest API stays, empty). |
|
||||
| Access app "Admin" | Cloudflare Zero Trust | Login for `/admin` + `/api/admin/*`. |
|
||||
| Turnstile widget | Cloudflare Turnstile | The verification gate in front of the web app (action `gate`) and anti-spam on `/suggest` (action `suggest`). |
|
||||
| Transform Rule "Verified bots skip the gate" | Cloudflare → Rules → Transform Rules | Adds `x-fig-known-bot: <GATE_BOT_TOKEN>` to requests from Cloudflare-verified bots, so search engines and link previews skip the gate (section 1, step 14). |
|
||||
| GitHub App | GitHub → Settings → Developer settings | The Worker's identity for commits (queue approvals, edits, removals). |
|
||||
@@ -24,7 +24,7 @@ secret: secrets are only ever typed into Cloudflare (`wrangler secret put`) or a
|
||||
Do these once (and again with the `-staging` names for the staging Worker).
|
||||
|
||||
1. **Build settings** (Worker → Settings → Build): root directory `webapp`; build command
|
||||
`npm run build`; deploy command `npx wrangler deploy`; build variable `NODE_VERSION=24` (no
|
||||
`npm run build`; deploy command `npx wrangler deploy`; build variable `NODE_VERSION=26` (no
|
||||
`VITE_CF_BEACON_TOKEN`: the zone injects Web Analytics, step 10). The staging Worker, if you use
|
||||
it, is deployed by hand (`npx wrangler deploy --env staging`); turn its builds for non-production
|
||||
branches **off** (otherwise every push to `main` also gets a preview build there).
|
||||
@@ -37,7 +37,8 @@ Do these once (and again with the `-staging` names for the staging Worker).
|
||||
(and `freeitchgames-thumbs-staging`). A deploy fails while its bucket is missing.
|
||||
4. **Images → Transformations**: enable for the zone; allowed source origins: `img.itch.zone` only.
|
||||
5. **D1**: production `freeitchgames` exists (created 2026-09-18 in APAC with
|
||||
`npx wrangler d1 create freeitchgames`; its id is in `wrangler.jsonc`; `0001_init.sql` applied).
|
||||
`npx wrangler d1 create freeitchgames`; its id is in `wrangler.jsonc`; `0001_init.sql` applied,
|
||||
`0002_automation.sql` (4.4: ingest outcomes, settings, followed collections) applied before the 4.4.0 merge).
|
||||
Staging: `cd webapp && npx wrangler d1 create freeitchgames-staging`, put the printed id in
|
||||
`env.staging.d1_databases`, then `npx wrangler d1 migrations apply freeitchgames-staging --remote --env staging`.
|
||||
Apply every new migration by hand (`npx wrangler d1 migrations apply freeitchgames --remote`)
|
||||
@@ -48,17 +49,15 @@ Do these once (and again with the `-staging` names for the staging Worker).
|
||||
- Access → Applications → Self-hosted **"Admin"**: `freeitchgames.win/admin`,
|
||||
`freeitchgames.win/admin/*`, `freeitchgames.win/api/admin/*`; policy *Allow* → your email(s);
|
||||
session 24 h. Copy its **AUD** tag.
|
||||
- Self-hosted **"Ingest"**: `freeitchgames.win/api/ingest`; policy *Service Auth* → a service token
|
||||
(Access → Service credentials) named e.g. `itch-f2p-extension`, with an expiry; copy the AUD tag.
|
||||
The extension sends `CF-Access-Client-Id` / `CF-Access-Client-Secret`.
|
||||
- 4.4.1 removed the browser extension's ingest API: delete the old self-hosted **"Ingest"** app
|
||||
(`freeitchgames.win/api/ingest`) and its service token (Access → Service credentials) if they still exist.
|
||||
- Staging: one self-hosted app covering all of `staging.freeitchgames.win`.
|
||||
7. **Turnstile**: add a widget (Managed) for `freeitchgames.win` and `staging.freeitchgames.win`. The
|
||||
same widget serves the verification gate (action `gate`) and the Suggest form (action `suggest`);
|
||||
the Worker checks the token, the hostname and the action, so a token from one can't be used for
|
||||
the other.
|
||||
8. **Fill in `wrangler.jsonc` vars** (not secret) and commit: `ACCESS_TEAM_DOMAIN`
|
||||
(`https://<team>.cloudflareaccess.com`, no trailing slash), `ACCESS_AUD_ADMIN`, `ACCESS_AUD_INGEST`,
|
||||
`GH_APP_ID`, `GH_APP_INSTALLATION_ID`, `TURNSTILE_SITEKEY` — in both the top level and
|
||||
(`https://<team>.cloudflareaccess.com`, no trailing slash), `ACCESS_AUD_ADMIN`, `GH_APP_ID`, `GH_APP_INSTALLATION_ID`, `TURNSTILE_SITEKEY` — in both the top level and
|
||||
`env.staging`. Then `npm run cf:types`. The production values are filled in (2026-09-18). Change
|
||||
them in the file, not only in the dashboard: every deploy — so every data commit — replaces the
|
||||
Worker's plain-text variables with the file's values. Secrets are not affected.
|
||||
@@ -83,9 +82,25 @@ Do these once (and again with the `-staging` names for the staging Worker).
|
||||
12. **Zone settings**: SSL/TLS Full (strict), Always Use HTTPS, minimum TLS 1.2, TLS 1.3 on; Smart
|
||||
Tiered Cache; Browser Cache TTL "Respect existing headers"; Rocket Loader and Email Obfuscation
|
||||
**off** (they rewrite HTML and break the CSP); WAF managed rules on; one rate-limiting rule for
|
||||
`/api/*`; Bot Fight Mode **off** (it can't be skipped for the extension or the apps; the
|
||||
`/api/*`; Bot Fight Mode **off** (it can't be skipped for the apps; the
|
||||
verification gate covers the web app instead). Keep HSTS
|
||||
`preload` only if every subdomain (including `staging`) will always be HTTPS.
|
||||
- Measured 2026-10-03: TLS 1.0/1.1 are refused, TLS 1.3 and HTTP/3 answer, `http://` redirects
|
||||
(301), and the zone sends `Strict-Transport-Security: max-age=31536000; includeSubDomains;
|
||||
preload`. The domain is **not** on the browsers' HSTS preload list (hstspreload.org); submitting
|
||||
it is optional and slow to undo.
|
||||
- **Smart Tiered Cache** is free, so leave it on, but it barely matters here: static assets (the
|
||||
app and `/data/pack/*`) have Cloudflare's own tiered asset cache, and `/img` uses the Worker's
|
||||
Cache API (per data centre, never tiered) plus R2. Only cacheable Worker `fetch()` subrequests
|
||||
use it — in practice the img.itch.zone fallback fetch. No Region Hint is needed for
|
||||
`api.github.com` (it shows up as an "origin" because the Worker calls it; those calls aren't
|
||||
cacheable).
|
||||
- Don't turn on **Workers Cache** (`cache.enabled` in `wrangler.jsonc`): it bills every
|
||||
static-asset request as a Worker request and would serve cached `/img` responses without running
|
||||
the verification gate.
|
||||
- Compression is automatic (zstd / Brotli / gzip, picked per browser) for JSON and the app files.
|
||||
The `.bin` packs are deflated at build time and served as `application/octet-stream`, which
|
||||
Cloudflare does not compress again.
|
||||
13. **Notifications**: enable the Workers Builds failure notification (independent of
|
||||
`deploy-status.yml`).
|
||||
14. **Verification gate** ([`webapp/worker/gate.ts`](../webapp/worker/gate.ts)). A visitor must pass a
|
||||
@@ -107,7 +122,8 @@ Do these once (and again with the `-staging` names for the staging Worker).
|
||||
5. `openssl rand -base64 48 | npx wrangler secret put GATE_SECRET` — the gate is on.
|
||||
6. Smoke test: `curl -sI https://freeitchgames.win/` shows `x-robots-tag: noindex, nofollow` and
|
||||
`cache-control: no-store` (the verification page); `/img/160/x` answers 403;
|
||||
`/data/index.json` answers 200 with `access-control-allow-origin: *`;
|
||||
`/data/pack/manifest-v1.json` answers 200 with `access-control-allow-origin: *` (the catalog
|
||||
checks in section 4 cover the rest);
|
||||
`curl -sI -A 'Googlebot/2.1' https://freeitchgames.win/` answers 503 (only verified bots, via
|
||||
the rule, get through); a real browser passes and lands on the page it opened; Search Console →
|
||||
URL Inspection → *Test live URL* renders the real app. Watch Security → Events and Search
|
||||
@@ -132,8 +148,8 @@ Do these once (and again with the `-staging` names for the staging Worker).
|
||||
policies don't claim them). Rulesets: target the default branch (block deletion and force
|
||||
pushes); protect tags `v*`. Merge PRs with squash or a merge commit — never fast-forward.
|
||||
3. **Secrets**: keep `DISCORD_CI_WEBHOOK`, the release webhooks and the Android keystore secrets.
|
||||
`GH_TOKEN`, `TELEGRAM_BOT_TOKEN` and the three `DISCUSSION_*` secrets are still set but no workflow
|
||||
uses them any more (checked 2026-09-24): delete them.
|
||||
`GH_TOKEN`, `TELEGRAM_BOT_TOKEN` and the three `DISCUSSION_*` secrets, which no workflow used any
|
||||
more, were deleted on 2026-10-16.
|
||||
4. **Signing key**: the old key `03F965C2E2DB5C6B` was exposed; sign with the replacement key.
|
||||
|
||||
## 3. Releasing a version
|
||||
@@ -158,14 +174,42 @@ Do these once (and again with the `-staging` names for the staging Worker).
|
||||
- **Mass-change alert** (Discord, "mass_change=true"): itch.io probably changed its page markup. Look at
|
||||
a few flagged games on itch.io. If the parser is wrong, fix `scripts/scraper.py`; if the removals
|
||||
are real, dispatch **Refresh catalog** with `allow_mass_removal`.
|
||||
- **Removal / takedown request**: `/admin` → Catalog → the game → Remove (the reason is public). A
|
||||
wrongly removed game: Catalog → Removed → Restore.
|
||||
- **Removal / takedown request**: `/admin` → Catalog → the game → Remove… (the reason is public; the
|
||||
presets get a translated label on the Removed page). Several games: tick up to 100 search results, or
|
||||
paste their URLs under **Remove by list** — one commit either way. A wrongly removed game: Catalog →
|
||||
Removed → Restore.
|
||||
- **Review queue**: `/admin` → Review queue. "Approve with override" is needed for games that were
|
||||
removed before.
|
||||
removed before. Selections of any size go out in batches of 200 (one commit each).
|
||||
- **Adding a list of games**: `/admin` → Add games. Paste any text (or load a file, or read an itch.io
|
||||
collection), press **Check** to see what Add would do, then **Add**. **Recent batches** follows each
|
||||
game through the ingest (added / waiting / dropped with the reason); the ingest runs right after each
|
||||
commit and handles about 650 games per 45-minute run, the rest waits in the queue.
|
||||
- **Automatic rules**: `/admin` → Settings. Auto-approve (on by default for RSS and Suggest, after 24 h)
|
||||
queues pending items without a flag that nobody rejected; it only considers items found after the
|
||||
moment it was switched on ("Include older pending items" lifts that), at most 100 per cron run.
|
||||
**Followed collections** are read in turn (≤ 30 pages per run, one a second); their new free games
|
||||
are queued directly, removed or flagged ones go to the review queue. Settings also shows what the
|
||||
last cron run did (Workers logs are sampled), and a collection whose page could not be read (status
|
||||
422: itch.io changed its markup) adds nothing until `worker/collections.ts` is fixed.
|
||||
- **Reviewing new games**: `/admin` → New games (added in the last 7 / 30 days): set 18+ and the virus
|
||||
check for up to 100 games at once. **Activity** lists every admin action and automatic run with its
|
||||
commit.
|
||||
- **Build failed** (Discord from `deploy-status.yml`): open the build log linked in the message; the
|
||||
site keeps serving the previous deployment meanwhile.
|
||||
- **New D1 migration**: add `webapp/worker/migrations/000N_*.sql` (never edit applied ones), apply it
|
||||
to staging and production by hand, then deploy.
|
||||
- **Catalog delivery** (after a release that touches it, or when the site shows no games):
|
||||
```sh
|
||||
B=https://freeitchgames.win/data
|
||||
curl -sI $B/pack/manifest-v1.json # 200, application/json, max-age=60, ACAO *
|
||||
H=$(curl -s $B/pack/manifest-v1.json | node -p "JSON.parse(require('fs').readFileSync(0)).chunks[0].main")
|
||||
curl -sI -H 'Accept-Encoding: br, gzip' $B/pack/$H # 200, octet-stream, no content-encoding, immutable
|
||||
curl -sI $B/pack/0000000000000000.bin # 404, no-store (answered by the Worker)
|
||||
curl -sI $B/index.json # 410, vary: Origin (browsers and scripts)
|
||||
curl -sI -H 'Origin: tauri://localhost' $B/index.json # 200 (apps 4.0–4.1.1)
|
||||
```
|
||||
`npx wrangler tail` shows the `legacy-data: 410` lines with the requesting `Origin`: an installed
|
||||
app that sends an unexpected one would show up there.
|
||||
- **Verification gate misbehaving** (visitors stuck on the check, crawl errors in Search Console):
|
||||
`npx wrangler secret delete GATE_SECRET` turns it off at once; then look at `npx wrangler tail` and
|
||||
`curl -s https://freeitchgames.win/api/verify`.
|
||||
+1
-1
@@ -7,7 +7,7 @@
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py314"
|
||||
target-version = "py315"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "W", "UP", "B", "BLE", "S113"]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
-r requirements.txt
|
||||
ruff>=0.16.8
|
||||
pytest>=9.1
|
||||
ruff>=0.17.1
|
||||
pytest>=9.1.1
|
||||
vulture>=2.16
|
||||
pillow>=12.3.0
|
||||
pillow>=13.0.0
|
||||
@@ -1,2 +1,3 @@
|
||||
requests>=2.34.2,<3
|
||||
beautifulsoup4>=4.15.0,<5
|
||||
urllib3>=2.8.0,<3
|
||||
@@ -12,6 +12,9 @@ always works on the latest data. Idempotent: applying the same patch twice
|
||||
- *_state: merged into scripts/state/*.json (null drops an entry); for
|
||||
refresh_state the newer `checked` wins and the newest `full`
|
||||
stamp from either side is kept
|
||||
- ingest_outcomes: merged into scripts/state/ingest_outcomes.json (the newer
|
||||
`at` wins; entries more than 90 days older than the newest one,
|
||||
and all but the newest 5,000, are dropped)
|
||||
- unlog / unblock_remove: an unblocked (restored) game that was re-added
|
||||
leaves the deleted log and scripts/state/unblocked.json; one whose
|
||||
ingest ended otherwise leaves only the allow-list
|
||||
@@ -32,6 +35,7 @@ Usage: python scripts/apply_patch.py PATCH.json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from canonical import canonicalize
|
||||
@@ -46,9 +50,12 @@ DELETED_LOG = "scripts/deleted_games.json"
|
||||
TEMP_LINK = "scripts/temp_link.json"
|
||||
REFRESH_STATE = "scripts/state/refresh_state.json"
|
||||
INGEST_STATE = "scripts/state/ingest_state.json"
|
||||
INGEST_OUTCOMES = "scripts/state/ingest_outcomes.json"
|
||||
UNBLOCKED = "scripts/state/unblocked.json"
|
||||
MASS_REMOVAL_FLOOR = 25
|
||||
MASS_REMOVAL_SHARE = 0.05
|
||||
OUTCOMES_DAYS = 90
|
||||
OUTCOMES_KEEP = 5000
|
||||
|
||||
|
||||
def mass_removal_limit(patch: dict, total: int) -> int:
|
||||
@@ -59,6 +66,31 @@ def mass_removal_limit(patch: dict, total: int) -> int:
|
||||
return limit
|
||||
|
||||
|
||||
def merge_outcomes(current: dict, delta: dict) -> dict:
|
||||
"""ingest_outcomes.json after a patch: per URL the newer `at` wins.
|
||||
|
||||
Pruned relative to the newest entry, not the clock, so applying the same
|
||||
patch twice gives the same file.
|
||||
"""
|
||||
merged = {
|
||||
url: e for url, e in current.items() if isinstance(e, dict) and isinstance(e.get("at"), str)
|
||||
}
|
||||
for url, entry in delta.items():
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get("at"), str):
|
||||
continue
|
||||
old = merged.get(url)
|
||||
if old is None or entry["at"] >= old["at"]:
|
||||
merged[url] = entry
|
||||
if not merged:
|
||||
return {}
|
||||
newest = datetime.fromisoformat(max(e["at"] for e in merged.values()).replace("Z", "+00:00"))
|
||||
cutoff = (newest - timedelta(days=OUTCOMES_DAYS)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
recent = sorted(
|
||||
((e["at"], url) for url, e in merged.items() if e["at"] >= cutoff), reverse=True
|
||||
)[:OUTCOMES_KEEP]
|
||||
return {url: merged[url] for _, url in recent}
|
||||
|
||||
|
||||
def _queue_key(url: str) -> str:
|
||||
return canonicalize(url) or url
|
||||
|
||||
@@ -161,6 +193,11 @@ def apply(patch: dict) -> dict:
|
||||
state = {u: e for u, e in state.items() if u in live_urls}
|
||||
save_state_map(path, state)
|
||||
|
||||
outcomes = patch.get("ingest_outcomes", {})
|
||||
if outcomes:
|
||||
current = load_json(INGEST_OUTCOMES, default={})
|
||||
save_state_map(INGEST_OUTCOMES, merge_outcomes(current, outcomes))
|
||||
|
||||
unlog = {u for u in patch.get("unlog", []) if isinstance(u, str)}
|
||||
finished = unlog | {u for u in patch.get("unblock_remove", []) if isinstance(u, str)}
|
||||
if unlog:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
canonical.py — One canonical form for itch.io game URLs.
|
||||
|
||||
Every ingest channel (extension, admin, Suggest form, RSS, workflow input) and
|
||||
Every ingest channel (admin, Suggest form, RSS, collections, workflow input) and
|
||||
the pipeline itself compare URLs through canonicalize(), so the same game can
|
||||
never be queued twice under two spellings. The Worker ports this function; both
|
||||
implementations are tested against tests/fixtures/url_vectors.json.
|
||||
|
||||
@@ -616,5 +616,185 @@
|
||||
"name": "Locksmith",
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-09-27T08:34:34Z"
|
||||
},
|
||||
{
|
||||
"url": "https://g0refreak.itch.io/yume-tenshi",
|
||||
"name": "Yume Tenshi 夢天使",
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-09-30T02:58:58Z"
|
||||
},
|
||||
{
|
||||
"url": "https://mor10th.itch.io/stop-at-the-gas-station",
|
||||
"name": "Stop at the Gas Station",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-09-30T02:59:02Z"
|
||||
},
|
||||
{
|
||||
"url": "https://sdf0aabd.itch.io/milo-gets-silly-when-he-is-in-love",
|
||||
"name": "Milo gets silly when he is in love!",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-02T02:57:50Z"
|
||||
},
|
||||
{
|
||||
"url": "https://abmanstudio.itch.io/dont-leave-chapter1",
|
||||
"name": "DON'T LEAVE — Chapter 1",
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-10-03T03:07:42Z"
|
||||
},
|
||||
{
|
||||
"url": "https://sdf0aabd.itch.io/my-personal-hater",
|
||||
"name": "my personal hater",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-06T02:59:07Z"
|
||||
},
|
||||
{
|
||||
"url": "https://lighttriadgames.itch.io/break-the-pinatas",
|
||||
"name": "Break the Piñatas",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-06T02:59:10Z"
|
||||
},
|
||||
{
|
||||
"url": "https://mauricioghy.itch.io/minecraft-caverns",
|
||||
"name": "Minecraft Caverns",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-06T02:59:13Z"
|
||||
},
|
||||
{
|
||||
"url": "https://burgerkingv.itch.io/swallowed",
|
||||
"name": "SWALLOWED",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-09T03:01:17Z"
|
||||
},
|
||||
{
|
||||
"url": "https://v0id2.itch.io/burrow",
|
||||
"name": "Burrow",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:17Z"
|
||||
},
|
||||
{
|
||||
"url": "https://wolf-hal.itch.io/sprunki",
|
||||
"name": "Sprunki",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:19Z"
|
||||
},
|
||||
{
|
||||
"url": "https://sekarsarasu1981.itch.io/my-raising-diary",
|
||||
"name": "My Raising Diary",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:23Z"
|
||||
},
|
||||
{
|
||||
"url": "https://sreejithnm78.itch.io/unquiet-grey",
|
||||
"name": "Unquiet Grey",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:27Z"
|
||||
},
|
||||
{
|
||||
"url": "https://elinzeiart-z.itch.io/salvation-of-the-righteous",
|
||||
"name": "Salvation of the Righteous",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:31Z"
|
||||
},
|
||||
{
|
||||
"url": "https://esgeetha5.itch.io/enter-furry-world",
|
||||
"name": "Enter Furry World",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:34Z"
|
||||
},
|
||||
{
|
||||
"url": "https://kanagarajv021.itch.io/endless-labyrinth-of-night-roads",
|
||||
"name": "Endless Labyrinth of Night Roads",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:38Z"
|
||||
},
|
||||
{
|
||||
"url": "https://coren-baili.itch.io/six-rules-the-visual-novel",
|
||||
"name": "Six Rules: The Visual Novel",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-11T03:02:40Z"
|
||||
},
|
||||
{
|
||||
"url": "https://everium.itch.io/underthethorns",
|
||||
"name": "Under the Thorns",
|
||||
"reason": "Game page is password-protected",
|
||||
"deleted_at": "2026-10-12T03:08:14Z"
|
||||
},
|
||||
{
|
||||
"url": "https://ronhelleredfn-boop.itch.io/bb-blackwood",
|
||||
"name": "B&B Blackwood",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-12T03:08:17Z"
|
||||
},
|
||||
{
|
||||
"url": "https://toro-comicss.itch.io/emergency-monster-team-chapter-0",
|
||||
"name": "EMERGENCY! Monster Team Chapter 0",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-12T03:08:20Z"
|
||||
},
|
||||
{
|
||||
"url": "https://life-crises.itch.io/your-redacted-boyfriend",
|
||||
"name": "your [REDACTED] boyfriend",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-12T03:08:22Z"
|
||||
},
|
||||
{
|
||||
"url": "https://antoine-aubin.itch.io/a-wave-that-i-a-furry-visual-novel",
|
||||
"name": "A wave that I . . . ( A furry visual novel)",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-12T03:08:25Z"
|
||||
},
|
||||
{
|
||||
"url": "https://imrank-seo.itch.io/stranded-with-you",
|
||||
"name": "Stranded with You",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-12T03:08:28Z"
|
||||
},
|
||||
{
|
||||
"url": "https://akaerocore.itch.io/raising-a-happy-neet",
|
||||
"name": "Raising a Happy NEET",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-12T03:08:31Z"
|
||||
},
|
||||
{
|
||||
"url": "https://chandhira2604.itch.io/sun-stars",
|
||||
"name": "Sun & Stars",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-18T02:53:39Z"
|
||||
},
|
||||
{
|
||||
"url": "https://ronnafemto.itch.io/recipe-frenzy",
|
||||
"name": "Recipe Frenzy",
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-10-18T02:53:43Z"
|
||||
},
|
||||
{
|
||||
"url": "https://bluedragondreaming.itch.io/elemental-harmony",
|
||||
"name": "Elemental Harmony",
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-10-19T02:57:14Z"
|
||||
},
|
||||
{
|
||||
"url": "https://writer31.itch.io/say-youll-forgive-me",
|
||||
"name": "Say You'll Forgive Me",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-19T02:57:18Z"
|
||||
},
|
||||
{
|
||||
"url": "https://lguidolin.itch.io/one-more-saturday",
|
||||
"name": "One More Saturday",
|
||||
"reason": "Game became paid",
|
||||
"deleted_at": "2026-10-19T02:57:22Z"
|
||||
},
|
||||
{
|
||||
"url": "https://meta-and-flash.itch.io/flash-taggers",
|
||||
"name": "🪵Twig Tagg🪵",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-20T02:55:15Z"
|
||||
},
|
||||
{
|
||||
"url": "https://baldigaming99.itch.io/geometry-dash-220922091-apkandpc",
|
||||
"name": "Geometry Dash 2.209/2.2091 (APK and PC).",
|
||||
"reason": "Game page no longer exists (HTTP 404)",
|
||||
"deleted_at": "2026-10-21T02:55:57Z"
|
||||
}
|
||||
]
|
||||
+6
-2
@@ -3,8 +3,8 @@ patch.py — The hand-off format between pipeline scans and the commit step.
|
||||
|
||||
Scanners (refresh.py, update_info.py) never write the catalog directly. They
|
||||
emit a patch; apply_patch.py applies it idempotently to whatever `main` looks
|
||||
like at push time, so concurrent writers (other workflows, the admin Worker,
|
||||
the browser extension) never overwrite each other.
|
||||
like at push time, so concurrent writers (other workflows, the admin Worker
|
||||
and its cron) never overwrite each other.
|
||||
|
||||
Patch v1:
|
||||
{
|
||||
@@ -16,6 +16,8 @@ Patch v1:
|
||||
"removals": [{"url", "name", "reason", "deleted_at"}],
|
||||
"refresh_state": {"<url>": {...} | null}, # null = drop entry
|
||||
"ingest_state": {"<url>": {...} | null},
|
||||
"ingest_outcomes": {"<url>": {"outcome", "at"}}, # how a queued link ended (added, paid,
|
||||
# dead, private, duplicate, deleted, gave_up)
|
||||
"queue_add": ["<url>", ...], # append to temp_link.json if absent
|
||||
"queue_remove": ["<url>", ...], # drop from temp_link.json
|
||||
"unlog": ["<url>", ...], # unblocked + re-added: drop from the
|
||||
@@ -42,6 +44,7 @@ def new_patch(kind: str) -> dict:
|
||||
"removals": [],
|
||||
"refresh_state": {},
|
||||
"ingest_state": {},
|
||||
"ingest_outcomes": {},
|
||||
"queue_add": [],
|
||||
"queue_remove": [],
|
||||
"unlog": [],
|
||||
@@ -71,6 +74,7 @@ def is_empty(patch: dict) -> bool:
|
||||
"removals",
|
||||
"refresh_state",
|
||||
"ingest_state",
|
||||
"ingest_outcomes",
|
||||
"queue_add",
|
||||
"queue_remove",
|
||||
"unlog",
|
||||
|
||||
+32
-9
@@ -10,8 +10,9 @@ then the least recently checked — about 1/7 of the catalog by default, so
|
||||
every game is re-checked weekly. From that single page it derives:
|
||||
- alive: 404/410 → "dead" strike
|
||||
- price: paid → "paid" strike
|
||||
- rating, rating_count, status (+ backfills release_date / thumbnail /
|
||||
updated_at when missing)
|
||||
- access: itch.io's password form instead of the game → "private" strike
|
||||
- rating, rating_count, status, ai_disclosure / ai_content / accessibility
|
||||
(+ backfills release_date / thumbnail / updated_at when missing)
|
||||
A game is removed only when the same strike is seen again at least
|
||||
MIN_STRIKE_GAP after it was first seen (a single 404 or a flaky price widget
|
||||
no longer deletes a game). If a run would remove, or newly strike, more than
|
||||
@@ -51,12 +52,14 @@ from scraper import (
|
||||
NA,
|
||||
Pacer,
|
||||
Page,
|
||||
ai_and_accessibility,
|
||||
create_session,
|
||||
extract_thumbnail,
|
||||
fetch_page,
|
||||
find_title,
|
||||
has_info_panel,
|
||||
is_free_game,
|
||||
is_password_protected,
|
||||
now_iso,
|
||||
parse_game,
|
||||
parse_info_table,
|
||||
@@ -66,6 +69,9 @@ STATE_FILE = "scripts/state/refresh_state.json"
|
||||
|
||||
# User-editable annotations: never overwritten by any scrape.
|
||||
PRESERVE_FIELDS = ("safe_virus", "notes", "nsfw")
|
||||
# A full re-scrape keeps these when the page no longer shows them: itch.io shows
|
||||
# the "Published" row only for a few weeks, and a game's cover doesn't go away.
|
||||
KEEP_KNOWN_FIELDS = ("release_date", "thumbnail")
|
||||
|
||||
DEAD_CODES = {404, 410}
|
||||
CHECKPOINT_EVERY = 50
|
||||
@@ -76,12 +82,18 @@ MASS_FLOOR = 10
|
||||
MASS_SHARE = 0.10
|
||||
|
||||
REASON_PAID = "Game became paid"
|
||||
REASON_PRIVATE = "Game page is password-protected"
|
||||
|
||||
|
||||
def reason_dead(code: int) -> str:
|
||||
return f"Game page no longer exists (HTTP {code})"
|
||||
|
||||
|
||||
def removal_kind(reason: str) -> str:
|
||||
"""The strike kind ("dead", "paid", "private") a removal reason was written for."""
|
||||
return {REASON_PAID: "paid", REASON_PRIVATE: "private"}.get(reason, "dead")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Target selection
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -152,11 +164,13 @@ def refresh_updates(game: dict, soup) -> dict:
|
||||
if not has_info_panel(soup):
|
||||
return updates
|
||||
info = parse_info_table(soup)
|
||||
for field, value in (
|
||||
("status", info.get("Status", NA)),
|
||||
("rating", info.get("Rating", NA)),
|
||||
("rating_count", info.get("RatingCount", NA)),
|
||||
):
|
||||
current = {
|
||||
"status": info.get("Status", NA),
|
||||
"rating": info.get("Rating", NA),
|
||||
"rating_count": info.get("RatingCount", NA),
|
||||
**ai_and_accessibility(info),
|
||||
}
|
||||
for field, value in current.items():
|
||||
if game.get(field) != value:
|
||||
updates[field] = value
|
||||
if game.get("release_date", NA) == NA:
|
||||
@@ -176,6 +190,9 @@ def full_updates(game: dict, soup) -> dict:
|
||||
"""Every scraper-owned field that differs from a fresh scrape."""
|
||||
fresh = parse_game(soup, game["url"])
|
||||
fresh.pop("is_free", None)
|
||||
for field in KEEP_KNOWN_FIELDS:
|
||||
if fresh.get(field) == NA and game.get(field, NA) != NA:
|
||||
fresh[field] = game[field]
|
||||
return {
|
||||
k: v
|
||||
for k, v in fresh.items()
|
||||
@@ -218,6 +235,8 @@ def check_game(
|
||||
return strike("dead", reason_dead(page.status))
|
||||
|
||||
soup = page.soup
|
||||
if soup is not None and is_password_protected(soup):
|
||||
return strike("private", REASON_PRIVATE)
|
||||
if soup is None or find_title(soup) == NA:
|
||||
entry["fails"] = entry.get("fails", 0) + 1
|
||||
return "parse_fail", {}, None, entry
|
||||
@@ -272,7 +291,10 @@ def _write_outputs(stats: dict) -> None:
|
||||
("Checked", stats["checked"]),
|
||||
("Fields changed (games)", stats["changed"]),
|
||||
("New strikes", stats["strikes"]),
|
||||
("Removed (dead / paid)", f"{stats['removed_dead']} / {stats['removed_paid']}"),
|
||||
(
|
||||
"Removed (dead / paid / private)",
|
||||
f"{stats['removed_dead']} / {stats['removed_paid']} / {stats['removed_private']}",
|
||||
),
|
||||
("Removals withheld (mass-change guard)", stats["withheld"]),
|
||||
("Errors / parse failures", f"{stats['errors']} / {stats['parse_fail']}"),
|
||||
("HTTP 429/503", stats["http_429"]),
|
||||
@@ -300,7 +322,7 @@ def keep_struck(patch: dict, state: dict, removals: list[dict], stats: dict, mar
|
||||
entry["full"] = removal["deleted_at"]
|
||||
patch["refresh_state"][removal["url"]] = entry
|
||||
stats["removed"] -= 1
|
||||
stats["removed_" + ("paid" if removal["reason"] == REASON_PAID else "dead")] -= 1
|
||||
stats["removed_" + removal_kind(removal["reason"])] -= 1
|
||||
patch["removals"] = [r for r in patch["removals"] if r["url"] not in urls]
|
||||
|
||||
|
||||
@@ -416,6 +438,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"removed",
|
||||
"removed_dead",
|
||||
"removed_paid",
|
||||
"removed_private",
|
||||
"errors",
|
||||
"parse_fail",
|
||||
"http_429",
|
||||
|
||||
+92
-28
@@ -5,8 +5,10 @@ Provides:
|
||||
- Session creation (honest User-Agent, retries on 5xx only)
|
||||
- fetch_page(): one GET with explicit 429 / Retry-After reporting
|
||||
- Pacer: polite delays, batch pauses, back-off after a 429
|
||||
- Free/paid detection, info table parsing (all fields → N/A on missing)
|
||||
- Description, thumbnail, NSFW extraction
|
||||
- Free/paid detection, info table parsing (all fields → N/A on missing),
|
||||
including the AI-disclosure and accessibility rows itch.io added in 2025–26
|
||||
- Description (the creator's tagline, else the first sentence), thumbnail,
|
||||
NSFW extraction
|
||||
- parse_game(): full single-game record from a fetched page
|
||||
"""
|
||||
|
||||
@@ -207,10 +209,15 @@ def is_free_game(soup: BeautifulSoup) -> bool:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _text(tag) -> str:
|
||||
"""Visible text with whitespace runs collapsed (a page may break lines inside a link)."""
|
||||
return " ".join(tag.get_text().split())
|
||||
|
||||
|
||||
def _safe_text(tag, fallback: str = NA) -> str:
|
||||
if tag is None:
|
||||
return fallback
|
||||
text = tag.get_text(strip=True)
|
||||
text = _text(tag)
|
||||
return text if text else fallback
|
||||
|
||||
|
||||
@@ -229,22 +236,40 @@ def has_info_panel(soup: BeautifulSoup) -> bool:
|
||||
return bool(wrapper and wrapper.find("table"))
|
||||
|
||||
|
||||
def is_password_protected(soup: BeautifulSoup) -> bool:
|
||||
"""itch.io's "A password is required to view this page" form instead of the game (HTTP 200)."""
|
||||
return bool(
|
||||
soup.find("div", class_="game_password_page")
|
||||
or soup.find("input", attrs={"name": "game_password"})
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Info-table parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Fields where itch.io lists multiple <a> links → store as JSON array
|
||||
_LIST_FIELDS = {"Tags", "Platforms", "Languages", "Inputs", "Made with"}
|
||||
_LIST_FIELDS = {"Tags", "Platforms", "Languages", "Inputs", "Made with", "Accessibility"}
|
||||
|
||||
# Date rows: prefer abbr@title ("23 April 2021 @ 19:30 UTC") over the relative text
|
||||
_DATE_FIELDS = {"Release date", "Published", "Updated"}
|
||||
|
||||
# Stored when the "Content" row says "No generative AI was used" (/games/tag-no-ai).
|
||||
AI_NONE = "No AI"
|
||||
|
||||
|
||||
def _links_to(a, slug: str) -> bool:
|
||||
return str(a.get("href", "")).split("?", 1)[0].rstrip("/").endswith(f"/games/{slug}")
|
||||
|
||||
|
||||
def parse_info_table(soup: BeautifulSoup) -> dict[str, Any]:
|
||||
"""Parse the right-side info panel into a dict.
|
||||
|
||||
Multi-value fields (Tags, Platforms, etc.) → list[str]
|
||||
Single-value fields (Genre, Status, etc.) → str
|
||||
"AI Disclosure" ("AI Assisted, Graphics, Text") → the classification as
|
||||
"AI Disclosure", what AI was used for as "AI content" (list); a "Content"
|
||||
row linking to /games/tag-no-ai also sets "No AI" = True.
|
||||
Missing fields are NOT inserted — caller handles defaults.
|
||||
"""
|
||||
info: dict[str, Any] = {}
|
||||
@@ -261,7 +286,7 @@ def parse_info_table(soup: BeautifulSoup) -> dict[str, Any]:
|
||||
if len(tds) < 2:
|
||||
continue
|
||||
|
||||
key = tds[0].get_text(strip=True)
|
||||
key = _text(tds[0])
|
||||
value_td = tds[1]
|
||||
|
||||
if key in _DATE_FIELDS:
|
||||
@@ -269,8 +294,7 @@ def parse_info_table(soup: BeautifulSoup) -> dict[str, Any]:
|
||||
if abbr and abbr.get("title"):
|
||||
info[key] = abbr["title"]
|
||||
else:
|
||||
text = value_td.get_text(strip=True)
|
||||
info[key] = text if text else NA
|
||||
info[key] = _safe_text(value_td)
|
||||
continue
|
||||
|
||||
# --- Rating: extract from itemprop attributes ---
|
||||
@@ -288,37 +312,69 @@ def parse_info_table(soup: BeautifulSoup) -> dict[str, Any]:
|
||||
|
||||
# --- Multi-value fields → list of strings ---
|
||||
links = value_td.find_all("a")
|
||||
texts = [_text(a) for a in links]
|
||||
if key in _LIST_FIELDS:
|
||||
info[key] = [a.get_text(strip=True) for a in links] if links else []
|
||||
info[key] = texts
|
||||
continue
|
||||
|
||||
# --- AI Disclosure: the classification first, then what AI was used for ---
|
||||
if key == "AI Disclosure":
|
||||
info[key] = texts[0] if texts else _safe_text(value_td)
|
||||
info["AI content"] = texts[1:]
|
||||
continue
|
||||
|
||||
if key == "Content" and any(_links_to(a, "tag-no-ai") for a in links):
|
||||
info["No AI"] = True
|
||||
|
||||
# --- Genre: use only the FIRST value (primary genre) ---
|
||||
if key == "Genre":
|
||||
if links:
|
||||
info[key] = links[0].get_text(strip=True)
|
||||
else:
|
||||
text = value_td.get_text(strip=True)
|
||||
info[key] = text if text else NA
|
||||
info[key] = texts[0] if texts else _safe_text(value_td)
|
||||
continue
|
||||
|
||||
# --- Generic single-value: prefer link texts joined, fallback plain ---
|
||||
if links:
|
||||
value = ", ".join(a.get_text(strip=True) for a in links)
|
||||
else:
|
||||
value = value_td.get_text(strip=True)
|
||||
|
||||
value = ", ".join(texts) if links else _text(value_td)
|
||||
info[key] = value if value else NA
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def ai_and_accessibility(info: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The stored fields from itch.io's newer rows (the daily refresh keeps them current).
|
||||
|
||||
ai_disclosure: "AI Assisted" (the "AI Disclosure" row), "No AI" (the
|
||||
"Content" row's /games/tag-no-ai) or N/A — the creator has not said.
|
||||
ai_content: what AI was used for ("Graphics", "Text", "Code", "Sounds").
|
||||
"""
|
||||
return {
|
||||
"ai_disclosure": info.get("AI Disclosure") or (AI_NONE if info.get("No AI") else NA),
|
||||
"ai_content": info.get("AI content", []),
|
||||
"accessibility": info.get("Accessibility", []),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Description
|
||||
# ---------------------------------------------------------------------------
|
||||
_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'“(])")
|
||||
DESCRIPTION_MAX = 200
|
||||
|
||||
|
||||
def extract_description(soup: BeautifulSoup) -> str:
|
||||
def _cap(text: str) -> str:
|
||||
"""`text`, cut on a word and marked with "..." when longer than DESCRIPTION_MAX."""
|
||||
if len(text) <= DESCRIPTION_MAX:
|
||||
return text
|
||||
cut = text[: DESCRIPTION_MAX - 3]
|
||||
return (cut.rsplit(" ", 1)[0] if " " in cut else cut) + "..."
|
||||
|
||||
|
||||
def extract_tagline(soup: BeautifulSoup) -> str:
|
||||
"""The creator's "short description or tagline" (itch.io puts it in og:description)."""
|
||||
meta = soup.find("meta", property="og:description")
|
||||
text = " ".join(str(meta.get("content", "")).split()) if meta else ""
|
||||
return _cap(text) if text else NA
|
||||
|
||||
|
||||
def extract_first_sentence(soup: BeautifulSoup) -> str:
|
||||
desc_tag = soup.find("div", class_="formatted_description")
|
||||
if not desc_tag:
|
||||
return NA
|
||||
@@ -329,11 +385,13 @@ def extract_description(soup: BeautifulSoup) -> str:
|
||||
return NA
|
||||
|
||||
# First sentence (a terminator followed by whitespace + a capital), capped at 200 chars.
|
||||
first = _SENTENCE_END.split(full, maxsplit=1)[0]
|
||||
if len(first) <= 200:
|
||||
return first
|
||||
cut = first[:197].rsplit(" ", 1)[0] if " " in first[:197] else first[:197]
|
||||
return cut + "..."
|
||||
return _cap(_SENTENCE_END.split(full, maxsplit=1)[0])
|
||||
|
||||
|
||||
def extract_description(soup: BeautifulSoup) -> str:
|
||||
"""The tagline, or the description's first sentence when the creator wrote none."""
|
||||
tagline = extract_tagline(soup)
|
||||
return tagline if tagline != NA else extract_first_sentence(soup)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -361,12 +419,15 @@ _NSFW_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def detect_nsfw(soup: BeautifulSoup, tags, description: str) -> str:
|
||||
"""Detect NSFW. `tags` can be list[str] or str. Matches whole words only."""
|
||||
def detect_nsfw(soup: BeautifulSoup, tags, *texts: str) -> str:
|
||||
"""Detect NSFW from tags, short texts (tagline, first sentence) and itch.io's warning.
|
||||
|
||||
`tags` can be list[str] or str. Matches whole words only.
|
||||
"""
|
||||
tags_text = " ".join(tags) if isinstance(tags, list) else (tags or "")
|
||||
if _NSFW_RE.search(tags_text):
|
||||
return "Yes"
|
||||
if description != NA and _NSFW_RE.search(description):
|
||||
if any(text != NA and _NSFW_RE.search(text) for text in texts):
|
||||
return "Yes"
|
||||
if soup.find("div", class_=["view_game_warning", "mature_content_notice"]):
|
||||
return "Yes"
|
||||
@@ -387,6 +448,8 @@ def parse_game(soup: BeautifulSoup, url: str) -> dict:
|
||||
tags = info.get("Tags", [])
|
||||
description = extract_description(soup)
|
||||
release_date = info.get("Release date") or info.get("Published") or NA
|
||||
# The first sentence too: a tagline rarely says what the description does.
|
||||
nsfw = detect_nsfw(soup, tags, description, extract_first_sentence(soup))
|
||||
|
||||
record = {
|
||||
"url": url,
|
||||
@@ -401,13 +464,14 @@ def parse_game(soup: BeautifulSoup, url: str) -> dict:
|
||||
"rating": info.get("Rating", NA),
|
||||
"rating_count": info.get("RatingCount", NA),
|
||||
"average_session": info.get("Average session", NA),
|
||||
"nsfw": detect_nsfw(soup, tags, description),
|
||||
"nsfw": nsfw,
|
||||
"thumbnail": extract_thumbnail(soup),
|
||||
"tags": tags,
|
||||
"platforms": info.get("Platforms", []),
|
||||
"languages": info.get("Languages", []),
|
||||
"inputs": info.get("Inputs", []),
|
||||
"made_with": info.get("Made with", []),
|
||||
**ai_and_accessibility(info),
|
||||
"safe_virus": "?",
|
||||
"notes": "",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"https://21acad.itch.io/ts-malfoy-ep1": {"at": "2026-10-16T08:25:01Z", "outcome": "added"},
|
||||
"https://2tonmucho.itch.io/suburb2065": {"at": "2026-10-15T08:23:37Z", "outcome": "added"},
|
||||
"https://3dgoblindev.itch.io/finishing-the-hat": {"at": "2026-10-14T20:24:24Z", "outcome": "added"},
|
||||
"https://8man-hikigaya.itch.io/artificial-affection": {"at": "2026-10-14T20:24:28Z", "outcome": "added"},
|
||||
"https://ackleyrc.itch.io/bridgemancer": {"at": "2026-10-19T20:23:34Z", "outcome": "added"},
|
||||
"https://activedev.itch.io/legend-of-idle-playtest-demo": {"at": "2026-10-16T04:24:52Z", "outcome": "added"},
|
||||
"https://aegeusgames.itch.io/drimm-remedy": {"at": "2026-10-14T01:02:26Z", "outcome": "added"},
|
||||
"https://aiainaine.itch.io/better-on-paper": {"at": "2026-10-16T04:24:59Z", "outcome": "added"},
|
||||
"https://akirakani-kani.itch.io/hayasuginovel": {"at": "2026-10-18T04:24:35Z", "outcome": "added"},
|
||||
"https://alchemysheep.itch.io/game-is-full-of-bugs": {"at": "2026-10-22T00:23:31Z", "outcome": "added"},
|
||||
"https://alfredo7.itch.io/dissosidate": {"at": "2026-10-20T08:23:34Z", "outcome": "added"},
|
||||
"https://antpunch.itch.io/memloss": {"at": "2026-10-19T16:23:36Z", "outcome": "added"},
|
||||
"https://ao-85.itch.io/who-is-nana": {"at": "2026-10-20T16:23:35Z", "outcome": "added"},
|
||||
"https://arcadekitten.itch.io/mit-juste": {"at": "2026-10-17T08:23:43Z", "outcome": "added"},
|
||||
"https://arisaogi.itch.io/natsu-ga-owaranai": {"at": "2026-10-14T20:24:31Z", "outcome": "added"},
|
||||
"https://arknoid.itch.io/lost-warden": {"at": "2026-10-14T01:04:53Z", "outcome": "added"},
|
||||
"https://armouredboar.itch.io/risen-kingdoms": {"at": "2026-10-14T12:24:21Z", "outcome": "added"},
|
||||
"https://arubberducky.itch.io/futanari-destroying-sex-robot": {"at": "2026-10-20T14:40:15Z", "outcome": "added"},
|
||||
"https://auguste-eden.itch.io/amber-forest": {"at": "2026-10-16T04:25:05Z", "outcome": "added"},
|
||||
"https://baldigaming99.itch.io/geometry-dash-22091": {"at": "2026-10-18T20:23:34Z", "outcome": "dead"},
|
||||
"https://baldigaming99.itch.io/geometry-dash-220922091-apkandpc": {"at": "2026-10-19T12:23:36Z", "outcome": "added"},
|
||||
"https://basic-eagles.itch.io/recaller": {"at": "2026-10-20T08:23:38Z", "outcome": "added"},
|
||||
"https://basilium.itch.io/the-excavation-project": {"at": "2026-10-14T01:02:12Z", "outcome": "added"},
|
||||
"https://becxah.itch.io/misty-memories": {"at": "2026-10-14T01:02:59Z", "outcome": "added"},
|
||||
"https://bedstespiliverden.itch.io/two-shades-of-pink": {"at": "2026-10-16T08:25:07Z", "outcome": "added"},
|
||||
"https://benjithepro17.itch.io/baldi-70": {"at": "2026-10-20T12:23:34Z", "outcome": "added"},
|
||||
"https://bergice.itch.io/lurkers": {"at": "2026-10-20T12:23:38Z", "outcome": "added"},
|
||||
"https://blackjet.itch.io/labbaye-des-morts": {"at": "2026-10-16T04:25:13Z", "outcome": "added"},
|
||||
"https://blanklake.itch.io/the-scrawled-book": {"at": "2026-10-16T08:25:11Z", "outcome": "added"},
|
||||
"https://bloopyloopygoop.itch.io/wailant": {"at": "2026-10-14T20:24:34Z", "outcome": "added"},
|
||||
"https://bombacatdev.itch.io/yandere-sim": {"at": "2026-10-19T08:23:31Z", "outcome": "added"},
|
||||
"https://breezy-the-cat.itch.io/houswatching": {"at": "2026-10-14T01:03:02Z", "outcome": "added"},
|
||||
"https://c-mastered-studio.itch.io/play-with-your-girlfriend": {"at": "2026-10-15T08:23:40Z", "outcome": "added"},
|
||||
"https://caleb545.itch.io/potters-field": {"at": "2026-10-14T01:03:19Z", "outcome": "added"},
|
||||
"https://capri-demon.itch.io/demon-lord-laid": {"at": "2026-10-14T01:02:29Z", "outcome": "added"},
|
||||
"https://cardio-pulmo.itch.io/loa-night-shift": {"at": "2026-10-18T08:23:33Z", "outcome": "added"},
|
||||
"https://ceiravarageplayer.itch.io/other-random-games-fkfgfjjnjgndsngbnap": {"at": "2026-10-19T20:23:38Z", "outcome": "added"},
|
||||
"https://changsikpassingby.itch.io/dracina-malfoy-project": {"at": "2026-10-16T08:25:19Z", "outcome": "added"},
|
||||
"https://churnveil.itch.io/scarlett-awakening": {"at": "2026-10-16T18:43:01Z", "outcome": "added"},
|
||||
"https://cicidigital.itch.io/retromine": {"at": "2026-10-16T12:24:18Z", "outcome": "added"},
|
||||
"https://cineeeekk.itch.io/castle-builddefend": {"at": "2026-10-21T12:23:36Z", "outcome": "added"},
|
||||
"https://cosmicadventuresquad.itch.io/project-hail-meowy": {"at": "2026-10-21T12:23:43Z", "outcome": "added"},
|
||||
"https://cosmos07.itch.io/idle-island": {"at": "2026-10-14T01:02:15Z", "outcome": "added"},
|
||||
"https://croissmouse.itch.io/malpoid": {"at": "2026-10-16T08:25:24Z", "outcome": "added"},
|
||||
"https://cz758175.itch.io/the-incense-still-burns": {"at": "2026-10-21T08:23:33Z", "outcome": "added"},
|
||||
"https://d3emo.itch.io/patantan": {"at": "2026-10-18T16:23:30Z", "outcome": "added"},
|
||||
"https://daisuki-yui-kon.itch.io/yuiazu": {"at": "2026-10-16T08:25:30Z", "outcome": "added"},
|
||||
"https://danchicu.itch.io/pigeon-prompt-engineer": {"at": "2026-10-20T08:23:41Z", "outcome": "added"},
|
||||
"https://danielmorenoroldan.itch.io/asdasd-the-clicker-game": {"at": "2026-10-20T12:23:41Z", "outcome": "added"},
|
||||
"https://dannpops.itch.io/karoshi": {"at": "2026-10-20T08:23:45Z", "outcome": "added"},
|
||||
"https://dazylude.itch.io/an-old-joke": {"at": "2026-10-20T16:23:39Z", "outcome": "added"},
|
||||
"https://deby-studio.itch.io/the-cursed-box": {"at": "2026-10-20T04:23:31Z", "outcome": "added"},
|
||||
"https://demo0606.itch.io/aholeinmywall": {"at": "2026-10-19T08:23:34Z", "outcome": "added"},
|
||||
"https://deskgeneral.itch.io/sir-we-have-a-situation": {"at": "2026-10-14T01:02:18Z", "outcome": "added"},
|
||||
"https://device25.itch.io/the-last-line": {"at": "2026-10-14T01:04:55Z", "outcome": "added"},
|
||||
"https://digitalwaifudelivery.itch.io/sin-igame": {"at": "2026-10-21T08:23:36Z", "outcome": "added"},
|
||||
"https://doctorhummer.itch.io/irule": {"at": "2026-10-21T12:23:49Z", "outcome": "added"},
|
||||
"https://doorwatchers.itch.io/cycle-of-the-first-dawn": {"at": "2026-10-16T12:24:20Z", "outcome": "added"},
|
||||
"https://dorabuay.itch.io/midnight-cocktail": {"at": "2026-10-14T01:03:23Z", "outcome": "added"},
|
||||
"https://drevengames.itch.io/idle-fantasy-shop": {"at": "2026-10-19T04:23:32Z", "outcome": "added"},
|
||||
"https://eden505.itch.io/outofframe": {"at": "2026-10-16T18:42:57Z", "outcome": "added"},
|
||||
"https://edmond00.itch.io/jellyfish-egg": {"at": "2026-10-14T01:02:06Z", "outcome": "added"},
|
||||
"https://eliasguyd.itch.io/rooklite": {"at": "2026-10-17T04:23:44Z", "outcome": "added"},
|
||||
"https://enoki-nohito.itch.io/kiminisika-dekinai": {"at": "2026-10-18T16:23:33Z", "outcome": "added"},
|
||||
"https://entink.itch.io/pip-os": {"at": "2026-10-21T08:23:40Z", "outcome": "added"},
|
||||
"https://erclyuaccd.itch.io/hattisan": {"at": "2026-10-14T01:03:26Z", "outcome": "added"},
|
||||
"https://eturnalove.itch.io/villainess-malfoy": {"at": "2026-10-19T08:23:38Z", "outcome": "added"},
|
||||
"https://eyelessx.itch.io/until-the-stars-forget-you": {"at": "2026-10-17T12:23:44Z", "outcome": "added"},
|
||||
"https://fatass-orange.itch.io/ghost-onsen-festival": {"at": "2026-10-16T08:25:36Z", "outcome": "added"},
|
||||
"https://filgreen3.itch.io/coup-of-appliance": {"at": "2026-10-20T12:23:45Z", "outcome": "added"},
|
||||
"https://fionafenali.itch.io/in-my-image": {"at": "2026-10-20T16:23:42Z", "outcome": "added"},
|
||||
"https://fiorane.itch.io/nadameru": {"at": "2026-10-17T08:23:46Z", "outcome": "added"},
|
||||
"https://frajdagames.itch.io/fireworks-incremental": {"at": "2026-10-17T04:23:47Z", "outcome": "added"},
|
||||
"https://gabrielcassimiro.itch.io/cake-clicker-recake": {"at": "2026-10-21T00:23:34Z", "outcome": "added"},
|
||||
"https://gabriellj.itch.io/dreamer": {"at": "2026-10-15T08:23:43Z", "outcome": "added"},
|
||||
"https://gamezeternal.itch.io/project-chromata": {"at": "2026-10-14T01:04:58Z", "outcome": "added"},
|
||||
"https://gdestroyer12.itch.io/teyvat-academy": {"at": "2026-10-20T16:23:44Z", "outcome": "added"},
|
||||
"https://gi-joe424.itch.io/sinister-mister": {"at": "2026-10-17T08:23:50Z", "outcome": "added"},
|
||||
"https://giantlight.itch.io/mr-magpies-harmless-card-game": {"at": "2026-10-14T01:05:02Z", "outcome": "added"},
|
||||
"https://goblinwithsocks.itch.io/idle-guild": {"at": "2026-10-20T12:23:47Z", "outcome": "added"},
|
||||
"https://gridmere.itch.io/gridmere": {"at": "2026-10-20T12:23:51Z", "outcome": "added"},
|
||||
"https://griffithsgames.itch.io/idle-dreadnought": {"at": "2026-10-14T01:02:21Z", "outcome": "added"},
|
||||
"https://hurby.itch.io/ruh": {"at": "2026-10-14T01:03:30Z", "outcome": "added"},
|
||||
"https://husbandogoddess.itch.io/a-little-inconvenience": {"at": "2026-10-18T16:23:36Z", "outcome": "added"},
|
||||
"https://ii-p.itch.io/ii-ii-ii-i-i-i-i-ii": {"at": "2026-10-14T01:05:05Z", "outcome": "added"},
|
||||
"https://jab-lao.itch.io/pokemon-overlord": {"at": "2026-10-15T12:24:12Z", "outcome": "added"},
|
||||
"https://jiufoduck.itch.io/recycle-yourself": {"at": "2026-10-19T08:23:40Z", "outcome": "added"},
|
||||
"https://jmrcl.itch.io/in-the-shadows": {"at": "2026-10-20T08:23:48Z", "outcome": "added"},
|
||||
"https://jotabot.itch.io/in-the-shadows": {"at": "2026-10-20T16:23:48Z", "outcome": "added"},
|
||||
"https://jproductions2.itch.io/30-times-table": {"at": "2026-10-18T04:24:50Z", "outcome": "added"},
|
||||
"https://junkfoode0.itch.io/synthesis": {"at": "2026-10-19T08:23:43Z", "outcome": "added"},
|
||||
"https://just-look-ing.itch.io/burst-blackjack": {"at": "2026-10-19T20:23:41Z", "outcome": "added"},
|
||||
"https://kaaexperienced.itch.io/hypnosnake-simulator": {"at": "2026-10-19T20:23:45Z", "outcome": "added"},
|
||||
"https://kazehita.itch.io/heartfall-a-tale-in-three-lives": {"at": "2026-10-14T23:45:13Z", "outcome": "added"},
|
||||
"https://keefbaker.itch.io/nemeshit": {"at": "2026-10-14T01:02:43Z", "outcome": "added"},
|
||||
"https://kelebeksiron.itch.io/paperweight": {"at": "2026-10-19T08:23:46Z", "outcome": "added"},
|
||||
"https://kenmaryuji.itch.io/mineminemine": {"at": "2026-10-14T01:03:33Z", "outcome": "added"},
|
||||
"https://kickyfunkfresh.itch.io/freestyletalesofitch": {"at": "2026-10-16T04:25:18Z", "outcome": "added"},
|
||||
"https://kihyunmathee.itch.io/gray-and-blue": {"at": "2026-10-14T01:03:36Z", "outcome": "added"},
|
||||
"https://kiku-o2.itch.io/ttmwdahsgtiavm": {"at": "2026-10-14T01:03:38Z", "outcome": "added"},
|
||||
"https://kirinkoumori.itch.io/essays": {"at": "2026-10-18T08:23:35Z", "outcome": "added"},
|
||||
"https://kiwimsy.itch.io/mewmewstruck": {"at": "2026-10-14T20:24:37Z", "outcome": "added"},
|
||||
"https://kripkode.itch.io/the-last-boba-shop": {"at": "2026-10-17T08:23:53Z", "outcome": "added"},
|
||||
"https://kronusgenovius.itch.io/a-shields-journey": {"at": "2026-10-16T08:25:43Z", "outcome": "added"},
|
||||
"https://lambdaforge.itch.io/kenzu": {"at": "2026-10-16T04:25:24Z", "outcome": "added"},
|
||||
"https://lechita03.itch.io/linea-heroica": {"at": "2026-10-14T01:02:33Z", "outcome": "added"},
|
||||
"https://leeevans.itch.io/pursuit-of-warmth": {"at": "2026-10-14T01:03:42Z", "outcome": "added"},
|
||||
"https://lennychip.itch.io/corpus-callosum": {"at": "2026-10-14T20:24:40Z", "outcome": "added"},
|
||||
"https://lex-voca.itch.io/lex-voca-section-1": {"at": "2026-10-15T08:23:47Z", "outcome": "added"},
|
||||
"https://lianaavk.itch.io/beyo": {"at": "2026-10-21T08:23:44Z", "outcome": "added"},
|
||||
"https://lifeofboris.itch.io/boristan": {"at": "2026-10-19T12:23:40Z", "outcome": "added"},
|
||||
"https://lil-beastman.itch.io/himbo-hymns": {"at": "2026-10-14T23:45:15Z", "outcome": "added"},
|
||||
"https://lilys-mascot.itch.io/aishite-kotoshi": {"at": "2026-10-14T01:03:44Z", "outcome": "added"},
|
||||
"https://liminace.itch.io/little-red-fangs": {"at": "2026-10-21T04:23:36Z", "outcome": "added"},
|
||||
"https://little-ones.itch.io/cozy-game-company": {"at": "2026-10-14T01:02:09Z", "outcome": "added"},
|
||||
"https://littledibble.itch.io/seas-the-day": {"at": "2026-10-19T16:23:40Z", "outcome": "added"},
|
||||
"https://loglogz.itch.io/lava-links": {"at": "2026-10-14T01:02:47Z", "outcome": "added"},
|
||||
"https://lonelynode.itch.io/killa": {"at": "2026-10-18T08:23:39Z", "outcome": "added"},
|
||||
"https://lostbyte.itch.io/thats-a-steal": {"at": "2026-10-14T01:03:48Z", "outcome": "added"},
|
||||
"https://lsremix.itch.io/astoria": {"at": "2026-10-14T01:03:52Z", "outcome": "added"},
|
||||
"https://lucaju.itch.io/gyra": {"at": "2026-10-14T23:45:19Z", "outcome": "added"},
|
||||
"https://madocactus.itch.io/interviolence": {"at": "2026-10-16T04:25:31Z", "outcome": "added"},
|
||||
"https://mariiartt.itch.io/face-it": {"at": "2026-10-14T01:02:37Z", "outcome": "added"},
|
||||
"https://matthewdrake.itch.io/fayonline": {"at": "2026-10-20T08:23:51Z", "outcome": "added"},
|
||||
"https://mawsters.itch.io/mawster": {"at": "2026-10-14T23:45:23Z", "outcome": "added"},
|
||||
"https://mdawg74.itch.io/unit-112": {"at": "2026-10-18T16:23:38Z", "outcome": "added"},
|
||||
"https://mediocrisis.itch.io/geeneses-game": {"at": "2026-10-19T16:23:44Z", "outcome": "added"},
|
||||
"https://melomell.itch.io/encaprichada": {"at": "2026-10-19T16:23:46Z", "outcome": "added"},
|
||||
"https://meniir.itch.io/twilight-at-the-banyan-tree": {"at": "2026-10-20T04:23:34Z", "outcome": "added"},
|
||||
"https://michellesalvatore9-sketch.itch.io/sombras-de-la-verdad": {"at": "2026-10-14T01:03:54Z", "outcome": "added"},
|
||||
"https://micpp.itch.io/loss-and-freedom": {"at": "2026-10-19T08:23:48Z", "outcome": "added"},
|
||||
"https://mirroring-foxes-studio.itch.io/rezero-if": {"at": "2026-10-20T08:23:55Z", "outcome": "added"},
|
||||
"https://misha.itch.io/echoes-of-twilight": {"at": "2026-10-21T00:23:38Z", "outcome": "added"},
|
||||
"https://missblossom.itch.io/university-of-progress": {"at": "2026-10-14T01:03:57Z", "outcome": "added"},
|
||||
"https://missooni.itch.io/nothing-of-value-lost": {"at": "2026-10-15T08:23:50Z", "outcome": "added"},
|
||||
"https://misstullie.itch.io/made-with-nostalgia": {"at": "2026-10-18T08:23:43Z", "outcome": "added"},
|
||||
"https://mithikagames.itch.io/breadwinner": {"at": "2026-10-20T04:23:36Z", "outcome": "added"},
|
||||
"https://mlouvet.itch.io/dallas": {"at": "2026-10-21T08:23:47Z", "outcome": "added"},
|
||||
"https://mocomo892.itch.io/offchat": {"at": "2026-10-16T08:25:49Z", "outcome": "added"},
|
||||
"https://mongoumery.itch.io/magicnumbers": {"at": "2026-10-14T23:45:27Z", "outcome": "added"},
|
||||
"https://monoon-games.itch.io/lady-malfoy": {"at": "2026-10-16T08:25:54Z", "outcome": "added"},
|
||||
"https://moochbear.itch.io/cyberpawn-2099": {"at": "2026-10-14T20:24:44Z", "outcome": "added"},
|
||||
"https://mors-games.itch.io/midas-crutch": {"at": "2026-10-16T04:25:37Z", "outcome": "added"},
|
||||
"https://mrormsbee.itch.io/wings-of-time-the-salem-witch-trials": {"at": "2026-10-14T23:45:29Z", "outcome": "added"},
|
||||
"https://muscle-cat-s.itch.io/surrounded-by-muscle-girls": {"at": "2026-10-16T08:26:01Z", "outcome": "added"},
|
||||
"https://n4hobino.itch.io/into-the-dragonrealm": {"at": "2026-10-19T16:23:49Z", "outcome": "added"},
|
||||
"https://nataly-torrijos.itch.io/nuestra-casa-compartida": {"at": "2026-10-15T08:23:53Z", "outcome": "added"},
|
||||
"https://natsu-madao.itch.io/erwin-smith-run": {"at": "2026-10-14T23:45:32Z", "outcome": "added"},
|
||||
"https://negentropy-ko.itch.io/the-shadows": {"at": "2026-10-20T08:23:58Z", "outcome": "added"},
|
||||
"https://nekolele.itch.io/agos": {"at": "2026-10-21T08:23:50Z", "outcome": "added"},
|
||||
"https://neon-namen.itch.io/dungeon-horde": {"at": "2026-10-14T01:02:50Z", "outcome": "added"},
|
||||
"https://nightmareweaverz.itch.io/captive": {"at": "2026-10-14T23:45:35Z", "outcome": "added"},
|
||||
"https://nilano.itch.io/the-last-delivery": {"at": "2026-10-14T01:02:53Z", "outcome": "added"},
|
||||
"https://nosleep2jull.itch.io/quiet-where-you-stand": {"at": "2026-10-14T20:24:47Z", "outcome": "added"},
|
||||
"https://olipool.itch.io/boeckhams-football-manager": {"at": "2026-10-16T04:25:43Z", "outcome": "added"},
|
||||
"https://onyyyxxxxxxx.itch.io/the-last-trailexe": {"at": "2026-10-14T20:24:51Z", "outcome": "added"},
|
||||
"https://orangesounds.itch.io/the-angels-devil-tail": {"at": "2026-10-16T08:26:07Z", "outcome": "added"},
|
||||
"https://ostsergey.itch.io/plaguebound": {"at": "2026-10-14T23:45:37Z", "outcome": "added"},
|
||||
"https://outstar.itch.io/blokia-designer": {"at": "2026-10-20T12:23:54Z", "outcome": "added"},
|
||||
"https://pamkat444.itch.io/no-se-habla-de-los-cuerpitos": {"at": "2026-10-19T08:23:51Z", "outcome": "added"},
|
||||
"https://pantaloon-io.itch.io/lemonade-apocalypse": {"at": "2026-10-16T00:23:47Z", "outcome": "added"},
|
||||
"https://pickpawket.itch.io/a-trick-of-his-light": {"at": "2026-10-21T16:23:38Z", "outcome": "added"},
|
||||
"https://placeholders.itch.io/007": {"at": "2026-10-16T04:25:48Z", "outcome": "added"},
|
||||
"https://pokolega.itch.io/nikoclick": {"at": "2026-10-14T01:02:56Z", "outcome": "added"},
|
||||
"https://polypaw.itch.io/the-no-jumpscare-show": {"at": "2026-10-18T12:23:32Z", "outcome": "added"},
|
||||
"https://poprise.itch.io/drag-race-simulator": {"at": "2026-10-18T04:24:54Z", "outcome": "added"},
|
||||
"https://potit-bob.itch.io/dungeon-swap-r": {"at": "2026-10-14T00:23:41Z", "outcome": "added"},
|
||||
"https://pravthelamb.itch.io/ignotum": {"at": "2026-10-14T01:04:01Z", "outcome": "added"},
|
||||
"https://progdamn.itch.io/brother-hurry-and-put-the-disc-in": {"at": "2026-10-20T16:23:52Z", "outcome": "added"},
|
||||
"https://puapka.itch.io/dentata": {"at": "2026-10-21T00:23:42Z", "outcome": "added"},
|
||||
"https://pumpkycookie.itch.io/pager-13": {"at": "2026-10-20T08:24:01Z", "outcome": "added"},
|
||||
"https://qixya.itch.io/3-seconds-chapter-2-episode-3": {"at": "2026-10-14T01:04:04Z", "outcome": "added"},
|
||||
"https://queencontract.itch.io/purple-pleasure": {"at": "2026-10-19T08:23:54Z", "outcome": "added"},
|
||||
"https://quiirisk.itch.io/stepmom-who-fell-on-a-hike": {"at": "2026-10-21T08:23:52Z", "outcome": "added"},
|
||||
"https://r3v3rsecze.itch.io/john-door-the-amazing-adventure-about-openning-doors": {"at": "2026-10-19T08:23:57Z", "outcome": "added"},
|
||||
"https://railgunrabbit.itch.io/super-safe-for-work": {"at": "2026-10-14T01:04:08Z", "outcome": "added"},
|
||||
"https://rainmutt.itch.io/wish-bone": {"at": "2026-10-15T08:23:55Z", "outcome": "added"},
|
||||
"https://rarelikeaunicorn.itch.io/vvvvvvv": {"at": "2026-10-21T04:23:40Z", "outcome": "added"},
|
||||
"https://ratking.itch.io/mops-and-mobs": {"at": "2026-10-14T20:24:20Z", "outcome": "gave_up"},
|
||||
"https://redembergames.itch.io/hyperballin": {"at": "2026-10-15T12:24:15Z", "outcome": "added"},
|
||||
"https://reeecook.itch.io/leaf-by-leaf-jam": {"at": "2026-10-14T00:23:47Z", "outcome": "added"},
|
||||
"https://retracegames.itch.io/youngforty": {"at": "2026-10-16T08:26:14Z", "outcome": "added"},
|
||||
"https://rhidel.itch.io/malfoid-sim": {"at": "2026-10-19T08:24:01Z", "outcome": "added"},
|
||||
"https://ruizukun-dev.itch.io/dictator-dating-sim": {"at": "2026-10-18T04:24:42Z", "outcome": "added"},
|
||||
"https://rychz.itch.io/the-binding-of-isaac-rtx": {"at": "2026-10-20T12:23:56Z", "outcome": "added"},
|
||||
"https://sakiness.itch.io/everlune": {"at": "2026-10-21T16:23:41Z", "outcome": "added"},
|
||||
"https://salamanderpress.itch.io/the-cottage-visitor": {"at": "2026-10-15T08:23:58Z", "outcome": "added"},
|
||||
"https://sashify.itch.io/popcorn-idle": {"at": "2026-10-16T00:23:51Z", "outcome": "added"},
|
||||
"https://seah0rsegg.itch.io/liminal": {"at": "2026-10-16T08:26:20Z", "outcome": "added"},
|
||||
"https://shadowflamedragn.itch.io/smoke-mirrors": {"at": "2026-10-18T16:23:40Z", "outcome": "added"},
|
||||
"https://sharkwithlasers.itch.io/rotetra": {"at": "2026-10-14T01:05:08Z", "outcome": "added"},
|
||||
"https://slambond.itch.io/packardos": {"at": "2026-10-14T01:05:12Z", "outcome": "added"},
|
||||
"https://slemkatt.itch.io/a-sandwich-adventure": {"at": "2026-10-19T08:24:03Z", "outcome": "added"},
|
||||
"https://smestorp.itch.io/candy-quest-3-edge-of-sweetness": {"at": "2026-10-20T12:24:00Z", "outcome": "added"},
|
||||
"https://sobigdrasil.itch.io/slamdating": {"at": "2026-10-14T01:04:11Z", "outcome": "added"},
|
||||
"https://soliles.itch.io/trust-the-system": {"at": "2026-10-14T23:45:40Z", "outcome": "added"},
|
||||
"https://soulofkazak.itch.io/spooks-spook-y-halloween-party": {"at": "2026-10-16T08:26:27Z", "outcome": "added"},
|
||||
"https://starismoi.itch.io/woods-of-ember": {"at": "2026-10-14T01:04:15Z", "outcome": "added"},
|
||||
"https://starzonie.itch.io/boolangerie": {"at": "2026-10-14T01:04:17Z", "outcome": "added"},
|
||||
"https://sticktrix.itch.io/dying-metals-last-scream": {"at": "2026-10-14T01:04:21Z", "outcome": "added"},
|
||||
"https://studio-chicle.itch.io/adamas": {"at": "2026-10-21T08:23:56Z", "outcome": "added"},
|
||||
"https://studio-spagunkel.itch.io/fake-news-detective-in-space": {"at": "2026-10-14T00:23:51Z", "outcome": "added"},
|
||||
"https://studiohexttrpg.itch.io/biophage-jaeger": {"at": "2026-10-21T08:23:59Z", "outcome": "added"},
|
||||
"https://superkuo.itch.io/dont-touch-that-chest": {"at": "2026-10-18T00:23:36Z", "outcome": "added"},
|
||||
"https://supertuxkart.itch.io/supertuxkart": {"at": "2026-10-15T12:24:18Z", "outcome": "added"},
|
||||
"https://superwuper.itch.io/royalconsumption": {"at": "2026-10-16T08:26:33Z", "outcome": "added"},
|
||||
"https://surrtasm.itch.io/the-art-shop": {"at": "2026-10-14T12:24:25Z", "outcome": "added"},
|
||||
"https://tabbiedearest.itch.io/cloistered-crown": {"at": "2026-10-14T23:45:43Z", "outcome": "added"},
|
||||
"https://targaciej.itch.io/paragra": {"at": "2026-10-20T04:23:38Z", "outcome": "added"},
|
||||
"https://tattershallstudios.itch.io/gravewell": {"at": "2026-10-16T00:23:54Z", "outcome": "added"},
|
||||
"https://te-eg.itch.io/lamplighter": {"at": "2026-10-14T12:24:28Z", "outcome": "added"},
|
||||
"https://teo-tormo.itch.io/sf2cengcd": {"at": "2026-10-14T01:02:23Z", "outcome": "added"},
|
||||
"https://thatwalshguy.itch.io/stclzine": {"at": "2026-10-16T12:24:24Z", "outcome": "added"},
|
||||
"https://thelatteteam.itch.io/bbieal-wiiu": {"at": "2026-10-20T12:24:03Z", "outcome": "added"},
|
||||
"https://theobalm-strawberry.itch.io/vampire-therapy": {"at": "2026-10-14T20:24:55Z", "outcome": "added"},
|
||||
"https://theoppaipriest.itch.io/hear-me-out": {"at": "2026-10-14T01:04:44Z", "outcome": "added"},
|
||||
"https://thiagodv.itch.io/combat-arena": {"at": "2026-10-14T01:05:15Z", "outcome": "added"},
|
||||
"https://thiagodv.itch.io/stick-fight-arena-pre-alpha": {"at": "2026-10-17T16:24:46Z", "outcome": "added"},
|
||||
"https://tinyworkers.itch.io/get-to-work-gnome": {"at": "2026-10-20T12:24:06Z", "outcome": "added"},
|
||||
"https://tjlazer.itch.io/lifeforce-aga-for-amiga-12004000": {"at": "2026-10-14T00:23:54Z", "outcome": "added"},
|
||||
"https://tjlazer.itch.io/salamander-aga-for-amiga": {"at": "2026-10-14T12:24:31Z", "outcome": "added"},
|
||||
"https://torcado.itch.io/manymancer": {"at": "2026-10-21T00:23:45Z", "outcome": "added"},
|
||||
"https://traumalab.itch.io/trauma-lab": {"at": "2026-10-22T00:23:35Z", "outcome": "added"},
|
||||
"https://tsomyst.itch.io/lavoice": {"at": "2026-10-21T04:23:44Z", "outcome": "added"},
|
||||
"https://twindark.itch.io/godseater": {"at": "2026-10-14T01:04:47Z", "outcome": "added"},
|
||||
"https://v-shr1.itch.io/dragon": {"at": "2026-10-18T08:23:46Z", "outcome": "added"},
|
||||
"https://vector5games.itch.io/splatch": {"at": "2026-10-14T23:45:47Z", "outcome": "added"},
|
||||
"https://vectorinox.itch.io/imperium-host": {"at": "2026-10-17T16:24:52Z", "outcome": "added"},
|
||||
"https://veryfujii.itch.io/be-my-lover-4-a-day": {"at": "2026-10-20T08:24:04Z", "outcome": "added"},
|
||||
"https://vhan-5.itch.io/u-dont-get-it": {"at": "2026-10-21T16:23:44Z", "outcome": "added"},
|
||||
"https://vvafelek.itch.io/dontkeepthedevilwaiting": {"at": "2026-10-14T01:05:19Z", "outcome": "added"},
|
||||
"https://waamr7.itch.io/317": {"at": "2026-10-14T01:02:40Z", "outcome": "added"},
|
||||
"https://waitingtreesproductions.itch.io/yesod-dating-sim": {"at": "2026-10-16T08:26:41Z", "outcome": "added"},
|
||||
"https://wake44466.itch.io/real-intelligence": {"at": "2026-10-21T08:24:03Z", "outcome": "added"},
|
||||
"https://waldorf.itch.io/bombermaid": {"at": "2026-10-16T04:25:54Z", "outcome": "added"},
|
||||
"https://willagem.itch.io/and-from-deep-down-below": {"at": "2026-10-14T01:04:50Z", "outcome": "added"},
|
||||
"https://writer31.itch.io/say-that-you-hate-me": {"at": "2026-10-18T08:23:50Z", "outcome": "added"},
|
||||
"https://writer31.itch.io/say-that-you-love-me": {"at": "2026-10-14T23:45:49Z", "outcome": "added"},
|
||||
"https://xtheredviper.itch.io/ashes-of-beau": {"at": "2026-10-21T08:24:08Z", "outcome": "added"},
|
||||
"https://yotogi-chan.itch.io/yuasama-cardgame": {"at": "2026-10-20T12:24:08Z", "outcome": "added"},
|
||||
"https://yuhaolin2005.itch.io/qingxuan-volume1": {"at": "2026-10-21T08:24:10Z", "outcome": "added"},
|
||||
"https://zanipixels.itch.io/the-chimera-gameboy": {"at": "2026-10-16T04:26:00Z", "outcome": "added"}
|
||||
}
|
||||
@@ -1 +1,3 @@
|
||||
{}
|
||||
{
|
||||
"https://ratking.itch.io/mops-and-mobs": {"attempts": 3, "gave_up": "2026-10-14T20:24:20Z", "last_attempt": "2026-10-14T20:24:20Z", "last_error": "error: HTTP 403"}
|
||||
}
|
||||
+3486
-2952
File diff suppressed because it is too large.
Load diff
+1
-38
@@ -1,38 +1 @@
|
||||
[
|
||||
"https://chilkus.itch.io/marble-marvell",
|
||||
"https://critchlowr.itch.io/you-i",
|
||||
"https://csaf.itch.io/coppice",
|
||||
"https://stingergames.itch.io/house-always-wins",
|
||||
"https://turtlevaniagames.itch.io/penguin-planet",
|
||||
"https://choicegaming.itch.io/wick-and-tide",
|
||||
"https://csaf.itch.io/gasholder",
|
||||
"https://csaf.itch.io/theodolite",
|
||||
"https://emberforgedstudios.itch.io/defenders-of-cinderkeep-demo",
|
||||
"https://indie-ian.itch.io/peg-defense-force",
|
||||
"https://jedimeisterx.itch.io/lifeseed",
|
||||
"https://keqh.itch.io/tumblestorm",
|
||||
"https://mors-games.itch.io/sploob",
|
||||
"https://punkcake.itch.io/spaghetti-celesti",
|
||||
"https://acro-projects.itch.io/date-night",
|
||||
"https://gunterjr.itch.io/waterfall",
|
||||
"https://haruka-apps-games.itch.io/komahime-stella",
|
||||
"https://ikemakesgames1.itch.io/glaggles-playhouse",
|
||||
"https://invisiblemilk.itch.io/your-new-roommate",
|
||||
"https://jomoby.itch.io/dv",
|
||||
"https://pavelbjorn.itch.io/brownwaters",
|
||||
"https://ronhelleredfn-boop.itch.io/bb-blackwood",
|
||||
"https://tiramisiu.itch.io/thinkingaboutyou",
|
||||
"https://toro-comicss.itch.io/emergency-monster-team-chapter-0",
|
||||
"https://yarheeguy.itch.io/immolate",
|
||||
"https://astaysdev.itch.io/vestige",
|
||||
"https://bboy6720.itch.io/cosmic-forge",
|
||||
"https://dredlam.itch.io/arson",
|
||||
"https://momentmori.itch.io/summer-yet-young",
|
||||
"https://raxeris.itch.io/99",
|
||||
"https://raxeris.itch.io/dig-to-australia",
|
||||
"https://shinh.itch.io/enter-the-nyangeon",
|
||||
"https://unoyado.itch.io/hoshitachi-no-ouchi",
|
||||
"https://dragonrider1223.itch.io/rooted",
|
||||
"https://peatstack.itch.io/clach",
|
||||
"https://transmutrix.itch.io/priiism"
|
||||
]
|
||||
[]
|
||||
+34
-7
@@ -4,9 +4,10 @@ update_info.py — Scrape queued game links into a pipeline patch.
|
||||
Flow:
|
||||
1. Read URLs from scripts/temp_link.json (+ INPUT_URL from workflow_dispatch)
|
||||
2. Canonicalize; drop invalid URLs, duplicates, games already in the catalog
|
||||
and games in scripts/deleted_games.json (the admin restores a deleted
|
||||
game by removing its log entry when approving it again)
|
||||
3. Fetch each page → keep only free games (with an `added_at` stamp)
|
||||
and games in scripts/deleted_games.json (unless the admin restored them:
|
||||
scripts/state/unblocked.json)
|
||||
3. Fetch each page → keep only free games (with an `added_at` stamp); a
|
||||
password-protected page ends the link like a dead one
|
||||
4. Emit a patch (see patch.py). URLs that finished (added / paid / dead /
|
||||
skipped) are dropped from the queue; transient failures stay queued and
|
||||
are retried; a link is given up after MAX_ATTEMPTS failures at least
|
||||
@@ -18,9 +19,12 @@ Flow:
|
||||
(`queue_add`), so it survives a retry, a deadline or a rate-limit stop.
|
||||
A second spelling of a queued game is left alone: the first copy's
|
||||
outcome removes every spelling (apply_patch compares canonical forms).
|
||||
5. How each link ended (added, paid, dead, private, duplicate, deleted,
|
||||
gave_up) goes to `ingest_outcomes` → scripts/state/ingest_outcomes.json,
|
||||
which the admin page reads to follow a batch it queued.
|
||||
|
||||
apply_patch.py then applies the patch to the latest `main`, so links queued
|
||||
while this ran (extension, admin) are never overwritten.
|
||||
while this ran (admin, cron) are never overwritten.
|
||||
|
||||
Usage: python scripts/update_info.py --out patch.json [--deadline-min 50]
|
||||
"""
|
||||
@@ -37,7 +41,7 @@ from data_store import get_all_urls, load_all_games
|
||||
from json_io import load_json
|
||||
from patch import new_patch, write_patch
|
||||
from refresh import fetch_with_backoff
|
||||
from scraper import NA, Pacer, create_session, now_iso, parse_game
|
||||
from scraper import NA, Pacer, create_session, is_password_protected, now_iso, parse_game
|
||||
|
||||
TEMP_LINK = "scripts/temp_link.json"
|
||||
DELETED_LOG = "scripts/deleted_games.json"
|
||||
@@ -69,6 +73,11 @@ def failed_attempt(previous: dict, outcome: str, now: str) -> dict:
|
||||
return {"attempts": previous["attempts"] + 1, "last_attempt": now, "last_error": outcome}
|
||||
|
||||
|
||||
def record_outcome(patch: dict, url: str, outcome: str) -> None:
|
||||
"""How a queued link ended, for the admin page's tracking of the batches it queued."""
|
||||
patch["ingest_outcomes"][url] = {"outcome": outcome, "at": now_iso()}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Scrape queued links into a patch.")
|
||||
ap.add_argument("--out", required=True, help="patch file to write")
|
||||
@@ -87,7 +96,18 @@ def main(argv: list[str] | None = None) -> int:
|
||||
print(f"Added from workflow input: {input_url}")
|
||||
|
||||
stats = dict.fromkeys(
|
||||
("queued", "added", "paid", "dead", "duplicate", "deleted", "invalid", "retry", "gave_up"),
|
||||
(
|
||||
"queued",
|
||||
"added",
|
||||
"paid",
|
||||
"dead",
|
||||
"private",
|
||||
"duplicate",
|
||||
"deleted",
|
||||
"invalid",
|
||||
"retry",
|
||||
"gave_up",
|
||||
),
|
||||
0,
|
||||
)
|
||||
stats["queued"] = len(queue)
|
||||
@@ -118,11 +138,13 @@ def main(argv: list[str] | None = None) -> int:
|
||||
if url in catalog_urls:
|
||||
print(f"Skip duplicate: {url}")
|
||||
stats["duplicate"] += 1
|
||||
record_outcome(patch, url, "duplicate")
|
||||
if url in unblocked:
|
||||
patch["unlog"].append(url)
|
||||
elif url in deleted_urls and url not in unblocked:
|
||||
print(f"Skip previously deleted: {url} (restore it from the admin page)")
|
||||
stats["deleted"] += 1
|
||||
record_outcome(patch, url, "deleted")
|
||||
else:
|
||||
todo.append(url)
|
||||
continue
|
||||
@@ -146,7 +168,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
continue
|
||||
|
||||
outcome = ""
|
||||
if page.status == 200 and page.soup is not None:
|
||||
if page.status == 200 and page.soup is not None and is_password_protected(page.soup):
|
||||
outcome = "private" # final: nobody can open it, so retrying won't help
|
||||
elif page.status == 200 and page.soup is not None:
|
||||
record = parse_game(page.soup, url)
|
||||
if record["name"] == NA:
|
||||
outcome = "error: no title"
|
||||
@@ -168,6 +192,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
patch["queue_remove"].append(url)
|
||||
patch["ingest_state"][url] = {**entry, "gave_up": entry["last_attempt"]}
|
||||
stats["gave_up"] += 1
|
||||
record_outcome(patch, url, "gave_up")
|
||||
if url in unblocked:
|
||||
patch["unblock_remove"].append(url)
|
||||
else:
|
||||
@@ -178,6 +203,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
print(f" → {outcome}")
|
||||
stats[outcome] += 1
|
||||
record_outcome(patch, url, outcome)
|
||||
patch["queue_remove"].append(url)
|
||||
patch["ingest_state"][url] = None
|
||||
if url in unblocked:
|
||||
@@ -186,6 +212,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
write_patch(args.out, patch)
|
||||
print(
|
||||
f"\nDone — {stats['added']} added, {stats['paid']} paid, {stats['dead']} dead, "
|
||||
f"{stats['private']} password-protected, "
|
||||
f"{stats['duplicate']} duplicates, {stats['deleted']} previously deleted, "
|
||||
f"{stats['invalid']} invalid, {stats['retry']} to retry, {stats['gave_up']} given up."
|
||||
)
|
||||
|
||||
+37
-5
@@ -8,10 +8,13 @@ print.
|
||||
Checks:
|
||||
- chunk files game_info_001..N are contiguous, ≤ CHUNK_SIZE, lists of records
|
||||
- every record has the 20 schema fields with the right types, a canonical
|
||||
URL, valid enums; URLs are unique across chunks
|
||||
URL, valid enums; URLs are unique across chunks; the optional fields
|
||||
(added_at, updated_at, ai_disclosure, ai_content, accessibility) have the
|
||||
right types when present
|
||||
- index.json agrees with the chunk files
|
||||
- count_history.json: one {date, total} row per day, sorted
|
||||
- deleted_games.json entries are complete; temp_link.json is a list of strings
|
||||
- state/ingest_outcomes.json (when present): canonical URL → {outcome, at}
|
||||
|
||||
`--fix` applies safe normalizations (safe_virus / nsfw spelling) and saves.
|
||||
"""
|
||||
@@ -44,16 +47,36 @@ STRING_FIELDS = (
|
||||
"notes",
|
||||
)
|
||||
LIST_FIELDS = ("tags", "platforms", "languages", "inputs", "made_with")
|
||||
OPTIONAL_STRING_FIELDS = ("added_at", "updated_at")
|
||||
# Optional: older records lack them until a refresh adds them (schema changes stay additive).
|
||||
OPTIONAL_STRING_FIELDS = ("added_at", "updated_at", "ai_disclosure")
|
||||
OPTIONAL_LIST_FIELDS = ("ai_content", "accessibility")
|
||||
|
||||
SAFE_VIRUS = {"?", "Yes", "No", "Caution"}
|
||||
NSFW = {"Yes", "No"}
|
||||
_SAFE_VIRUS_ALIASES = {"y": "Yes", "yes": "Yes", "n": "No", "no": "No", "caution": "Caution"}
|
||||
_NSFW_ALIASES = {"y": "Yes", "yes": "Yes", "n": "No", "no": "No"}
|
||||
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
_STAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
|
||||
INGEST_OUTCOMES_VALUES = {"added", "duplicate", "paid", "dead", "private", "deleted", "gave_up"}
|
||||
|
||||
DELETED_LOG = "scripts/deleted_games.json"
|
||||
TEMP_LINK = "scripts/temp_link.json"
|
||||
INGEST_OUTCOMES = "scripts/state/ingest_outcomes.json"
|
||||
|
||||
|
||||
def check_outcomes(outcomes) -> list[str]:
|
||||
"""scripts/state/ingest_outcomes.json: canonical URL → {outcome, at}."""
|
||||
if not isinstance(outcomes, dict):
|
||||
return ["ingest_outcomes.json: top level must be an object"]
|
||||
errors = []
|
||||
for url, entry in outcomes.items():
|
||||
if not is_canonical(url):
|
||||
errors.append(f"ingest_outcomes.json: {url!r} is not a canonical game URL")
|
||||
elif not isinstance(entry, dict) or entry.get("outcome") not in INGEST_OUTCOMES_VALUES:
|
||||
errors.append(f"ingest_outcomes.json[{url}]: unknown outcome")
|
||||
elif not isinstance(entry.get("at"), str) or not _STAMP_RE.match(entry["at"]):
|
||||
errors.append(f"ingest_outcomes.json[{url}]: 'at' must be YYYY-MM-DDTHH:MM:SSZ")
|
||||
return errors
|
||||
|
||||
|
||||
def check_record(game, where: str) -> tuple[list[str], list[str]]:
|
||||
@@ -66,10 +89,11 @@ def check_record(game, where: str) -> tuple[list[str], list[str]]:
|
||||
errors.append(f"{where}: missing field '{field}'")
|
||||
elif not isinstance(game[field], str):
|
||||
errors.append(f"{where}: '{field}' must be a string")
|
||||
for field in LIST_FIELDS:
|
||||
for field in LIST_FIELDS + OPTIONAL_LIST_FIELDS:
|
||||
value = game.get(field)
|
||||
if field not in game:
|
||||
errors.append(f"{where}: missing field '{field}'")
|
||||
if field in LIST_FIELDS:
|
||||
errors.append(f"{where}: missing field '{field}'")
|
||||
elif not isinstance(value, list) or not all(isinstance(v, str) for v in value):
|
||||
errors.append(f"{where}: '{field}' must be a list of strings")
|
||||
for field in OPTIONAL_STRING_FIELDS:
|
||||
@@ -82,7 +106,12 @@ def check_record(game, where: str) -> tuple[list[str], list[str]]:
|
||||
errors.append(f"{where}: safe_virus {game.get('safe_virus')!r} not in {sorted(SAFE_VIRUS)}")
|
||||
if game.get("nsfw") not in NSFW and "nsfw" in game:
|
||||
errors.append(f"{where}: nsfw {game.get('nsfw')!r} not in {sorted(NSFW)}")
|
||||
known = set(STRING_FIELDS) | set(LIST_FIELDS) | set(OPTIONAL_STRING_FIELDS)
|
||||
known = (
|
||||
set(STRING_FIELDS)
|
||||
| set(LIST_FIELDS)
|
||||
| set(OPTIONAL_STRING_FIELDS)
|
||||
| set(OPTIONAL_LIST_FIELDS)
|
||||
)
|
||||
extra = sorted(set(game) - known)
|
||||
if extra:
|
||||
warnings.append(f"{where}: unknown field(s) {extra}")
|
||||
@@ -151,6 +180,9 @@ def validate() -> tuple[list[str], list[str]]:
|
||||
if not isinstance(queue, list) or not all(isinstance(u, str) for u in queue):
|
||||
errors.append("temp_link.json must be a JSON array of URL strings")
|
||||
|
||||
if os.path.exists(INGEST_OUTCOMES):
|
||||
errors += check_outcomes(load_json(INGEST_OUTCOMES))
|
||||
|
||||
return errors, warnings
|
||||
|
||||
|
||||
|
||||
Vendored
+58
@@ -0,0 +1,58 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta content="https://img.itch.zone/aW1nLzM=/original/Lamp.png" property="og:image"/>
|
||||
<meta content="Keep the lighthouse burning
|
||||
through one long night." property="og:description"/>
|
||||
<meta content="Keep the lighthouse burning through one long night." name="description"/>
|
||||
<title>Lantern Keeper by Moth Lane</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1 class="game_title">Lantern Keeper</h1>
|
||||
<div class="formatted_description user_formatted"><p>You are the last keeper of the northern light. Storms are coming, and the oil is running out.</p></div>
|
||||
<div class="info_panel_wrapper">
|
||||
<div id="game_info_panel_5551234" class="game_info_panel_widget base_widget">
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>Published</td>
|
||||
<td><abbr title="01 October 2026 @ 11:11 UTC"><span aria-hidden="true" class="icon icon-stopwatch"></span> 8 days ago</abbr></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Status</td>
|
||||
<td><a href="https://itch.io/games/released">Released</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Platforms</td>
|
||||
<td><a href="https://itch.io/games/html5">HTML5</a>, <a href="https://itch.io/games/platform-windows">Windows</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Author</td>
|
||||
<td><a href="https://mothlane.itch.io">Moth Lane</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Genre</td>
|
||||
<td><a href="https://itch.io/games/genre-simulation">Simulation</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Tags</td>
|
||||
<td><a href="https://itch.io/games/tag-atmospheric">Atmospheric</a>, <a href="https://itch.io/games/tag-pixel-art">Pixel Art</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Accessibility</td>
|
||||
<td><a href="https://itch.io/games/accessibility-subtitles">Subtitles</a>, <a href="https://itch.io/games/accessibility-configurable-controls">Configurable controls</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>AI Disclosure</td>
|
||||
<td><a href="https://itch.io/games/ai-assisted">AI Assisted</a>, <a href="https://itch.io/games/ai-graphics">Graphics</a>, <a href="https://itch.io/games/ai-text">Text</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="buy_row">
|
||||
<div class="button_message"><a class="button buy_btn" href="https://mothlane.itch.io/lantern-keeper/purchase">Download Now</a><span class="buy_message"><span class="sub">Name your own price</span></span></div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Vendored
+85
@@ -0,0 +1,85 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Quiet Orchard [Demo] by Ash & Ivy</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Markup as a browser's "copy outer HTML" + formatter leaves it: line breaks
|
||||
inside link texts, the info panel opened by script. -->
|
||||
<div class="header align_center has_image" id="header">
|
||||
<h1 itemprop="name" class="game_title">Quiet Orchard
|
||||
[Demo]</h1>
|
||||
</div>
|
||||
<div class="formatted_description user_formatted">
|
||||
<p>A gentle story about
|
||||
growing back. Pick apples, mend fences and talk to the neighbours.</p>
|
||||
</div>
|
||||
<div class="more_information_toggle open">
|
||||
<div class="info_panel_wrapper" style="display: block;">
|
||||
<div id="game_info_panel_6667267" class="game_info_panel_widget base_widget">
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>Updated</td>
|
||||
<td><abbr title="03 October 2026 @ 11:28 UTC"><span aria-hidden="true"
|
||||
class="icon icon-stopwatch"></span> 6 days ago</abbr></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Status</td>
|
||||
<td><a href="https://itch.io/games/in-development">In development</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Platforms</td>
|
||||
<td><a href="https://itch.io/games/html5">HTML5</a>, <a
|
||||
href="https://itch.io/games/platform-windows">Windows</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Author</td>
|
||||
<td><a href="https://ashandivy.itch.io">Ash & Ivy</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Genre</td>
|
||||
<td><a href="https://itch.io/games/genre-visual-novel">Visual Novel</a>, <a
|
||||
href="https://itch.io/games/tag-interactive-fiction">Interactive
|
||||
Fiction</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Made with</td>
|
||||
<td><a href="https://itch.io/games/made-with-renpy">Ren'Py</a>, <a
|
||||
href="https://itch.io/games/made-with-photoshop">Adobe
|
||||
Photoshop</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Tags</td>
|
||||
<td><a href="https://itch.io/games/tag-meaningful-choices">Meaningful
|
||||
Choices</a>, <a href="https://itch.io/games/tag-cozy">Cozy</a></td>
|
||||
</tr>
|
||||
<tr
|
||||
title="License for any source-code provided with the project's files.">
|
||||
<td>Code license</td>
|
||||
<td><a href="https://itch.io/games/code-mpl">Mozilla Public License 2.0
|
||||
(MPL)</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Average session</td>
|
||||
<td><a href="https://itch.io/games/duration-half-hour">About a half-hour</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Content</td>
|
||||
<td><a href="https://itch.io/games/tag-no-ai">No generative AI was used</a>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="buy_row">
|
||||
<div class="button_message"><a class="button buy_btn"
|
||||
href="https://ashandivy.itch.io/quiet-orchard/purchase">Download Now</a><span
|
||||
class="buy_message"><span class="sub">Name your own price</span></span></div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Vendored
+22
@@ -0,0 +1,22 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Secret Garden by Quiet Studio - itch.io</title>
|
||||
<meta content="Secret Garden by Quiet Studio" property="og:title"/>
|
||||
</head>
|
||||
<body class="locale_en layout_widget responsive" data-page_name="view_game">
|
||||
<div class="header_widget base_widget"><h1 class="title">itch.io</h1></div>
|
||||
<div class="main"><div class="inner_column">
|
||||
<div class="game_password_page page_widget base_widget" id="game_password_123456"><div class="padded"><section>
|
||||
<h2>A password is required to view this page</h2>
|
||||
<form action="https://quietstudio.itch.io/secret-garden/password" autocomplete="off" class="form" method="post">
|
||||
<input name="csrf_token" type="hidden" value="token"/>
|
||||
<div class="input_row"><label><input autocomplete="new-password" name="game_password" placeholder="Required" required="required" type="password"/></label></div>
|
||||
<input name="return_to" type="hidden" value="https://quietstudio.itch.io/secret-garden"/>
|
||||
<div class="buttons"><button class="button">Submit</button></div>
|
||||
</form>
|
||||
</section></div></div>
|
||||
</div></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -220,3 +220,68 @@ def test_refresh_patch_removals_are_capped_by_games_checked(repo):
|
||||
patch["removals"] = patch["removals"][:12]
|
||||
write_patch("patch.json", patch)
|
||||
assert apply_patch.main(["patch.json"]) == 0
|
||||
|
||||
|
||||
def _outcome(outcome, at):
|
||||
return {"outcome": outcome, "at": at}
|
||||
|
||||
|
||||
def test_ingest_outcomes_merge_newest_wins_and_reapply_is_a_noop(repo):
|
||||
seed_catalog([make_game("a")])
|
||||
save_json(
|
||||
_state_file("ingest_outcomes"),
|
||||
{
|
||||
"https://dev.itch.io/a": _outcome("added", "2026-10-01T00:00:00Z"),
|
||||
"https://dev.itch.io/b": _outcome("paid", "2026-10-05T00:00:00Z"),
|
||||
},
|
||||
)
|
||||
patch = new_patch("ingest")
|
||||
patch["ingest_outcomes"] = {
|
||||
"https://dev.itch.io/b": _outcome("added", "2026-10-04T00:00:00Z"), # older: ignored
|
||||
"https://dev.itch.io/c": _outcome("dead", "2026-10-06T00:00:00Z"),
|
||||
}
|
||||
write_patch("patch.json", patch)
|
||||
assert apply_patch.main(["patch.json"]) == 0
|
||||
expected = {
|
||||
"https://dev.itch.io/a": _outcome("added", "2026-10-01T00:00:00Z"),
|
||||
"https://dev.itch.io/b": _outcome("paid", "2026-10-05T00:00:00Z"),
|
||||
"https://dev.itch.io/c": _outcome("dead", "2026-10-06T00:00:00Z"),
|
||||
}
|
||||
assert read_json(_state_file("ingest_outcomes")) == expected
|
||||
before = (repo / _state_file("ingest_outcomes")).read_bytes()
|
||||
assert apply_patch.main(["patch.json"]) == 0
|
||||
assert (repo / _state_file("ingest_outcomes")).read_bytes() == before
|
||||
|
||||
|
||||
def test_a_patch_with_only_outcomes_is_applied(repo):
|
||||
seed_catalog([make_game("a")])
|
||||
patch = new_patch("ingest")
|
||||
patch["ingest_outcomes"] = {
|
||||
"https://dev.itch.io/a": _outcome("duplicate", "2026-10-06T00:00:00Z")
|
||||
}
|
||||
write_patch("patch.json", patch)
|
||||
assert apply_patch.main(["patch.json"]) == 0
|
||||
assert (
|
||||
read_json(_state_file("ingest_outcomes"))["https://dev.itch.io/a"]["outcome"] == "duplicate"
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_outcomes_are_pruned_by_age_and_count(monkeypatch):
|
||||
current = {
|
||||
"https://dev.itch.io/old": _outcome("paid", "2026-06-01T00:00:00Z"), # > 90 days before
|
||||
"https://dev.itch.io/kept": _outcome("added", "2026-07-10T00:00:00Z"),
|
||||
"not a dict": "x",
|
||||
}
|
||||
delta = {"https://dev.itch.io/new": _outcome("dead", "2026-10-01T00:00:00Z")}
|
||||
merged = apply_patch.merge_outcomes(current, delta)
|
||||
assert set(merged) == {"https://dev.itch.io/kept", "https://dev.itch.io/new"}
|
||||
|
||||
monkeypatch.setattr(apply_patch, "OUTCOMES_KEEP", 2)
|
||||
many = {
|
||||
f"https://dev.itch.io/g{i}": _outcome("added", f"2026-10-0{i}T00:00:00Z")
|
||||
for i in range(1, 5)
|
||||
}
|
||||
assert set(apply_patch.merge_outcomes(many, {})) == {
|
||||
"https://dev.itch.io/g4",
|
||||
"https://dev.itch.io/g3",
|
||||
}
|
||||
@@ -10,6 +10,8 @@ from scraper import (
|
||||
Pacer,
|
||||
detect_nsfw,
|
||||
extract_description,
|
||||
extract_tagline,
|
||||
is_password_protected,
|
||||
parse_game,
|
||||
parse_retry_after,
|
||||
)
|
||||
@@ -31,10 +33,63 @@ def test_free_game_full_record():
|
||||
assert record["thumbnail"].startswith("https://img.itch.zone/")
|
||||
assert record["safe_virus"] == "?" and record["notes"] == ""
|
||||
assert "updated_at" not in record
|
||||
# No AI or accessibility rows: the creator has not said.
|
||||
assert record["ai_disclosure"] == NA
|
||||
assert record["ai_content"] == [] and record["accessibility"] == []
|
||||
# Schema order: the 20 stored fields (+ is_free flag) in a fixed order.
|
||||
assert list(record)[:4] == ["url", "name", "is_free", "dev"]
|
||||
|
||||
|
||||
def test_ai_assisted_game_with_accessibility():
|
||||
record = parse_game(load_fixture_soup("game_ai_assisted.html"), "https://x.itch.io/y")
|
||||
assert record["is_free"] is True
|
||||
assert record["ai_disclosure"] == "AI Assisted"
|
||||
assert record["ai_content"] == ["Graphics", "Text"]
|
||||
assert record["accessibility"] == ["Subtitles", "Configurable controls"]
|
||||
assert record["release_date"] == "01 October 2026 @ 11:11 UTC" # "Published"
|
||||
# The creator's tagline (og:description), whitespace collapsed, wins over the first sentence.
|
||||
assert record["description"] == "Keep the lighthouse burning through one long night."
|
||||
|
||||
|
||||
def test_no_ai_game_from_reformatted_markup():
|
||||
record = parse_game(load_fixture_soup("game_no_ai.html"), "https://x.itch.io/y")
|
||||
assert record["ai_disclosure"] == "No AI"
|
||||
assert record["ai_content"] == [] and record["accessibility"] == []
|
||||
# Line breaks inside titles and links collapse to single spaces.
|
||||
assert record["name"] == "Quiet Orchard [Demo]"
|
||||
assert record["genre"] == "Visual Novel"
|
||||
assert record["made_with"] == ["Ren'Py", "Adobe Photoshop"]
|
||||
assert record["tags"] == ["Meaningful Choices", "Cozy"]
|
||||
assert record["status"] == "In development"
|
||||
assert record["updated_at"] == "03 October 2026 @ 11:28 UTC"
|
||||
# No tagline: the first sentence of the description.
|
||||
assert record["description"] == "A gentle story about growing back."
|
||||
|
||||
|
||||
def test_tagline_is_capped_on_a_word():
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
long = "tagline " * 40
|
||||
soup = BeautifulSoup(f'<meta property="og:description" content="{long}">', "html.parser")
|
||||
tagline = extract_tagline(soup)
|
||||
assert len(tagline) <= 200 and tagline.endswith("...")
|
||||
blank = BeautifulSoup('<meta property="og:description" content=" ">', "html.parser")
|
||||
assert extract_tagline(blank) == NA
|
||||
|
||||
|
||||
def test_nsfw_checks_the_first_sentence_behind_a_tagline():
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(
|
||||
'<meta property="og:description" content="A love story.">'
|
||||
'<div class="formatted_description"><p>An erotic visual novel. Enjoy.</p></div>',
|
||||
"html.parser",
|
||||
)
|
||||
record = parse_game(soup, "https://x.itch.io/y")
|
||||
assert record["description"] == "A love story."
|
||||
assert record["nsfw"] == "Yes"
|
||||
|
||||
|
||||
def test_description_first_sentence_keeps_version_numbers():
|
||||
record = parse_game(load_fixture_soup("game_free_full.html"), "https://x.itch.io/y")
|
||||
assert record["description"].startswith("Bake cookies with Jonut")
|
||||
@@ -61,6 +116,12 @@ def test_page_without_info_panel():
|
||||
assert record["genre"] == NA and record["tags"] == []
|
||||
|
||||
|
||||
def test_password_page_is_recognised():
|
||||
assert is_password_protected(load_fixture_soup("game_password.html"))
|
||||
assert not is_password_protected(load_fixture_soup("game_free_full.html"))
|
||||
assert not is_password_protected(load_fixture_soup("game_no_info.html"))
|
||||
|
||||
|
||||
def test_mature_notice_marks_nsfw():
|
||||
record = parse_game(load_fixture_soup("game_nsfw_warning.html"), "https://x.itch.io/y")
|
||||
assert record["nsfw"] == "Yes"
|
||||
|
||||
@@ -83,6 +83,23 @@ def test_paid_page_strike_then_removal():
|
||||
assert outcome == "removed_paid" and removal["reason"] == "Game became paid"
|
||||
|
||||
|
||||
def test_password_protected_page_strike_then_removal():
|
||||
game = make_game("a")
|
||||
outcome, updates, removal, entry = refresh.check_game(
|
||||
game, ok_page("game_password.html"), {"fails": 3}, NOW
|
||||
)
|
||||
assert outcome == "strike_private" and removal is None and not updates
|
||||
assert entry["strike"] == "private" and "fails" in entry
|
||||
outcome, _, removal, _ = refresh.check_game(
|
||||
game, ok_page("game_password.html"), entry, TOMORROW_NOW
|
||||
)
|
||||
assert outcome == "removed_private"
|
||||
assert removal["reason"] == "Game page is password-protected"
|
||||
assert refresh.removal_kind(removal["reason"]) == "private"
|
||||
assert refresh.removal_kind("Game became paid") == "paid"
|
||||
assert refresh.removal_kind("Game page no longer exists (HTTP 404)") == "dead"
|
||||
|
||||
|
||||
def test_different_strike_kind_restarts_the_count():
|
||||
game = make_game("a")
|
||||
entry = {"strike": "paid", "strike_at": "2026-09-10T03:00:00Z"}
|
||||
@@ -121,6 +138,36 @@ def test_full_mode_rescrapes_but_preserves_annotations():
|
||||
assert entry["full"] == NOW
|
||||
|
||||
|
||||
def test_daily_refresh_keeps_ai_and_accessibility_current():
|
||||
# An older record without the fields gets them; a changed disclosure is updated.
|
||||
_, updates, _, _ = refresh.check_game(make_game("a"), ok_page("game_ai_assisted.html"), {}, NOW)
|
||||
assert updates["ai_disclosure"] == "AI Assisted"
|
||||
assert updates["ai_content"] == ["Graphics", "Text"]
|
||||
assert updates["accessibility"] == ["Subtitles", "Configurable controls"]
|
||||
|
||||
game = make_game("b", ai_disclosure="AI Assisted", ai_content=["Code"], accessibility=[])
|
||||
_, updates, _, _ = refresh.check_game(game, ok_page("game_no_ai.html"), {}, NOW)
|
||||
assert updates["ai_disclosure"] == "No AI" and updates["ai_content"] == []
|
||||
assert "accessibility" not in updates
|
||||
|
||||
# Not disclosed on the page: stored as N/A / empty lists.
|
||||
_, updates, _, _ = refresh.check_game(make_game("c"), ok_page(), {}, NOW)
|
||||
assert (updates["ai_disclosure"], updates["ai_content"]) == ("N/A", [])
|
||||
assert updates["accessibility"] == []
|
||||
|
||||
|
||||
def test_full_mode_keeps_a_known_release_date_and_thumbnail():
|
||||
# The "Published" row is gone a few weeks after publishing; no og:image / screenshots.
|
||||
game = make_game("a", release_date="02 September 2026 @ 10:00 UTC")
|
||||
_, updates, _, _ = refresh.check_game(game, ok_page("game_no_ai.html"), {}, NOW, full=True)
|
||||
assert "release_date" not in updates and "thumbnail" not in updates
|
||||
assert updates["ai_disclosure"] == "No AI"
|
||||
|
||||
# An unknown date is still filled in from the page.
|
||||
_, updates, _, _ = refresh.check_game(make_game("b"), ok_page(), {}, NOW, full=True)
|
||||
assert updates["release_date"] == "23 April 2021 @ 19:30 UTC"
|
||||
|
||||
|
||||
def test_errors_and_parse_failures_keep_the_game():
|
||||
game = make_game("a")
|
||||
for page, expected in (
|
||||
|
||||
@@ -33,12 +33,16 @@ def test_ingest_outcomes(repo, monkeypatch):
|
||||
"https://dev.itch.io/free": Page(status=200, soup=load_fixture_soup("game_free_full.html")),
|
||||
"https://dev.itch.io/paid": Page(status=200, soup=load_fixture_soup("game_paid.html")),
|
||||
"https://dev.itch.io/dead": Page(status=404),
|
||||
"https://dev.itch.io/locked": Page(
|
||||
status=200, soup=load_fixture_soup("game_password.html")
|
||||
),
|
||||
}
|
||||
queue = [
|
||||
"https://DEV.itch.io/free/", # canonicalized
|
||||
"https://dev.itch.io/free", # duplicate within the queue
|
||||
"https://dev.itch.io/paid",
|
||||
"https://dev.itch.io/dead",
|
||||
"https://dev.itch.io/locked", # password-protected: final, not retried
|
||||
"https://dev.itch.io/known",
|
||||
"https://dev.itch.io/removed",
|
||||
"https://dev.itch.io/flaky", # 500 → retry later
|
||||
@@ -54,13 +58,28 @@ def test_ingest_outcomes(repo, monkeypatch):
|
||||
assert flaky["attempts"] == 1 and flaky["last_error"] == "error: HTTP 500"
|
||||
stats = patch["stats"]
|
||||
assert (stats["added"], stats["paid"], stats["dead"], stats["duplicate"]) == (1, 1, 1, 1)
|
||||
# A password-protected page is final: dropped from the queue, no attempt kept.
|
||||
assert stats["private"] == 1 and patch["ingest_state"]["https://dev.itch.io/locked"] is None
|
||||
assert (stats["deleted"], stats["invalid"], stats["retry"]) == (1, 1, 1)
|
||||
# How each finished link ended, for the admin's batch tracking: retries, invalid
|
||||
# entries and second spellings leave no outcome.
|
||||
outcomes = {url: e["outcome"] for url, e in patch["ingest_outcomes"].items()}
|
||||
assert outcomes == {
|
||||
"https://dev.itch.io/free": "added",
|
||||
"https://dev.itch.io/paid": "paid",
|
||||
"https://dev.itch.io/dead": "dead",
|
||||
"https://dev.itch.io/locked": "private",
|
||||
"https://dev.itch.io/known": "duplicate",
|
||||
"https://dev.itch.io/removed": "deleted",
|
||||
}
|
||||
assert all(e["at"].endswith("Z") for e in patch["ingest_outcomes"].values())
|
||||
# apply_patch compares canonical forms, so each processed link is covered once.
|
||||
removed = {canonicalize(u) or u for u in patch["queue_remove"]}
|
||||
assert removed == {
|
||||
"https://dev.itch.io/free",
|
||||
"https://dev.itch.io/paid",
|
||||
"https://dev.itch.io/dead",
|
||||
"https://dev.itch.io/locked",
|
||||
"https://dev.itch.io/known",
|
||||
"https://dev.itch.io/removed",
|
||||
"not a url",
|
||||
@@ -74,6 +93,7 @@ def test_gives_up_after_three_attempts_and_resets_on_requeue(repo, monkeypatch):
|
||||
patch = _run(monkeypatch, {}, [url])
|
||||
assert url in patch["queue_remove"]
|
||||
assert patch["ingest_state"][url]["attempts"] == 3 and "gave_up" in patch["ingest_state"][url]
|
||||
assert patch["ingest_outcomes"][url]["outcome"] == "gave_up"
|
||||
|
||||
save_json("scripts/state/ingest_state.json", {url: patch["ingest_state"][url]})
|
||||
patch = _run(monkeypatch, {}, [url])
|
||||
|
||||
@@ -30,6 +30,23 @@ def test_detects_schema_and_consistency_errors(repo):
|
||||
assert any("unknown field" in w for w in warnings)
|
||||
|
||||
|
||||
def test_optional_fields_are_typed_when_present(repo):
|
||||
seed_catalog(
|
||||
[
|
||||
make_game("a", ai_disclosure="No AI", ai_content=[], accessibility=["Subtitles"]),
|
||||
make_game("b"), # older record without them
|
||||
]
|
||||
)
|
||||
assert validate.validate() == ([], [])
|
||||
chunk = read_json("data_game/game_info_001.json")
|
||||
chunk[0]["ai_content"] = "Graphics"
|
||||
chunk[1]["ai_disclosure"] = ["No AI"]
|
||||
save_json("data_game/game_info_001.json", chunk)
|
||||
errors, _ = validate.validate()
|
||||
assert any("'ai_content' must be a list of strings" in e for e in errors)
|
||||
assert any("'ai_disclosure' must be a string" in e for e in errors)
|
||||
|
||||
|
||||
def test_fix_normalizes_enums(repo):
|
||||
seed_catalog([make_game("a", safe_virus="y"), make_game("b", nsfw="yes")])
|
||||
assert validate.fix() == 2
|
||||
@@ -70,3 +87,22 @@ def test_state_map_one_line_per_url(tmp_path):
|
||||
assert json.loads(path.read_text(encoding="utf-8"))["https://b.itch.io/x"]["fails"] == 1
|
||||
save_state_map(str(path), {})
|
||||
assert json.loads(path.read_text(encoding="utf-8")) == {}
|
||||
|
||||
|
||||
def test_ingest_outcomes_file_is_checked_when_present(repo):
|
||||
seed_catalog([make_game("a")])
|
||||
good = {"https://dev.itch.io/a": {"outcome": "added", "at": "2026-10-01T00:00:00Z"}}
|
||||
save_state_map("scripts/state/ingest_outcomes.json", good)
|
||||
assert validate.validate() == ([], [])
|
||||
save_json(
|
||||
"scripts/state/ingest_outcomes.json",
|
||||
{
|
||||
"https://DEV.itch.io/a/": {"outcome": "added", "at": "2026-10-01T00:00:00Z"},
|
||||
"https://dev.itch.io/b": {"outcome": "maybe", "at": "2026-10-01T00:00:00Z"},
|
||||
"https://dev.itch.io/c": {"outcome": "paid", "at": "yesterday"},
|
||||
},
|
||||
)
|
||||
text = "\n".join(validate.validate()[0])
|
||||
assert "not a canonical game URL" in text
|
||||
assert "unknown outcome" in text
|
||||
assert "'at' must be" in text
|
||||
@@ -1 +1 @@
|
||||
24
|
||||
26
|
||||
+69
-31
@@ -3,12 +3,12 @@
|
||||
One codebase for everything people see of the catalog:
|
||||
|
||||
- the website **<https://freeitchgames.win>** — read-only for everyone (no accounts, no sign-in);
|
||||
- its **Cloudflare Worker** — image proxy, the public Suggest form, the browser-extension ingest API,
|
||||
the admin API and the scheduled RSS discovery;
|
||||
- its **Cloudflare Worker** — image proxy, the public Suggest form, the admin API and the scheduled RSS
|
||||
discovery;
|
||||
- the **admin** app at `/admin/` — Maintainer only, behind Cloudflare Access;
|
||||
- the **desktop and Android apps** — the same public app wrapped with Tauri 2 (see [`TAURI.md`](TAURI.md)).
|
||||
|
||||
Stack: React 19 + TypeScript 6 + Vite 8 (Rolldown) + Tailwind CSS 4 (`@tailwindcss/vite`, tokens in
|
||||
Stack: React 19 + TypeScript 7 + Vite 8 (Rolldown) + Tailwind CSS 4 (`@tailwindcss/vite`, tokens in
|
||||
[`src/index.css`](src/index.css)) + shadcn/ui (Radix), TanStack Query, React Router 8, Zustand, and the
|
||||
self-hosted Geist fonts. The Worker is plain TypeScript on Cloudflare Workers
|
||||
(R2, D1, Images, Turnstile, Rate Limiting, Access).
|
||||
@@ -25,7 +25,7 @@ key entries) and rewrites old `#/…` links to real paths ([`src/lib/legacy.ts`]
|
||||
| [`admin/`](admin/) | Admin app, built to `dist/admin/` (web only) |
|
||||
| [`worker/`](worker/) | Cloudflare Worker ([`worker/index.ts`](worker/index.ts) lists the routes) |
|
||||
| [`worker/migrations/`](worker/migrations/) | D1 schema for the review queue |
|
||||
| [`vite-plugins/catalog-data.ts`](vite-plugins/catalog-data.ts) | Bundles the catalog into `dist/data/` + `sitemap.xml` |
|
||||
| [`vite-plugins/catalog-data.ts`](vite-plugins/catalog-data.ts) | Bundles the catalog into `dist/data/` (packs + legacy JSON) + `sitemap.xml` |
|
||||
| [`public/`](public/) | Static files; [`public/_headers`](public/_headers) sets headers + CSP for them |
|
||||
| [`src-tauri/`](src-tauri/) | Tauri 2 shell for the desktop and Android apps |
|
||||
| [`scripts/`](scripts/) | `gen_assets.py` (PWA icons + OG image), `i18n-edit.mjs` (edit `en`/`vi` keys together) |
|
||||
@@ -33,7 +33,7 @@ key entries) and rewrites old `#/…` links to real paths ([`src/lib/legacy.ts`]
|
||||
|
||||
## Local dev
|
||||
|
||||
Requires Node.js 22.22+ (CI uses the version in [`.node-version`](.node-version)) and npm. The full
|
||||
Requires Node.js 26+ (CI uses the version in [`.node-version`](.node-version)) and npm. The full
|
||||
setup, including the Python pipeline, is in [`docs/dev_env.md`](../docs/dev_env.md).
|
||||
|
||||
```sh
|
||||
@@ -116,8 +116,18 @@ verification page instead of the shell on every app route (and `403` on `/img`);
|
||||
Maintainer only. Cloudflare Access (the "Admin" application) guards `/admin` and `/api/admin/*`, and
|
||||
the Worker re-verifies the Access JWT and the `ADMIN_EMAILS` allow-list on every request
|
||||
([`worker/access.ts`](worker/access.ts), [`worker/admin.ts`](worker/admin.ts)). The app shell is only
|
||||
served after that check. Pages (hash routes): **Review queue**, **Add games**, **Catalog** (edit
|
||||
`safe_virus` / `notes` / `nsfw`, remove or restore a game).
|
||||
served after that check. Pages (hash routes):
|
||||
|
||||
- **Review queue**: approve (with override for removed games) or reject, in batches of 200.
|
||||
- **Add games**: paste any text or load a file, or read an itch.io collection; **Check** shows what Add
|
||||
would do with each URL before anything is written; Add queues the list in batches of 200 and then
|
||||
follows each game through the ingest (added / waiting / dropped and why, from
|
||||
`scripts/state/ingest_outcomes.json`).
|
||||
- **Catalog**: search, edit one game, or select up to 100 for a bulk edit (`nsfw` / `safe_virus` /
|
||||
`notes`) or a removal with a public reason (one commit); remove by list; restore removed games.
|
||||
- **New games** (added in the last 7 / 30 days, to review 18+ and virus checks in bulk), **Activity**
|
||||
(the D1 audit log) and **Settings** (auto-approve rules, followed itch.io collections, the last
|
||||
automatic run).
|
||||
|
||||
Writes go to the repo as a **GitHub App** (GraphQL `createCommitOnBranch`, so commits are
|
||||
"Verified"; [`worker/github.ts`](worker/github.ts), [`worker/repo.ts`](worker/repo.ts)). Approving a
|
||||
@@ -126,29 +136,31 @@ Maintainer's email is kept only in the D1 audit log, never in commits.
|
||||
|
||||
## Worker
|
||||
|
||||
Existing static files (the app, `/data/*`) are served by the assets layer without running the
|
||||
Worker. It runs for `wrangler.jsonc` `assets.run_worker_first` (`/img/*`, `/api/*`, `/admin`,
|
||||
`/admin/*`) and for every request that matches no file.
|
||||
Existing static files (the app, `/data/pack/*`) are served by the assets layer without running the
|
||||
Worker. It runs for `wrangler.jsonc` `assets.run_worker_first` (`/`, `/img/*`, `/api/*`, `/admin`,
|
||||
`/admin/*`, and the five readable catalog files under `/data`) and for every request that matches no file.
|
||||
|
||||
| Path | File | Purpose |
|
||||
|---|---|---|
|
||||
| `/img/<160\|640>/<path>` | [worker/img.ts](worker/img.ts) | Resized WebP covers, stored in R2; only thumbnails that are in the catalog (`/data/urls.json`), upstream fixed to `img.itch.zone` |
|
||||
| `/api/health` | [worker/index.ts](worker/index.ts) | Health check |
|
||||
| `/api/suggest` | [worker/suggest.ts](worker/suggest.ts) | Suggest form: same-origin POST, per-IP rate limit, Turnstile, canonical itch.io URL, note ≤ 500 chars |
|
||||
| `/api/ingest` | [worker/ingest.ts](worker/ingest.ts) | Browser extension: Access service token, `Idempotency-Key`, rate limit per token |
|
||||
| `/api/admin/*` | [worker/admin.ts](worker/admin.ts) | Admin API (queue, add, edit, remove, restore) |
|
||||
| `/api/admin/*` | [worker/admin.ts](worker/admin.ts) | Admin API (queue, check, add, track, edit and remove one or up to 100 games, restore, activity, settings, itch.io collections) |
|
||||
| `/admin`, `/admin/*` | [worker/admin.ts](worker/admin.ts) | Admin app shell, after the Access check |
|
||||
| anything else | [worker/spa.ts](worker/spa.ts) | App shell for routes, real 404 for missing files |
|
||||
| `/data/*.json` (not `/data/pack/`) | [worker/legacy-data.ts](worker/legacy-data.ts) | The readable catalog JSON, passed only to apps 4.0–4.1.1 (Tauri `Origin`); everyone else gets 410 + the repository links |
|
||||
| anything else | [worker/spa.ts](worker/spa.ts) | App shell for routes, real 404 for missing files (a missing pack too: `no-store`) |
|
||||
|
||||
**Cron** (`23 */4 * * *`, [`worker/discover.ts`](worker/discover.ts)): polls **one** itch.io RSS feed
|
||||
per run into the review queue, then does queue bookkeeping (queued → ingested / failed) and retention
|
||||
(idempotency records after 7 days, Suggest notes after 180). Nothing enters the catalog without the
|
||||
Maintainer's approval.
|
||||
**Cron** (`23 */4 * * *`, [`worker/index.ts`](worker/index.ts)), four steps that fail independently:
|
||||
[`discover.ts`](worker/discover.ts) polls **one** itch.io RSS feed into the review queue; queue
|
||||
bookkeeping settles queued rows with the pipeline's ingest outcomes (queued → ingested / failed);
|
||||
[`automation.ts`](worker/automation.ts) applies the Maintainer's automatic rules (auto-approve of
|
||||
unflagged items after the waiting time, new free games of followed collections, read with
|
||||
[`collections.ts`](worker/collections.ts)) in **one** commit; then retention (Suggest notes after 180
|
||||
days).
|
||||
|
||||
**Bindings**: `ASSETS`, `THUMBS` (R2 `freeitchgames-thumbs`), `DB` (D1 `freeitchgames`),
|
||||
`RL_SUGGEST` / `RL_INGEST` (rate limits). **Vars**: `SITE_ORIGIN`, `GITHUB_REPO`, `BOT_UA`, `GH_APP_ID`,
|
||||
`GH_APP_INSTALLATION_ID`, `ACCESS_TEAM_DOMAIN`, `ACCESS_AUD_ADMIN`, `ACCESS_AUD_INGEST`,
|
||||
`TURNSTILE_SITEKEY`. **Secrets**: `GH_APP_PRIVATE_KEY`, `ADMIN_EMAILS`, `TURNSTILE_SECRET`
|
||||
`RL_SUGGEST` / `RL_GATE` (rate limits). **Vars**: `SITE_ORIGIN`, `GITHUB_REPO`, `BOT_UA`, `GH_APP_ID`,
|
||||
`GH_APP_INSTALLATION_ID`, `ACCESS_TEAM_DOMAIN`, `ACCESS_AUD_ADMIN`, `TURNSTILE_SITEKEY`. **Secrets**: `GH_APP_PRIVATE_KEY`, `ADMIN_EMAILS`, `TURNSTILE_SECRET`
|
||||
(`DEV_ADMIN_EMAIL` is for local runs only). Logs: Workers Observability, 10% head sampling.
|
||||
|
||||
`canonical.ts` and `catalog.ts` are ports of `scripts/canonical.py` and `scripts/data_store.py` +
|
||||
@@ -158,20 +170,46 @@ Maintainer's approval.
|
||||
## Catalog data
|
||||
|
||||
[`vite-plugins/catalog-data.ts`](vite-plugins/catalog-data.ts) reads `../data_game/` and
|
||||
`../scripts/deleted_games.json` at build time and emits `dist/data/` (`index.json`,
|
||||
`game_info_NNN.json`, `count_history.json`, `deleted_games.json`, `urls.json`) plus `sitemap.xml`.
|
||||
The build **fails** on invalid JSON or an index/chunk mismatch, so a broken data commit never replaces
|
||||
a working deployment. In dev, a middleware serves the same files.
|
||||
`../scripts/deleted_games.json` at build time and emits:
|
||||
|
||||
- **Web**: reads same-origin `/data/*`; covers go through `/img`, so the visitor's browser never
|
||||
- **`dist/data/pack/`** — what the site and the apps read. `manifest-v1.json` names, per chunk, a
|
||||
**main** pack (every field but `description`) and a **desc** pack, plus the removal-log and history
|
||||
packs. A pack is the records stored by column (dictionaries for repeated strings and every tag-like
|
||||
list; [`src/lib/data/pack-format.ts`](src/lib/data/pack-format.ts), shared with the plugin), deflated
|
||||
at level 9 and named `<sha256 prefix>.bin`: a name changes exactly when the bytes do, so packs are
|
||||
served `immutable` and a returning visitor fetches only the ones that changed. About 264 KB on a
|
||||
first visit for ~3,100 games (descriptions, ~170 KB, load on demand).
|
||||
- **The readable JSON**, byte-for-byte as before (`index.json`, `game_info_NNN.json`,
|
||||
`count_history.json`, `deleted_games.json`) and `urls.json`: the Worker reads `urls.json` and
|
||||
`deleted_games.json` through `env.ASSETS`, and passes the files to apps 4.0–4.1.1 only.
|
||||
- `sitemap.xml`.
|
||||
|
||||
The build **fails** on invalid JSON or an index/chunk mismatch, so a broken data commit never replaces
|
||||
a working deployment. Nothing from the catalog may reach the JS bundle (a data-only rebuild keeps the JS
|
||||
byte-identical). In dev, a middleware serves the same files (rebuilt when a source file changes).
|
||||
|
||||
- **Web**: reads same-origin `/data/pack/*`; covers go through `/img`, so the visitor's browser never
|
||||
contacts itch.io until they click a link.
|
||||
- **Tauri apps**: read `https://freeitchgames.win/data/*` (served with
|
||||
- **Tauri apps**: read `https://freeitchgames.win/data/pack/*` (served with
|
||||
`Access-Control-Allow-Origin: *`) and load covers straight from `img.itch.zone`
|
||||
([`src/lib/config.ts`](src/lib/config.ts), [`src/lib/thumbnail.ts`](src/lib/thumbnail.ts)).
|
||||
|
||||
The public catalog queries (`db`, `deleted`, `count-history`) are persisted to IndexedDB
|
||||
(`webapp.query-cache`, up to 7 days) in [`src/main.tsx`](src/main.tsx). `gcTime` must stay ≥ the
|
||||
persist `maxAge`, or restored queries are garbage-collected right after hydration.
|
||||
Loading ([`src/lib/data/catalog.ts`](src/lib/data/catalog.ts)): the manifest is revalidated on every
|
||||
load (`cache: 'no-cache'`, shared for 10 s by the catalog, removal-log and history queries); packs use
|
||||
the normal HTTP cache. A 404 on a main pack means the manifest predates a deploy: it is fetched again
|
||||
(`cache: 'reload'`) and the load retried once. Packs are inflated with `DecompressionStream`, or with
|
||||
`fflate` (a lazy chunk) where the browser lacks it ([`src/lib/data/compression.ts`](src/lib/data/compression.ts)).
|
||||
Descriptions ([`src/hooks/useDescriptions.ts`](src/hooks/useDescriptions.ts)) are queries
|
||||
`['desc', <file>]`: a game page loads its chunk's, a search all of them. They match their games by
|
||||
row, so only the desc file named in the same catalog is ever loaded; a 404 reloads the catalog instead.
|
||||
|
||||
The public catalog queries (`db`, `deleted`, `count-history`, and the `desc` packs the current catalog
|
||||
names) are persisted to IndexedDB (`webapp.query-cache`, up to 7 days) by a small binary persister
|
||||
([`src/lib/data/persister.ts`](src/lib/data/persister.ts)): one byte of format, then the deflated JSON,
|
||||
so DevTools shows bytes, not readable data. Writes are throttled (1 s) and skipped when no data object
|
||||
changed. `gcTime` must stay ≥ the persist `maxAge`, or restored queries are garbage-collected right
|
||||
after hydration; bump the `data-vN` part of `CACHE_BUSTER` in [`src/main.tsx`](src/main.tsx) when the
|
||||
cached shape changes.
|
||||
|
||||
## Code splitting and targets
|
||||
|
||||
@@ -195,7 +233,7 @@ explicitly; the Tauri build keeps Vite's default (`baseline-widely-available`, S
|
||||
| `src/components/ui/*` | shadcn/ui primitives |
|
||||
| `src/components/legal-gate.tsx` | First-visit click-to-accept gate (`LEGAL_VERSION` in `src/stores/prefs.ts`) |
|
||||
| `src/components/ext-link.tsx` | Runtime-aware external link (web vs Tauri) |
|
||||
| `src/lib/data/*`, `src/lib/config.ts` | Catalog fetching, data base URL |
|
||||
| `src/lib/data/*`, `src/lib/config.ts` | Pack format, loader, compression, the binary IndexedDB persister; the pack base URL |
|
||||
| `src/lib/thumbnail.ts` | Cover URLs: `/img` on the web, `img.itch.zone` in the apps |
|
||||
| `src/lib/game-filters.ts`, `src/lib/format.ts`, `src/lib/platforms.ts` | The Games query (URL ⇄ filters, sorting, facet counts), locale-aware numbers and dates, platform labels |
|
||||
| `src/lib/turnstile.ts` | Loads Turnstile on demand (Suggest page only) |
|
||||
@@ -204,7 +242,7 @@ explicitly; the Tauri build keeps Vite's default (`baseline-widely-available`, S
|
||||
| `src/lib/legacy.ts` | One-time cleanup of v3 storage and routes |
|
||||
| `src/lib/about.ts` | App / social / third-party constants for the About page |
|
||||
| `src/stores/*` | Zustand: `prefs` (`webapp.prefs`), `theme` (`webapp.theme`) |
|
||||
| `src/hooks/*` | Data hooks, SEO head, theme / density effects, Android back button |
|
||||
| `src/hooks/*` | Data hooks (catalog, lazy descriptions), SEO head, theme / density / scrollbar effects, Android back button |
|
||||
|
||||
## Deploy (web)
|
||||
|
||||
|
||||
+10
-6
@@ -8,10 +8,13 @@ the same React app as the website <https://freeitchgames.win>. Desktop instructi
|
||||
|
||||
**Read-only viewers** of the public catalog: no sign-in, no editing, no telemetry.
|
||||
|
||||
- **Catalog data** is downloaded from **<https://freeitchgames.win/data>** (served with
|
||||
`Access-Control-Allow-Origin: *`). The Tauri build does not bundle the catalog (`vite.config.ts`
|
||||
only adds the `catalogData` plugin to the web build), so the apps always show the current catalog.
|
||||
The last downloaded copy is cached in IndexedDB for up to 7 days.
|
||||
- **Catalog data** is downloaded from **<https://freeitchgames.win/data/pack>** (`manifest-v1.json`
|
||||
+ compact `.bin` packs, served with `Access-Control-Allow-Origin: *`; descriptions load on demand).
|
||||
The Tauri build does not bundle the catalog (`vite.config.ts` only adds the `catalogData` plugin to
|
||||
the web build), so the apps always show the current catalog. The last downloaded copy is cached,
|
||||
compressed, in IndexedDB for up to 7 days. Apps 4.0–4.1.1 read the older readable JSON under `/data`,
|
||||
which the Worker serves only to the apps' webview origins (`tauri://localhost`,
|
||||
`http(s)://tauri.localhost`; [`worker/legacy-data.ts`](worker/legacy-data.ts)).
|
||||
- **Cover images** load **directly from itch.io's image CDN (`img.itch.zone`)**. The website's `/img`
|
||||
proxy is not used by the apps ([`src/lib/thumbnail.ts`](src/lib/thumbnail.ts)).
|
||||
- **Links** open in the system browser through `tauri-plugin-opener` (capability scope: `https://*`
|
||||
@@ -59,7 +62,8 @@ npm run tauri:dev
|
||||
|
||||
This runs `npm run dev` (Vite on port 5173, `beforeDevCommand`) and opens a native window pointing at
|
||||
it; HMR works as in web dev. The dev app, like the release app, reads the catalog from the live site,
|
||||
so it needs internet access. The Worker (`wrangler dev`) is not needed.
|
||||
so it needs internet access, and a change to the pack format works there only once the site that
|
||||
serves it is deployed. The Worker (`wrangler dev`) is not needed.
|
||||
|
||||
## Build native installers
|
||||
|
||||
@@ -110,7 +114,7 @@ if (isTauri()) {
|
||||
|
||||
The app uses it for the data URL, cover URLs, external links, the router, the Suggest page and the
|
||||
app-only EULA link in the legal gate (which has no "Leave" button in the apps). The footer shows a
|
||||
small "Desktop app (Tauri)" / "Mobile app (Tauri)" badge (picked by `useIsMobile()`), and on Android
|
||||
small "Desktop app (Tauri)" / "Mobile app (Tauri)" badge (picked by `isAndroid()`), and on Android
|
||||
the back button walks the router history (`src/hooks/useBackButton.ts`).
|
||||
|
||||
## Adding new Rust commands
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { useState } from 'react'
|
||||
import { useInfiniteQuery } from '@tanstack/react-query'
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { adminApi, shortSha, type AuditRow } from './api'
|
||||
import { errorText, when } from './texts'
|
||||
import { Notice, TabBar } from './ui'
|
||||
|
||||
const FILTERS = {
|
||||
all: { label: 'All', actions: [] },
|
||||
approvals: { label: 'Approvals', actions: ['approve', 'approve_override', 'auto_approve'] },
|
||||
collections: { label: 'Collections', actions: ['collection_add', 'collection_watch', 'collection_unwatch', 'collection_toggle'] },
|
||||
removals: { label: 'Removals', actions: ['delete'] },
|
||||
edits: { label: 'Edits', actions: ['edit'] },
|
||||
rejections: { label: 'Rejections', actions: ['reject'] },
|
||||
other: { label: 'Restores & settings', actions: ['restore', 'settings'] },
|
||||
} as const
|
||||
|
||||
type Filter = keyof typeof FILTERS
|
||||
|
||||
const ACTION: Record<string, string> = {
|
||||
approve: 'approved',
|
||||
approve_override: 'approved (override)',
|
||||
auto_approve: 'auto-approved',
|
||||
collection_add: 'added from a collection',
|
||||
collection_watch: 'followed a collection',
|
||||
collection_unwatch: 'stopped following',
|
||||
collection_toggle: 'switched a collection',
|
||||
delete: 'removed',
|
||||
edit: 'edited',
|
||||
reject: 'rejected',
|
||||
restore: 'restored',
|
||||
settings: 'changed settings',
|
||||
}
|
||||
|
||||
/** The audit log: every admin action and automatic run, with its commit. */
|
||||
export function ActivityPage({ repo }: { repo: string | null }) {
|
||||
const [filter, setFilter] = useState<Filter>('all')
|
||||
const log = useInfiniteQuery({
|
||||
queryKey: ['activity', filter],
|
||||
queryFn: ({ pageParam }) => adminApi.activity({ before: pageParam, actions: [...FILTERS[filter].actions] }),
|
||||
initialPageParam: null as number | null,
|
||||
getNextPageParam: (last) => last.next,
|
||||
})
|
||||
const rows = log.data?.pages.flatMap((p) => p.items) ?? []
|
||||
|
||||
return (
|
||||
<section>
|
||||
<h1 className="mb-3 text-lg font-semibold">Activity</h1>
|
||||
<TabBar
|
||||
label="Kind of action"
|
||||
value={filter}
|
||||
onChange={setFilter}
|
||||
options={(Object.keys(FILTERS) as Filter[]).map((id) => ({ id, label: FILTERS[id].label }))}
|
||||
/>
|
||||
{log.error && <Notice text={errorText(log.error)} tone="error" onClose={() => void log.refetch()} />}
|
||||
{log.isPending && <p className="text-sm text-muted-foreground">Loading…</p>}
|
||||
{log.data && rows.length === 0 && <p className="text-sm text-muted-foreground">Nothing recorded yet.</p>}
|
||||
{rows.length > 0 && (
|
||||
<table className="w-full text-sm">
|
||||
<thead className="border-b text-left text-xs uppercase text-muted-foreground">
|
||||
<tr>
|
||||
<th className="py-2 pr-3">When</th>
|
||||
<th className="py-2 pr-3">Who</th>
|
||||
<th className="py-2 pr-3">What</th>
|
||||
<th className="py-2">Details</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r) => (
|
||||
<ActivityRow key={r.id} row={r} repo={repo} />
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
{log.hasNextPage && (
|
||||
<Button className="mt-3" variant="outline" size="sm" disabled={log.isFetchingNextPage} onClick={() => void log.fetchNextPage()}>
|
||||
{log.isFetchingNextPage ? 'Loading…' : 'Load older'}
|
||||
</Button>
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function ActivityRow({ row, repo }: { row: AuditRow; repo: string | null }) {
|
||||
const targets = (row.target ?? '').split(' ').filter(Boolean)
|
||||
const link = (t: string) =>
|
||||
t.startsWith('https://') ? (
|
||||
<a href={t} target="_blank" rel="noreferrer noopener" className="hover:underline">
|
||||
{t.replace('https://', '')}
|
||||
</a>
|
||||
) : (
|
||||
t
|
||||
)
|
||||
return (
|
||||
<tr className="border-b align-top last:border-0">
|
||||
<td className="py-2 pr-3 text-xs whitespace-nowrap text-muted-foreground tabular-nums">{when(row.at)}</td>
|
||||
<td className="max-w-40 truncate py-2 pr-3 text-xs">{row.actor === 'cron' ? 'automatic' : row.actor}</td>
|
||||
<td className="py-2 pr-3">
|
||||
<Badge variant={row.action === 'delete' || row.action === 'reject' ? 'destructive' : 'outline'}>
|
||||
{ACTION[row.action] ?? row.action}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="py-2 text-xs">
|
||||
{targets.length === 1 && <div className="truncate">{link(targets[0])}</div>}
|
||||
{targets.length > 1 && (
|
||||
<details>
|
||||
<summary className="cursor-pointer">
|
||||
{targets.length} games: {link(targets[0])} …
|
||||
</summary>
|
||||
<ul className="mt-1 max-h-48 overflow-y-auto">
|
||||
{targets.map((t) => (
|
||||
<li key={t} className="truncate">
|
||||
{link(t)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
{row.detail && <div className="mt-0.5 break-all text-muted-foreground">{row.detail}</div>}
|
||||
{row.commit_sha && (
|
||||
<div className="mt-0.5 text-muted-foreground">
|
||||
commit{' '}
|
||||
{repo ? (
|
||||
<a
|
||||
href={`https://github.com/${repo}/commit/${row.commit_sha}`}
|
||||
target="_blank"
|
||||
rel="noreferrer noopener"
|
||||
className="font-mono hover:underline"
|
||||
>
|
||||
{shortSha(row.commit_sha)}
|
||||
</a>
|
||||
) : (
|
||||
<span className="font-mono">{shortSha(row.commit_sha)}</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
)
|
||||
}
|
||||
+459
-64
@@ -1,93 +1,488 @@
|
||||
import { useState } from 'react'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { useMemo, useRef, useState, type DragEvent } from 'react'
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { adminApi, errorText, shortSha, type AddResult } from './api'
|
||||
import { Notice, PUBLISH_NOTE } from './app'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { MAX_TRACK_URLS, MAX_URLS } from '../worker/limits'
|
||||
import {
|
||||
adminApi,
|
||||
shortSha,
|
||||
type AddResult,
|
||||
type CheckResult,
|
||||
type CollectionItem,
|
||||
type CollectionRead,
|
||||
type TrackItem,
|
||||
} from './api'
|
||||
import { chunk, runBatches } from './batch'
|
||||
import { GROUPS, groupResults } from './check-groups'
|
||||
import { confirm } from './confirm'
|
||||
import { extractGameUrls } from './extract'
|
||||
import { INGEST_TEXT, PUBLISH_NOTE, errorText, plural, when } from './texts'
|
||||
import { summarize, useTracking, type TrackedBatch } from './tracking'
|
||||
import { BatchProgress, Notice, TabBar, type NoticeState } from './ui'
|
||||
|
||||
const OUTCOME: Record<string, string> = {
|
||||
queued: 'new',
|
||||
invalid_url: 'not an itch.io game URL',
|
||||
duplicate_in_request: 'listed twice',
|
||||
duplicate_catalog: 'already in the catalog',
|
||||
duplicate_pending: 'already in the queue',
|
||||
previously_deleted: 'removed before',
|
||||
rejected: 'rejected before',
|
||||
const MAX_FILE_BYTES = 5 * 1024 * 1024
|
||||
/** One ingest run (45 minutes at ≈ 14.5 games a minute) handles about this many games. */
|
||||
const PER_INGEST_RUN = 650
|
||||
|
||||
type Source = 'paste' | 'collection'
|
||||
|
||||
interface Collected {
|
||||
read: CollectionRead['collection']
|
||||
free: CollectionItem[]
|
||||
priced: number
|
||||
pages: number
|
||||
nextPage: number | null
|
||||
stopped?: string
|
||||
}
|
||||
|
||||
export function AddPage() {
|
||||
const [source, setSource] = useState<Source>('paste')
|
||||
const [text, setText] = useState('')
|
||||
const [override, setOverride] = useState(false)
|
||||
const [result, setResult] = useState<AddResult | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [collected, setCollected] = useState<Collected | null>(null)
|
||||
const [checked, setChecked] = useState<{ key: string; results: CheckResult[] } | null>(null)
|
||||
const [withRemoved, setWithRemoved] = useState(false)
|
||||
const [withRejected, setWithRejected] = useState(false)
|
||||
const [progress, setProgress] = useState<{ label: string; done: number; total: number } | null>(null)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
const client = useQueryClient()
|
||||
const urls = text
|
||||
.split(/\s+/)
|
||||
.map((u) => u.trim())
|
||||
.filter(Boolean)
|
||||
const track = useTracking((s) => s.track)
|
||||
|
||||
const add = useMutation({
|
||||
mutationFn: () => adminApi.add(urls, override),
|
||||
onSuccess: (r) => {
|
||||
setResult(r)
|
||||
setError(null)
|
||||
if (r.results.some((x) => x.approved)) setText('')
|
||||
void client.invalidateQueries({ queryKey: ['queue'] })
|
||||
},
|
||||
onError: (e) => setError(errorText(e)),
|
||||
})
|
||||
const pasted = useMemo(() => extractGameUrls(text), [text])
|
||||
const urls = source === 'paste' ? pasted.urls : (collected?.free.map((i) => i.url) ?? [])
|
||||
const via = source === 'collection' && collected ? `collection:${collected.read.id}` : undefined
|
||||
const key = `${source}\n${urls.join('\n')}`
|
||||
const results = checked?.key === key ? checked.results : null
|
||||
const groups = useMemo(() => (results ? groupResults(results) : null), [results])
|
||||
const busy = progress !== null
|
||||
|
||||
const check = async () => {
|
||||
setNotice(null)
|
||||
const run = await runBatches(
|
||||
chunk(urls, MAX_URLS),
|
||||
(batch) => adminApi.check(batch),
|
||||
(done, total) => setProgress({ label: 'Checking', done, total }),
|
||||
)
|
||||
setProgress(null)
|
||||
if (run.failed) setNotice({ tone: 'error', text: errorText(run.failed.error) })
|
||||
else setChecked({ key, results: run.results.flatMap((r) => r.results) })
|
||||
}
|
||||
|
||||
const add = async () => {
|
||||
if (!groups) return
|
||||
const normal = [...groups.new, ...groups.waiting]
|
||||
const override = [...(withRemoved ? groups.removed : []), ...(withRejected ? groups.rejected : [])]
|
||||
const total = normal.length + override.length
|
||||
if (total > 1000) {
|
||||
const runs = Math.ceil(total / PER_INGEST_RUN)
|
||||
const ok = await confirm({
|
||||
title: `Queue ${plural(total, 'game')}?`,
|
||||
body: `The ingest workflow handles about ${PER_INGEST_RUN} games per run, so this takes about ${runs} runs (45 minutes each); the rest waits in the queue meanwhile.`,
|
||||
action: `Queue ${plural(total, 'game')}`,
|
||||
})
|
||||
if (!ok) return
|
||||
}
|
||||
setNotice(null)
|
||||
const batches = [
|
||||
...chunk(normal, MAX_URLS).map((b) => ({ urls: b, override: false })),
|
||||
...chunk(override, MAX_URLS).map((b) => ({ urls: b, override: true })),
|
||||
]
|
||||
const run = await runBatches(
|
||||
batches.map((b) => [b]),
|
||||
([b]) => adminApi.add(b.urls, b.override, via),
|
||||
(done, all) => setProgress({ label: 'Adding', done, total: all }),
|
||||
)
|
||||
setProgress(null)
|
||||
const done: AddResult[] = run.results
|
||||
const queued = done.flatMap((r) => r.results.filter((x) => x.approved && x.url).map((x) => x.url as string))
|
||||
if (queued.length > 0 && done[0]) {
|
||||
const label = via ? `Collection #${collected?.read.id}: ${plural(queued.length, 'game')}` : `Add: ${plural(queued.length, 'game')}`
|
||||
track(label, done[0].at, queued)
|
||||
}
|
||||
const commits = done.map((r) => shortSha(r.sha)).filter((s) => s !== 'no commit')
|
||||
const lines: string[] = []
|
||||
if (queued.length > 0 || !run.failed) {
|
||||
lines.push(
|
||||
`Queued ${plural(queued.length, 'game')} for ingest${commits.length ? ` (${commits.join(', ')})` : ''}. Follow them under “Recent batches”. ${PUBLISH_NOTE}`,
|
||||
)
|
||||
}
|
||||
if (run.failed) {
|
||||
lines.push(errorText(run.failed.error))
|
||||
lines.push(
|
||||
`${plural(run.remaining.flat().reduce((n, b) => n + b.urls.length, 0), 'game')} not sent: Check again, then Add sends what is left.`,
|
||||
)
|
||||
}
|
||||
setNotice({ tone: run.failed ? 'error' : 'info', text: lines.join('\n') })
|
||||
if (!run.failed) {
|
||||
setChecked(null)
|
||||
if (source === 'paste') setText('')
|
||||
else setCollected(null)
|
||||
} else {
|
||||
// Re-check what is left, so a second Add sends only that.
|
||||
setChecked(null)
|
||||
}
|
||||
void client.invalidateQueries({ queryKey: ['queue'] })
|
||||
}
|
||||
|
||||
const addable = groups ? groups.new.length + groups.waiting.length : 0
|
||||
const overridable = groups ? (withRemoved ? groups.removed.length : 0) + (withRejected ? groups.rejected.length : 0) : 0
|
||||
|
||||
return (
|
||||
<section className="max-w-3xl">
|
||||
<section className="max-w-4xl">
|
||||
<h1 className="mb-1 text-lg font-semibold">Add games</h1>
|
||||
<p className="mb-3 text-sm text-muted-foreground">
|
||||
Paste itch.io game URLs (one per line, up to 200). New ones are approved right away and scraped by the
|
||||
ingest workflow; paid games are dropped there.
|
||||
Paste or upload any list (plain links, Markdown, CSV, HTML, JSON), or read an itch.io collection. Check it
|
||||
first: new games are queued for the ingest workflow, which drops the ones that turn out to be paid.
|
||||
</p>
|
||||
|
||||
<TabBar
|
||||
label="Where the list comes from"
|
||||
value={source}
|
||||
onChange={setSource}
|
||||
options={[
|
||||
{ id: 'paste', label: 'Paste or upload' },
|
||||
{ id: 'collection', label: 'itch.io collection' },
|
||||
]}
|
||||
/>
|
||||
|
||||
{source === 'paste' ? (
|
||||
<PasteInput text={text} onText={setText} found={pasted.urls.length} repeats={pasted.duplicates} onError={setNotice} />
|
||||
) : (
|
||||
<CollectionInput collected={collected} onCollected={setCollected} onError={setNotice} />
|
||||
)}
|
||||
|
||||
{notice && (
|
||||
<div className="mt-4">
|
||||
<Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />
|
||||
</div>
|
||||
)}
|
||||
{progress && <BatchProgress label={progress.label} done={progress.done} total={progress.total} />}
|
||||
|
||||
<div className="mt-3 flex flex-wrap items-center gap-3 text-sm">
|
||||
<Button variant="outline" disabled={urls.length === 0 || busy} onClick={() => void check()}>
|
||||
{busy && progress?.label === 'Checking' ? 'Checking…' : `Check ${urls.length ? plural(urls.length, 'URL') : ''}`}
|
||||
</Button>
|
||||
{groups && (
|
||||
<Button disabled={addable + overridable === 0 || busy} onClick={() => void add()}>
|
||||
{busy && progress?.label === 'Adding' ? 'Adding…' : `Add ${plural(addable + overridable, 'game')}`}
|
||||
</Button>
|
||||
)}
|
||||
{urls.length > 0 && !groups && <span className="text-muted-foreground">Check the list to see what Add would do.</span>}
|
||||
</div>
|
||||
|
||||
{groups && (
|
||||
<div className="mt-4 space-y-2">
|
||||
{GROUPS.filter((g) => groups[g.id].length > 0).map((g) => (
|
||||
<details key={g.id} className="rounded-md border px-3 py-2 text-sm" open={g.id === 'new' && groups.new.length <= 20}>
|
||||
<summary className="flex cursor-pointer flex-wrap items-center gap-2">
|
||||
<span className="font-medium">{g.label}</span>
|
||||
<span className="tabular-nums text-muted-foreground">{groups[g.id].length.toLocaleString('en-GB')}</span>
|
||||
<span className="text-xs text-muted-foreground">{g.hint}</span>
|
||||
{g.id === 'removed' && (
|
||||
<label className="ml-auto flex items-center gap-2" onClick={(e) => e.stopPropagation()}>
|
||||
<input type="checkbox" checked={withRemoved} onChange={(e) => setWithRemoved(e.target.checked)} />
|
||||
Add with override
|
||||
</label>
|
||||
)}
|
||||
{g.id === 'rejected' && (
|
||||
<label className="ml-auto flex items-center gap-2" onClick={(e) => e.stopPropagation()}>
|
||||
<input type="checkbox" checked={withRejected} onChange={(e) => setWithRejected(e.target.checked)} />
|
||||
Add with override
|
||||
</label>
|
||||
)}
|
||||
</summary>
|
||||
<ul className="mt-2 max-h-60 overflow-y-auto font-mono text-xs">
|
||||
{groups[g.id].map((u) => (
|
||||
<li key={u} className="truncate py-0.5">
|
||||
{u}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<RecentBatches />
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function PasteInput({
|
||||
text,
|
||||
onText,
|
||||
found,
|
||||
repeats,
|
||||
onError,
|
||||
}: {
|
||||
text: string
|
||||
onText: (text: string) => void
|
||||
found: number
|
||||
repeats: number
|
||||
onError: (n: NoticeState) => void
|
||||
}) {
|
||||
const file = useRef<HTMLInputElement>(null)
|
||||
const [dragging, setDragging] = useState(false)
|
||||
|
||||
const load = async (files: FileList | null) => {
|
||||
const picked = files?.[0]
|
||||
if (!picked) return
|
||||
if (picked.size > MAX_FILE_BYTES) {
|
||||
onError({ tone: 'error', text: `${picked.name} is larger than 5 MB.` })
|
||||
return
|
||||
}
|
||||
const content = await picked.text()
|
||||
onText(text ? `${text}\n${content}` : content)
|
||||
}
|
||||
const drop = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
setDragging(false)
|
||||
void load(e.dataTransfer.files)
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<label htmlFor="urls" className="sr-only">
|
||||
Game URLs
|
||||
Game URLs or any text containing them
|
||||
</label>
|
||||
<textarea
|
||||
id="urls"
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
onChange={(e) => onText(e.target.value)}
|
||||
onDragOver={(e) => {
|
||||
e.preventDefault()
|
||||
setDragging(true)
|
||||
}}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={drop}
|
||||
rows={8}
|
||||
spellCheck={false}
|
||||
placeholder="https://creator.itch.io/game"
|
||||
className="w-full rounded-md border bg-background p-2 font-mono text-sm"
|
||||
placeholder="https://creator.itch.io/game — or drop a .txt / .csv / .md / .html / .json file here"
|
||||
className={cn('w-full rounded-md border bg-background p-2 font-mono text-sm', dragging && 'border-primary ring-2 ring-primary/30')}
|
||||
/>
|
||||
<div className="mt-2 flex flex-wrap items-center gap-4 text-sm">
|
||||
<label className="flex items-center gap-2">
|
||||
<input type="checkbox" checked={override} onChange={(e) => setOverride(e.target.checked)} />
|
||||
Override: also bring back removed or rejected games
|
||||
</label>
|
||||
<Button className="ml-auto" disabled={!urls.length || urls.length > 200 || add.isPending} onClick={() => add.mutate()}>
|
||||
{add.isPending ? 'Adding…' : `Add ${urls.length || ''}`}
|
||||
<div className="mt-1 flex flex-wrap items-center gap-3 text-sm">
|
||||
<span className="text-muted-foreground">
|
||||
{found ? `${plural(found, 'game URL')} found` : 'No itch.io game URL yet'}
|
||||
{repeats > 0 && ` (${plural(repeats, 'repeat')} ignored)`}
|
||||
</span>
|
||||
<input
|
||||
ref={file}
|
||||
type="file"
|
||||
accept=".txt,.csv,.md,.html,.htm,.json,text/plain,text/csv,text/markdown,text/html,application/json"
|
||||
className="hidden"
|
||||
onChange={(e) => {
|
||||
void load(e.target.files)
|
||||
e.target.value = ''
|
||||
}}
|
||||
/>
|
||||
<Button size="sm" variant="ghost" onClick={() => file.current?.click()}>
|
||||
Load a file…
|
||||
</Button>
|
||||
{text && (
|
||||
<Button size="sm" variant="ghost" onClick={() => onText('')}>
|
||||
Clear
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
{error && (
|
||||
<div className="mt-4">
|
||||
<Notice text={error} tone="error" onClose={() => setError(null)} />
|
||||
function CollectionInput({
|
||||
collected,
|
||||
onCollected,
|
||||
onError,
|
||||
}: {
|
||||
collected: Collected | null
|
||||
onCollected: (c: Collected | null) => void
|
||||
onError: (n: NoticeState) => void
|
||||
}) {
|
||||
const [url, setUrl] = useState('')
|
||||
const [reading, setReading] = useState(false)
|
||||
const [watching, setWatching] = useState(false)
|
||||
const stop = useRef(false)
|
||||
const client = useQueryClient()
|
||||
|
||||
/** Ten pages per request, until the end of the collection or Stop. */
|
||||
const read = async () => {
|
||||
stop.current = false
|
||||
setReading(true)
|
||||
onError(null)
|
||||
let page = 1
|
||||
let free: CollectionItem[] = []
|
||||
const seen = new Set<string>()
|
||||
let priced = 0
|
||||
let pages = 0
|
||||
try {
|
||||
for (;;) {
|
||||
const r = await adminApi.readCollection(url, page)
|
||||
const fresh = r.items.filter((i) => i.price === null && !seen.has(i.url))
|
||||
for (const i of fresh) seen.add(i.url)
|
||||
free = [...free, ...fresh]
|
||||
priced += r.items.filter((i) => i.price !== null).length
|
||||
pages += r.pages
|
||||
onCollected({ read: r.collection, free, priced, pages, nextPage: r.nextPage, stopped: r.stopped })
|
||||
if (r.nextPage === null || r.stopped || stop.current) break
|
||||
page = r.nextPage
|
||||
}
|
||||
} catch (e) {
|
||||
onError({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setReading(false)
|
||||
}
|
||||
}
|
||||
|
||||
const watch = async () => {
|
||||
setWatching(true)
|
||||
try {
|
||||
const { collection } = await adminApi.watch(collected?.read.url ?? url)
|
||||
onError({ tone: 'info', text: `Following collection #${collection.id}: the cron queues its new free games (Settings lists it).` })
|
||||
void client.invalidateQueries({ queryKey: ['settings'] })
|
||||
} catch (e) {
|
||||
onError({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setWatching(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Input
|
||||
value={url}
|
||||
onChange={(e) => setUrl(e.target.value)}
|
||||
placeholder="https://itch.io/c/1234567/collection-name"
|
||||
aria-label="itch.io collection URL"
|
||||
className="max-w-xl flex-1"
|
||||
/>
|
||||
{reading ? (
|
||||
<Button variant="outline" onClick={() => (stop.current = true)}>
|
||||
Stop
|
||||
</Button>
|
||||
) : (
|
||||
<Button variant="outline" disabled={!url.trim()} onClick={() => void read()}>
|
||||
Read collection
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{collected && (
|
||||
<div className="flex flex-wrap items-center gap-3 text-muted-foreground">
|
||||
<span>
|
||||
<a href={collected.read.url} target="_blank" rel="noreferrer noopener" className="font-medium text-foreground hover:underline">
|
||||
Collection #{collected.read.id}
|
||||
</a>
|
||||
: {plural(collected.free.length, 'free game')} from {plural(collected.pages, 'page')}
|
||||
{collected.priced > 0 && ` (${plural(collected.priced, 'priced game')} skipped)`}
|
||||
{reading && ' — reading…'}
|
||||
{!reading && collected.nextPage !== null && ' — stopped before the end'}
|
||||
{collected.stopped && ` — itch.io: ${collected.stopped.replace(/_/g, ' ')}`}
|
||||
</span>
|
||||
<Button size="sm" variant="ghost" disabled={watching} onClick={() => void watch()}>
|
||||
Follow this collection
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{result && (
|
||||
<div className="mt-6">
|
||||
<p className="mb-2 text-sm">
|
||||
{result.results.filter((r) => r.approved).length} queued for ingest ({shortSha(result.sha)}). {PUBLISH_NOTE}
|
||||
</p>
|
||||
<table className="w-full text-sm">
|
||||
<tbody>
|
||||
{result.results.map((r, i) => (
|
||||
<tr key={`${r.input}-${i}`} className="border-b last:border-0">
|
||||
<td className="py-1.5 font-mono text-xs">{r.url ?? r.input}</td>
|
||||
<td className="py-1.5 text-muted-foreground">{OUTCOME[r.outcome] ?? r.outcome}</td>
|
||||
<td className="py-1.5">{r.approved ? 'queued' : r.reason ? r.reason.replace(/_/g, ' ') : '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const STATE_TEXT: Record<TrackItem['state'], string> = {
|
||||
waiting: 'waiting for the ingest',
|
||||
added: 'added',
|
||||
dropped: 'not added',
|
||||
unknown: 'unknown',
|
||||
}
|
||||
|
||||
/** The batches queued from this browser, and how each game's ingest went. */
|
||||
function RecentBatches() {
|
||||
const batches = useTracking((s) => s.batches)
|
||||
if (batches.length === 0) return null
|
||||
return (
|
||||
<div className="mt-8">
|
||||
<h2 className="mb-2 text-base font-semibold">Recent batches</h2>
|
||||
<p className="mb-3 text-xs text-muted-foreground">
|
||||
Queued from this browser. The ingest workflow starts right after each commit; results show up here as it
|
||||
finishes (checked every minute while something is waiting).
|
||||
</p>
|
||||
<div className="space-y-2">
|
||||
{batches.map((b, i) => (
|
||||
<BatchCard key={b.id} batch={b} live={i < 3} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
async function trackAll(batch: TrackedBatch): Promise<TrackItem[]> {
|
||||
const items: TrackItem[] = []
|
||||
for (const part of chunk(batch.urls, MAX_TRACK_URLS)) items.push(...(await adminApi.track(part, batch.at)).items)
|
||||
return items
|
||||
}
|
||||
|
||||
function BatchCard({ batch, live }: { batch: TrackedBatch; live: boolean }) {
|
||||
const [open, setOpen] = useState(false)
|
||||
const forget = useTracking((s) => s.forget)
|
||||
const status = useQuery({
|
||||
queryKey: ['track', batch.id],
|
||||
queryFn: () => trackAll(batch),
|
||||
enabled: live || open,
|
||||
refetchInterval: (q) => (q.state.data?.some((i) => i.state === 'waiting') ? 60_000 : false),
|
||||
refetchIntervalInBackground: false,
|
||||
})
|
||||
const sum = status.data ? summarize(status.data) : null
|
||||
return (
|
||||
<details className="rounded-md border px-3 py-2 text-sm" onToggle={(e) => setOpen(e.currentTarget.open)}>
|
||||
<summary className="flex cursor-pointer flex-wrap items-center gap-x-3 gap-y-1">
|
||||
<span className="font-medium">{batch.label}</span>
|
||||
<span className="text-xs text-muted-foreground">{when(batch.at)}</span>
|
||||
{sum && (
|
||||
<span className="text-xs tabular-nums">
|
||||
<span className="text-success">{sum.added} added</span>
|
||||
{sum.waiting > 0 && <span className="text-muted-foreground"> · {sum.waiting} waiting</span>}
|
||||
{sum.dropped > 0 && <span className="text-destructive"> · {sum.dropped} not added</span>}
|
||||
{sum.unknown > 0 && <span className="text-muted-foreground"> · {sum.unknown} unknown</span>}
|
||||
</span>
|
||||
)}
|
||||
{status.isFetching && <span className="text-xs text-muted-foreground">checking…</span>}
|
||||
{status.error && <span className="text-xs text-destructive">{errorText(status.error).split('\n')[0]}</span>}
|
||||
<button
|
||||
className="ml-auto text-xs text-muted-foreground underline-offset-4 hover:underline"
|
||||
onClick={(e) => {
|
||||
e.preventDefault()
|
||||
void status.refetch()
|
||||
}}
|
||||
>
|
||||
Check now
|
||||
</button>
|
||||
<button
|
||||
className="text-xs text-muted-foreground underline-offset-4 hover:underline"
|
||||
onClick={(e) => {
|
||||
e.preventDefault()
|
||||
forget(batch.id)
|
||||
}}
|
||||
>
|
||||
Forget
|
||||
</button>
|
||||
</summary>
|
||||
{status.data && (
|
||||
<table className="mt-2 w-full text-xs">
|
||||
<tbody>
|
||||
{status.data.map((i) => (
|
||||
<tr key={i.url} className="border-b last:border-0">
|
||||
<td className="max-w-0 truncate py-1 pr-2 font-mono">
|
||||
<a href={i.url} target="_blank" rel="noreferrer noopener" className="hover:underline">
|
||||
{i.url.replace('https://', '')}
|
||||
</a>
|
||||
</td>
|
||||
<td className="py-1 whitespace-nowrap text-muted-foreground">
|
||||
{i.outcome ? (INGEST_TEXT[i.outcome] ?? i.outcome) : STATE_TEXT[i.state]}
|
||||
{i.attempts ? ` (${plural(i.attempts, 'failed attempt')}: ${i.error ?? ''})` : ''}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</section>
|
||||
</details>
|
||||
)
|
||||
}
|
||||
+145
-31
@@ -11,7 +11,7 @@ export class SessionExpired extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
class ApiError extends Error {
|
||||
export class ApiError extends Error {
|
||||
readonly status: number
|
||||
readonly code: string
|
||||
|
||||
@@ -42,13 +42,18 @@ export function isSessionExpired(): boolean {
|
||||
}
|
||||
|
||||
async function api<T>(method: string, path: string, body?: unknown): Promise<T> {
|
||||
const res = await fetch(path, {
|
||||
method,
|
||||
credentials: 'same-origin',
|
||||
redirect: 'manual',
|
||||
headers: body === undefined ? undefined : { 'Content-Type': 'application/json' },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
})
|
||||
let res: Response
|
||||
try {
|
||||
res = await fetch(path, {
|
||||
method,
|
||||
credentials: 'same-origin',
|
||||
redirect: 'manual',
|
||||
headers: body === undefined ? undefined : { 'Content-Type': 'application/json' },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
})
|
||||
} catch {
|
||||
throw new ApiError(0, 'network_error')
|
||||
}
|
||||
if (res.type === 'opaqueredirect' || res.status === 401) throw new SessionExpired()
|
||||
const data = (await res.json().catch(() => ({}))) as { error?: string; message?: string }
|
||||
if (!res.ok) throw new ApiError(res.status, data.error ?? `http_${res.status}`, data.message)
|
||||
@@ -56,10 +61,11 @@ async function api<T>(method: string, path: string, body?: unknown): Promise<T>
|
||||
}
|
||||
|
||||
export type CandidateStatus = 'pending' | 'rejected' | 'queued' | 'ingested' | 'failed'
|
||||
export type CandidateSource = 'rss' | 'form' | 'admin'
|
||||
|
||||
export interface Candidate {
|
||||
url: string
|
||||
source: 'rss' | 'form' | 'ext' | 'admin'
|
||||
source: CandidateSource
|
||||
status: CandidateStatus
|
||||
title: string | null
|
||||
image_url: string | null
|
||||
@@ -69,57 +75,165 @@ export interface Candidate {
|
||||
submitter: string | null
|
||||
discovered_at: string
|
||||
decided_at: string | null
|
||||
decided_by: string | null
|
||||
/** How the last ingest ended (added, paid, dead, private, …, timeout). */
|
||||
outcome: string | null
|
||||
outcome_at: string | null
|
||||
}
|
||||
|
||||
export interface Me {
|
||||
email: string
|
||||
writes: boolean
|
||||
/** "owner/name" of the data repository, for commit links. */
|
||||
repo: string | null
|
||||
}
|
||||
|
||||
export interface Decision {
|
||||
sha: string | null
|
||||
approved: string[]
|
||||
skipped: { url: string; reason: string }[]
|
||||
at: string
|
||||
}
|
||||
|
||||
export interface CheckResult {
|
||||
input: string
|
||||
url: string | null
|
||||
outcome: string
|
||||
/** The queue status of a URL the queue already knows. */
|
||||
status: CandidateStatus | null
|
||||
reason?: string
|
||||
}
|
||||
|
||||
export interface AddResult {
|
||||
sha: string | null
|
||||
at: string
|
||||
results: { input: string; url: string | null; outcome: string; approved: boolean; reason?: string }[]
|
||||
}
|
||||
|
||||
export interface TrackItem {
|
||||
url: string
|
||||
state: 'waiting' | 'added' | 'dropped' | 'unknown'
|
||||
outcome?: string
|
||||
at?: string
|
||||
attempts?: number
|
||||
error?: string | null
|
||||
}
|
||||
|
||||
export type GameEdit = Partial<Record<'safe_virus' | 'nsfw' | 'notes', string>>
|
||||
|
||||
export interface RemoveResult {
|
||||
sha: string | null
|
||||
removed: string[]
|
||||
notFound: string[]
|
||||
invalid: string[]
|
||||
covers: { deleted: number; failed: number }
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
export interface BulkEditResult {
|
||||
sha: string | null
|
||||
edited: string[]
|
||||
unchanged: string[]
|
||||
notFound: string[]
|
||||
}
|
||||
|
||||
export interface AuditRow {
|
||||
id: number
|
||||
at: string
|
||||
actor: string
|
||||
action: string
|
||||
target: string | null
|
||||
commit_sha: string | null
|
||||
detail: string | null
|
||||
}
|
||||
|
||||
export interface Automation {
|
||||
enabled: boolean
|
||||
sources: CandidateSource[]
|
||||
hours: number
|
||||
since: string
|
||||
}
|
||||
|
||||
interface LastRun {
|
||||
at: string
|
||||
approved: number
|
||||
added: number
|
||||
review: number
|
||||
duplicates: number
|
||||
pages: number
|
||||
sha: string | null
|
||||
errors: string[]
|
||||
collections: { id: string; status: number | null; found: number; added: number; next_page: number }[]
|
||||
}
|
||||
|
||||
export interface WatchedCollection {
|
||||
id: string
|
||||
slug: string
|
||||
enabled: number
|
||||
added_at: string
|
||||
last_polled_at: string | null
|
||||
last_status: number | null
|
||||
backoff_until: string | null
|
||||
next_page: number
|
||||
last_found: number | null
|
||||
last_added: number | null
|
||||
}
|
||||
|
||||
export interface SettingsData {
|
||||
automation: Automation
|
||||
lastRun: LastRun | null
|
||||
collections: WatchedCollection[]
|
||||
writes: boolean
|
||||
}
|
||||
|
||||
export interface CollectionItem {
|
||||
url: string
|
||||
title: string
|
||||
image: string | null
|
||||
genre: string | null
|
||||
price: string | null
|
||||
}
|
||||
|
||||
export interface CollectionRead {
|
||||
collection: { id: string; slug: string; url: string }
|
||||
items: CollectionItem[]
|
||||
pages: number
|
||||
nextPage: number | null
|
||||
/** Set when the read stopped early on an error, after some pages. */
|
||||
stopped?: string
|
||||
}
|
||||
|
||||
export const adminApi = {
|
||||
me: () => api<Me>('GET', '/api/admin/me'),
|
||||
queue: (status: CandidateStatus) =>
|
||||
api<{ counts: Record<CandidateStatus, number>; items: Candidate[] }>(
|
||||
'GET',
|
||||
`/api/admin/queue?status=${status}&limit=500`,
|
||||
),
|
||||
api<{ counts: Record<CandidateStatus, number>; items: Candidate[] }>('GET', `/api/admin/queue?status=${status}&limit=500`),
|
||||
approve: (urls: string[], override: boolean) =>
|
||||
api<Decision>('POST', '/api/admin/queue/decide', { action: 'approve', urls, override }),
|
||||
reject: (urls: string[]) => api<{ rejected: string[] }>('POST', '/api/admin/queue/decide', { action: 'reject', urls }),
|
||||
add: (urls: string[], override: boolean) => api<AddResult>('POST', '/api/admin/add', { urls, override }),
|
||||
check: (urls: string[]) => api<{ results: CheckResult[] }>('POST', '/api/admin/check', { urls }),
|
||||
add: (urls: string[], override: boolean, via?: string) =>
|
||||
api<AddResult>('POST', '/api/admin/add', { urls, override, ...(via ? { via } : {}) }),
|
||||
track: (urls: string[], since: string) => api<{ head: string; items: TrackItem[] }>('POST', '/api/admin/track', { urls, since }),
|
||||
edit: (url: string, edit: GameEdit) => api<{ sha: string | null; edited: number }>('PATCH', '/api/admin/games', { url, edit }),
|
||||
remove: (url: string, reason: string) =>
|
||||
api<{ sha: string | null; removed: number }>('DELETE', '/api/admin/games', { url, reason }),
|
||||
restore: (url: string) => api<{ sha: string | null; queued: boolean }>('POST', '/api/admin/games/restore', { url }),
|
||||
editMany: (edits: { url: string; edit: GameEdit }[]) => api<BulkEditResult>('PATCH', '/api/admin/games/bulk', { edits }),
|
||||
removeMany: (urls: string[], reason: string) => api<RemoveResult>('POST', '/api/admin/games/remove', { urls, reason }),
|
||||
restore: (url: string) => api<{ sha: string | null; queued: boolean; at: string }>('POST', '/api/admin/games/restore', { url }),
|
||||
activity: (params: { before?: number | null; actions?: string[] }) => {
|
||||
const q = new URLSearchParams({ limit: '50' })
|
||||
if (params.before) q.set('before', String(params.before))
|
||||
if (params.actions?.length) q.set('actions', params.actions.join(','))
|
||||
return api<{ items: AuditRow[]; next: number | null }>('GET', `/api/admin/activity?${q}`)
|
||||
},
|
||||
settings: () => api<SettingsData>('GET', '/api/admin/settings'),
|
||||
saveAutomation: (automation: Omit<Automation, 'since'> & { includeOlder?: boolean }) =>
|
||||
api<{ automation: Automation }>('PUT', '/api/admin/settings', { automation }),
|
||||
readCollection: (url: string, page: number) => api<CollectionRead>('POST', '/api/admin/collections/read', { url, page }),
|
||||
watch: (url: string) => api<{ collection: WatchedCollection }>('POST', '/api/admin/collections', { url }),
|
||||
toggleCollection: (id: string, enabled: boolean) =>
|
||||
api<{ collection: WatchedCollection }>('PATCH', '/api/admin/collections', { id, enabled }),
|
||||
unwatch: (id: string) => api<{ removed: boolean }>('DELETE', '/api/admin/collections', { id }),
|
||||
}
|
||||
|
||||
export function shortSha(sha: string | null): string {
|
||||
return sha ? sha.slice(0, 7) : 'no commit'
|
||||
}
|
||||
|
||||
export function errorText(e: unknown): string {
|
||||
if (e instanceof SessionExpired) return e.message
|
||||
if (e instanceof ApiError) {
|
||||
const known: Record<string, string> = {
|
||||
forbidden: 'This account is not allowed to use the admin.',
|
||||
not_configured: 'The admin is not configured yet (Cloudflare Access / D1).',
|
||||
github_not_configured: 'Writes are off: the GitHub App is not configured.',
|
||||
needs_override: 'Previously removed — approve with override.',
|
||||
}
|
||||
return known[e.code] ?? e.message
|
||||
}
|
||||
return e instanceof Error ? e.message : String(e)
|
||||
}
|
||||
+35
-33
@@ -1,19 +1,29 @@
|
||||
import { useSyncExternalStore } from 'react'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { NavLink, Navigate, Route, Routes } from 'react-router'
|
||||
import { useThemeEffect } from '@/hooks/useThemeEffect'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { SessionExpired, adminApi, errorText, isSessionExpired, subscribeSession } from './api'
|
||||
import { SessionExpired, adminApi, isSessionExpired, subscribeSession } from './api'
|
||||
import { ActivityPage } from './activity'
|
||||
import { AddPage } from './add'
|
||||
import { CatalogPage } from './catalog'
|
||||
import { NewGamesPage } from './new-games'
|
||||
import { QueuePage } from './queue'
|
||||
import { SettingsPage } from './settings'
|
||||
import { errorText } from './texts'
|
||||
import { ConfirmDialog, ThemeToggle } from './ui'
|
||||
|
||||
const NAV = [
|
||||
{ to: '/queue', label: 'Review queue' },
|
||||
{ to: '/add', label: 'Add games' },
|
||||
{ to: '/catalog', label: 'Catalog' },
|
||||
{ to: '/new', label: 'New games' },
|
||||
{ to: '/activity', label: 'Activity' },
|
||||
{ to: '/settings', label: 'Settings' },
|
||||
]
|
||||
|
||||
export function AdminApp() {
|
||||
useThemeEffect()
|
||||
const me = useQuery({ queryKey: ['me'], queryFn: adminApi.me })
|
||||
const expired = useSyncExternalStore(subscribeSession, isSessionExpired)
|
||||
|
||||
@@ -21,7 +31,7 @@ export function AdminApp() {
|
||||
return (
|
||||
<div className="mx-auto max-w-lg p-8 text-sm">
|
||||
<h1 className="mb-2 text-lg font-semibold">Admin unavailable</h1>
|
||||
<p className="mb-4 text-muted-foreground">{errorText(me.error)}</p>
|
||||
<p className="mb-4 whitespace-pre-line text-muted-foreground">{errorText(me.error)}</p>
|
||||
{me.error instanceof SessionExpired && (
|
||||
<button className="underline" onClick={() => window.location.reload()}>
|
||||
Sign in again
|
||||
@@ -34,28 +44,36 @@ export function AdminApp() {
|
||||
return (
|
||||
<div className="min-h-screen">
|
||||
<header className="sticky top-0 z-10 border-b bg-background/95 backdrop-blur-sm">
|
||||
<div className="mx-auto flex h-12 max-w-6xl items-center gap-6 px-4 text-sm">
|
||||
<span className="font-semibold">freeitchgames.win · admin</span>
|
||||
<nav className="flex gap-1">
|
||||
<div className="mx-auto flex h-12 max-w-6xl items-center gap-4 px-4 text-sm">
|
||||
<span className="shrink-0 font-semibold">freeitchgames.win · admin</span>
|
||||
<nav className="scrollbar-hide flex min-w-0 gap-1 overflow-x-auto">
|
||||
{NAV.map((n) => (
|
||||
<NavLink
|
||||
key={n.to}
|
||||
to={n.to}
|
||||
className={({ isActive }) =>
|
||||
cn('rounded-md px-3 py-1.5 hover:bg-accent', isActive && 'bg-accent font-medium')
|
||||
cn('shrink-0 rounded-md px-3 py-1.5 hover:bg-accent', isActive && 'bg-accent font-medium')
|
||||
}
|
||||
>
|
||||
{n.label}
|
||||
</NavLink>
|
||||
))}
|
||||
</nav>
|
||||
<span className="ml-auto truncate text-muted-foreground">
|
||||
{me.data ? me.data.email : '…'}
|
||||
{me.data && !me.data.writes && ' · read-only (GitHub App not configured)'}
|
||||
</span>
|
||||
<a className="text-muted-foreground hover:text-foreground" href="/" target="_blank" rel="noreferrer">
|
||||
Site ↗
|
||||
</a>
|
||||
<div className="ml-auto flex min-w-0 items-center gap-3">
|
||||
<span className="hidden truncate text-muted-foreground xl:inline">{me.data ? me.data.email : '…'}</span>
|
||||
{me.data && !me.data.writes && (
|
||||
<span
|
||||
className="shrink-0 rounded-md border border-destructive/40 px-2 py-0.5 text-xs text-destructive"
|
||||
title="The GitHub App is not configured: nothing can be written."
|
||||
>
|
||||
read-only
|
||||
</span>
|
||||
)}
|
||||
<ThemeToggle />
|
||||
<a className="shrink-0 text-muted-foreground hover:text-foreground" href="/" target="_blank" rel="noreferrer">
|
||||
Site ↗
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
{expired && (
|
||||
@@ -76,29 +94,13 @@ export function AdminApp() {
|
||||
<Route path="/queue" element={<QueuePage />} />
|
||||
<Route path="/add" element={<AddPage />} />
|
||||
<Route path="/catalog" element={<CatalogPage />} />
|
||||
<Route path="/new" element={<NewGamesPage />} />
|
||||
<Route path="/activity" element={<ActivityPage repo={me.data?.repo ?? null} />} />
|
||||
<Route path="/settings" element={<SettingsPage />} />
|
||||
<Route path="*" element={<Navigate to="/queue" replace />} />
|
||||
</Routes>
|
||||
</main>
|
||||
<ConfirmDialog />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** A dismissible line reporting what the last action did. */
|
||||
export function Notice({ text, tone = 'info', onClose }: { text: string; tone?: 'info' | 'error'; onClose: () => void }) {
|
||||
return (
|
||||
<div
|
||||
role={tone === 'error' ? 'alert' : 'status'}
|
||||
className={cn(
|
||||
'mb-4 flex items-start gap-3 rounded-md border px-3 py-2 text-sm',
|
||||
tone === 'error' ? 'border-destructive/50 text-destructive' : 'bg-muted/50',
|
||||
)}
|
||||
>
|
||||
<span className="flex-1 whitespace-pre-line">{text}</span>
|
||||
<button aria-label="Dismiss" className="text-muted-foreground hover:text-foreground" onClick={onClose}>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export const PUBLISH_NOTE = 'The public site shows it after the next build (a few minutes).'
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { chunk, runBatches } from './batch'
|
||||
|
||||
describe('batches', () => {
|
||||
it('splits a list into batches of a given size', () => {
|
||||
expect(chunk([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]])
|
||||
expect(chunk([], 200)).toEqual([])
|
||||
})
|
||||
|
||||
it('sends batches one at a time and stops at the first error, keeping the rest', async () => {
|
||||
const progress = vi.fn()
|
||||
const send = vi.fn(async (batch: number[]) => {
|
||||
if (batch[0] === 3) throw new Error('down')
|
||||
return batch.length
|
||||
})
|
||||
const run = await runBatches([[1, 2], [3], [4]], send, progress)
|
||||
expect(run.results).toEqual([2])
|
||||
expect(run.failed?.batch).toEqual([3])
|
||||
expect(run.remaining).toEqual([[3], [4]])
|
||||
expect(send).toHaveBeenCalledTimes(2)
|
||||
expect(progress.mock.calls).toEqual([
|
||||
[0, 3],
|
||||
[1, 3],
|
||||
])
|
||||
expect((await runBatches([[1]], async () => 'ok')).failed).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
/** Splitting long lists into the API's batch sizes and sending them one after another. */
|
||||
|
||||
export function chunk<T>(items: readonly T[], size: number): T[][] {
|
||||
const out: T[][] = []
|
||||
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))
|
||||
return out
|
||||
}
|
||||
|
||||
export interface BatchRun<T, R> {
|
||||
results: R[]
|
||||
/** The first batch that failed (nothing after it was sent), with its error. */
|
||||
failed: { batch: T[]; error: unknown } | null
|
||||
/** The batches not sent yet (the failed one first): retry by running them again. */
|
||||
remaining: T[][]
|
||||
}
|
||||
|
||||
/**
|
||||
* Send `batches` one at a time (each is one commit / one request). Stops at the
|
||||
* first error, so a lost session or a GitHub outage never leaves half a list
|
||||
* silently skipped; the rest can be run again as it is.
|
||||
*/
|
||||
export async function runBatches<T, R>(
|
||||
batches: T[][],
|
||||
send: (batch: T[], index: number) => Promise<R>,
|
||||
onProgress?: (done: number, total: number) => void,
|
||||
): Promise<BatchRun<T, R>> {
|
||||
const results: R[] = []
|
||||
onProgress?.(0, batches.length)
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
try {
|
||||
results.push(await send(batches[i], i))
|
||||
} catch (error) {
|
||||
return { results, failed: { batch: batches[i], error }, remaining: batches.slice(i) }
|
||||
}
|
||||
onProgress?.(i + 1, batches.length)
|
||||
}
|
||||
return { results, failed: null, remaining: [] }
|
||||
}
|
||||
+243
-129
@@ -1,12 +1,20 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { useQueryClient } from '@tanstack/react-query'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { useAllGames, useDeletedGames } from '@/hooks/useGames'
|
||||
import { cn } from '@/lib/utils'
|
||||
import type { Game } from '@/types/game'
|
||||
import { adminApi, errorText, shortSha, type GameEdit } from './api'
|
||||
import { Notice, PUBLISH_NOTE } from './app'
|
||||
import { MAX_BULK } from '../worker/limits'
|
||||
import { adminApi, shortSha, type GameEdit } from './api'
|
||||
import { confirm } from './confirm'
|
||||
import { BulkEditDialog, RemoveDialog, type Picked } from './dialogs'
|
||||
import { extractGameUrls } from './extract'
|
||||
import { rememberEdits, useOverrides, withEdits } from './overrides'
|
||||
import { useSelection } from './selection'
|
||||
import { PUBLISH_NOTE, errorText, plural } from './texts'
|
||||
import { useTracking } from './tracking'
|
||||
import { Notice, SelectionBar, TabBar, type NoticeState } from './ui'
|
||||
|
||||
const MAX_RESULTS = 50
|
||||
|
||||
@@ -15,126 +23,173 @@ function matches(query: string, ...fields: string[]): boolean {
|
||||
return q.length > 0 && fields.some((f) => f.toLowerCase().includes(q))
|
||||
}
|
||||
|
||||
type Tab = 'catalog' | 'removed' | 'list'
|
||||
|
||||
export function CatalogPage() {
|
||||
const games = useAllGames()
|
||||
const deleted = useDeletedGames()
|
||||
const [query, setQuery] = useState('')
|
||||
const [selected, setSelected] = useState<Game | null>(null)
|
||||
const [tab, setTab] = useState<'catalog' | 'removed'>('catalog')
|
||||
// Values saved in this session, per game: the published /data lags behind
|
||||
// the repository until the next build, so it can't be the baseline.
|
||||
const [saved] = useState(() => new Map<string, EditableValues>())
|
||||
|
||||
const found = useMemo(
|
||||
() => (games.data?.games ?? []).filter((g) => matches(query, g.name, g.url, g.dev)).slice(0, MAX_RESULTS),
|
||||
[games.data, query],
|
||||
)
|
||||
const removed = useMemo(
|
||||
() => (deleted.data ?? []).filter((d) => !query.trim() || matches(query, d.name, d.url)).slice(-MAX_RESULTS).reverse(),
|
||||
[deleted.data, query],
|
||||
)
|
||||
const [tab, setTab] = useState<Tab>('catalog')
|
||||
|
||||
return (
|
||||
<section className="grid gap-6 md:grid-cols-[minmax(0,1fr)_minmax(0,1.2fr)]">
|
||||
<section>
|
||||
<TabBar
|
||||
label="Catalog, removed games or removal by list"
|
||||
value={tab}
|
||||
onChange={setTab}
|
||||
options={[
|
||||
{ id: 'catalog', label: `Catalog ${games.data ? `(${games.data.games.length.toLocaleString('en-GB')})` : ''}` },
|
||||
{ id: 'removed', label: 'Removed' },
|
||||
{ id: 'list', label: 'Remove by list' },
|
||||
]}
|
||||
/>
|
||||
<p className="mb-3 text-xs text-muted-foreground">
|
||||
Data of the published site; changes apply to the latest repository state and show here at once.
|
||||
</p>
|
||||
{tab === 'catalog' && <CatalogSearch games={games.data?.games ?? []} loading={games.isPending} />}
|
||||
{tab === 'removed' && <RemovedList entries={deleted.data ?? []} />}
|
||||
{tab === 'list' && <RemoveByList games={games.data?.games ?? []} deleted={deleted.data ?? []} />}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
/** Search, select up to 100 games for a bulk edit or removal, or edit one in detail. */
|
||||
function CatalogSearch({ games, loading }: { games: Game[]; loading: boolean }) {
|
||||
const [query, setQuery] = useState('')
|
||||
const [onlySelected, setOnlySelected] = useState(false)
|
||||
const [focused, setFocused] = useState<Game | null>(null)
|
||||
const [dialog, setDialog] = useState<{ kind: 'edit' | 'remove'; games: Picked[] } | null>(null)
|
||||
const selection = useSelection()
|
||||
const { edits, removed } = useOverrides()
|
||||
|
||||
const byUrl = useMemo(() => new Map(games.map((g) => [g.url, g])), [games])
|
||||
const found = useMemo(() => {
|
||||
const list = onlySelected
|
||||
? [...selection.selected].map((u) => byUrl.get(u)).filter((g): g is Game => g !== undefined)
|
||||
: games.filter((g) => matches(query, g.name, g.url, g.dev)).slice(0, MAX_RESULTS)
|
||||
return list.map((g) => withEdits(g, edits))
|
||||
}, [games, byUrl, query, onlySelected, selection.selected, edits])
|
||||
const picked: Picked[] = [...selection.selected].map((url) => ({ url, name: byUrl.get(url)?.name ?? url }))
|
||||
const selectable = found.filter((g) => !removed.has(g.url))
|
||||
|
||||
return (
|
||||
<div className="grid gap-6 md:grid-cols-[minmax(0,1fr)_minmax(0,1.1fr)]">
|
||||
<div>
|
||||
<div className="mb-3 flex gap-1 text-sm" role="tablist" aria-label="Catalog or removed games">
|
||||
{(['catalog', 'removed'] as const).map((t) => (
|
||||
<button
|
||||
key={t}
|
||||
role="tab"
|
||||
aria-selected={tab === t}
|
||||
onClick={() => setTab(t)}
|
||||
className={cn('rounded-md px-3 py-1.5 hover:bg-accent', tab === t && 'bg-accent font-medium')}
|
||||
>
|
||||
{t === 'catalog' ? `Catalog ${games.data ? `(${games.data.games.length})` : ''}` : 'Removed'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<Input
|
||||
type="search"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
onChange={(e) => {
|
||||
setQuery(e.target.value)
|
||||
setOnlySelected(false)
|
||||
}}
|
||||
placeholder="Search by name, developer or URL"
|
||||
aria-label="Search games"
|
||||
/>
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
Data of the published site; edits apply to the latest repository state.
|
||||
</p>
|
||||
{tab === 'catalog' ? (
|
||||
<ul className="mt-3 divide-y text-sm">
|
||||
{games.isPending && <li className="py-2 text-muted-foreground">Loading catalog…</li>}
|
||||
{found.map((g) => (
|
||||
<li key={g.url}>
|
||||
<div className="mt-2 flex flex-wrap items-center gap-3 text-xs text-muted-foreground">
|
||||
<button
|
||||
className="underline-offset-4 hover:underline disabled:opacity-50"
|
||||
disabled={selectable.length === 0 || selection.full}
|
||||
onClick={() => selection.setMany(selectable.map((g) => g.url), true)}
|
||||
>
|
||||
Select shown
|
||||
</button>
|
||||
<button
|
||||
className="underline-offset-4 hover:underline disabled:opacity-50"
|
||||
disabled={selection.selected.size === 0}
|
||||
onClick={() => setOnlySelected((v) => !v)}
|
||||
>
|
||||
{onlySelected ? 'Show search results' : `Show selected (${selection.selected.size})`}
|
||||
</button>
|
||||
{!onlySelected && query && found.length === MAX_RESULTS && <span>first {MAX_RESULTS} matches</span>}
|
||||
</div>
|
||||
|
||||
<SelectionBar count={selection.selected.size} max={MAX_BULK} onClear={selection.clear}>
|
||||
<Button size="sm" variant="outline" onClick={() => setDialog({ kind: 'edit', games: picked })}>
|
||||
Edit…
|
||||
</Button>
|
||||
<Button size="sm" variant="destructive" onClick={() => setDialog({ kind: 'remove', games: picked })}>
|
||||
Remove…
|
||||
</Button>
|
||||
</SelectionBar>
|
||||
|
||||
<ul className="mt-3 divide-y text-sm">
|
||||
{loading && <li className="py-2 text-muted-foreground">Loading catalog…</li>}
|
||||
{found.map((g) => {
|
||||
const gone = removed.has(g.url)
|
||||
return (
|
||||
<li key={g.url} className={cn('flex items-center gap-3', gone && 'opacity-50')}>
|
||||
<input
|
||||
type="checkbox"
|
||||
aria-label={`Select ${g.name}`}
|
||||
disabled={gone || (!selection.selected.has(g.url) && selection.full)}
|
||||
checked={selection.selected.has(g.url)}
|
||||
onChange={() => selection.toggle(g.url)}
|
||||
/>
|
||||
<button
|
||||
onClick={() => setSelected(g)}
|
||||
className={cn('w-full py-2 text-left hover:bg-accent/50', selected?.url === g.url && 'bg-accent')}
|
||||
onClick={() => setFocused(g)}
|
||||
className={cn('min-w-0 flex-1 py-2 text-left hover:bg-accent/50', focused?.url === g.url && 'bg-accent')}
|
||||
>
|
||||
<span className="font-medium">{g.name}</span>
|
||||
<span className="block text-xs text-muted-foreground">{g.url.replace('https://', '')}</span>
|
||||
{gone && <span className="ml-2 text-xs text-destructive">removed (publishing…)</span>}
|
||||
{g.nsfw === 'Yes' && <span className="ml-2 text-xs text-muted-foreground">18+</span>}
|
||||
<span className="block truncate text-xs text-muted-foreground">{g.url.replace('https://', '')}</span>
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
{query && games.data && found.length === 0 && <li className="py-2 text-muted-foreground">No match.</li>}
|
||||
</ul>
|
||||
) : (
|
||||
<RemovedList entries={removed} />
|
||||
)}
|
||||
)
|
||||
})}
|
||||
{query && !loading && found.length === 0 && <li className="py-2 text-muted-foreground">No match.</li>}
|
||||
</ul>
|
||||
</div>
|
||||
<div>
|
||||
{selected && tab === 'catalog' && (
|
||||
{focused && (
|
||||
<GameEditor
|
||||
key={selected.url}
|
||||
game={selected}
|
||||
known={saved.get(selected.url) ?? published(selected)}
|
||||
onSaved={(values) => saved.set(selected.url, values)}
|
||||
key={focused.url}
|
||||
game={withEdits(focused, edits)}
|
||||
removed={removed.has(focused.url)}
|
||||
onRemove={() => setDialog({ kind: 'remove', games: [{ url: focused.url, name: focused.name }] })}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{dialog?.kind === 'edit' && <BulkEditDialog games={dialog.games} onClose={() => setDialog(null)} />}
|
||||
{dialog?.kind === 'remove' && (
|
||||
<RemoveDialog games={dialog.games} onClose={() => setDialog(null)} onRemoved={(urls) => selection.setMany(urls, false)} />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
type EditableValues = Required<GameEdit>
|
||||
const EDITABLE = ['safe_virus', 'nsfw', 'notes'] as const
|
||||
|
||||
function published(game: Game): EditableValues {
|
||||
function values(game: Game): EditableValues {
|
||||
return { safe_virus: game.safe_virus, nsfw: game.nsfw, notes: game.notes ?? '' }
|
||||
}
|
||||
|
||||
/**
|
||||
* `known` is what the repository holds as far as this page knows: the
|
||||
* published data, or what was saved in this session. Changes are computed
|
||||
* against it, so a saved edit can be reverted and never silently dropped.
|
||||
*/
|
||||
function GameEditor({ game, known, onSaved }: { game: Game; known: EditableValues; onSaved: (v: EditableValues) => void }) {
|
||||
const [baseline, setBaseline] = useState<EditableValues>(known)
|
||||
/** One game in detail. `game` carries the edits saved in this session (the published data lags). */
|
||||
function GameEditor({ game, removed, onRemove }: { game: Game; removed: boolean; onRemove: () => void }) {
|
||||
const known = values(game)
|
||||
const [edit, setEdit] = useState<EditableValues>(known)
|
||||
const [reason, setReason] = useState('')
|
||||
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
|
||||
const [saving, setSaving] = useState(false)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
|
||||
const changed = EDITABLE.filter((k) => edit[k] !== baseline[k])
|
||||
const save = useMutation({
|
||||
mutationFn: (sent: GameEdit) => adminApi.edit(game.url, sent),
|
||||
onSuccess: (r, sent) => {
|
||||
const next = { ...baseline, ...sent }
|
||||
setBaseline(next)
|
||||
onSaved(next)
|
||||
setNotice({
|
||||
tone: 'info',
|
||||
text: r.edited ? `Saved (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Nothing changed in the repository.',
|
||||
})
|
||||
},
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
const changed = EDITABLE.filter((k) => edit[k] !== known[k])
|
||||
const save = async () => {
|
||||
const sent: GameEdit = Object.fromEntries(changed.map((k) => [k, edit[k]]))
|
||||
setSaving(true)
|
||||
try {
|
||||
const r = await adminApi.edit(game.url, sent)
|
||||
rememberEdits([{ url: game.url, edit: sent }])
|
||||
setNotice({ tone: 'info', text: r.edited ? `Saved (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Nothing changed in the repository.' })
|
||||
} catch (e) {
|
||||
setNotice({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}
|
||||
const change = (next: EditableValues) => {
|
||||
setEdit(next)
|
||||
if (notice?.tone === 'info') setNotice(null)
|
||||
}
|
||||
const remove = useMutation({
|
||||
mutationFn: () => adminApi.remove(game.url, reason.trim()),
|
||||
onSuccess: (r) => setNotice({ tone: 'info', text: `Removed (${shortSha(r.sha)}); its resized covers were deleted. ${PUBLISH_NOTE}` }),
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border p-4 text-sm">
|
||||
@@ -142,6 +197,7 @@ function GameEditor({ game, known, onSaved }: { game: Game; known: EditableValue
|
||||
<a href={game.url} target="_blank" rel="noreferrer noopener" className="text-xs text-muted-foreground hover:underline">
|
||||
{game.url}
|
||||
</a>
|
||||
{game.added_at && <p className="mt-1 text-xs text-muted-foreground">Added {game.added_at.slice(0, 10)}</p>}
|
||||
<div className="mt-4">{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}</div>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
@@ -181,55 +237,67 @@ function GameEditor({ game, known, onSaved }: { game: Game; known: EditableValue
|
||||
className="rounded-md border bg-background p-2"
|
||||
/>
|
||||
</label>
|
||||
<Button
|
||||
className="mt-3"
|
||||
disabled={!changed.length || save.isPending}
|
||||
onClick={() => save.mutate(Object.fromEntries(changed.map((k) => [k, edit[k]])))}
|
||||
>
|
||||
{save.isPending ? 'Saving…' : 'Save'}
|
||||
<Button className="mt-3" disabled={!changed.length || saving || removed} onClick={() => void save()}>
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</Button>
|
||||
|
||||
<div className="mt-8 border-t pt-4">
|
||||
<h3 className="mb-1 font-medium text-destructive">Remove from the catalog</h3>
|
||||
<p className="mb-2 text-xs text-muted-foreground">
|
||||
The reason is public (removed-games list). Use this for content-removal requests too.
|
||||
The reason is public (Removed page). Use this for content-removal requests too.
|
||||
</p>
|
||||
<div className="flex gap-2">
|
||||
<Input value={reason} maxLength={200} onChange={(e) => setReason(e.target.value)} placeholder="Reason" aria-label="Removal reason" />
|
||||
<Button
|
||||
variant="destructive"
|
||||
disabled={!reason.trim() || remove.isPending}
|
||||
onClick={() => {
|
||||
if (window.confirm(`Remove "${game.name}" from the catalog?`)) remove.mutate()
|
||||
}}
|
||||
>
|
||||
Remove
|
||||
</Button>
|
||||
</div>
|
||||
<Button variant="destructive" disabled={removed} onClick={onRemove}>
|
||||
{removed ? 'Removed' : 'Remove…'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function RemovedList({ entries }: { entries: { url: string; name: string; reason: string; deleted_at: string }[] }) {
|
||||
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
|
||||
const [query, setQuery] = useState('')
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
const [busy, setBusy] = useState<string | null>(null)
|
||||
const client = useQueryClient()
|
||||
const restore = useMutation({
|
||||
mutationFn: (url: string) => adminApi.restore(url),
|
||||
onSuccess: (r) => {
|
||||
setNotice({
|
||||
tone: 'info',
|
||||
text: r.queued ? `Queued for re-ingest (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Already queued.',
|
||||
})
|
||||
const track = useTracking((s) => s.track)
|
||||
const shown = useMemo(
|
||||
() => entries.filter((d) => !query.trim() || matches(query, d.name, d.url)).slice(-MAX_RESULTS).reverse(),
|
||||
[entries, query],
|
||||
)
|
||||
|
||||
const restore = async (d: { url: string; name: string }) => {
|
||||
const ok = await confirm({
|
||||
title: `Bring “${d.name}” back?`,
|
||||
body: 'It is re-checked by the ingest workflow; if it is still paid or gone, it stays on the removed list.',
|
||||
action: 'Restore',
|
||||
})
|
||||
if (!ok) return
|
||||
setBusy(d.url)
|
||||
try {
|
||||
const r = await adminApi.restore(d.url)
|
||||
if (r.queued) track(`Restore: ${d.name}`, r.at, [d.url])
|
||||
setNotice({ tone: 'info', text: r.queued ? `Queued for re-ingest (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Already queued.' })
|
||||
void client.invalidateQueries({ queryKey: ['queue'] })
|
||||
},
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
} catch (e) {
|
||||
setNotice({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setBusy(null)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mt-3">
|
||||
{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}
|
||||
<div className="max-w-3xl">
|
||||
<Input
|
||||
type="search"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
placeholder="Search removed games by name or URL"
|
||||
aria-label="Search removed games"
|
||||
/>
|
||||
<p className="mt-2 text-xs text-muted-foreground">The {MAX_RESULTS} most recent removals that match.</p>
|
||||
<div className="mt-3">{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}</div>
|
||||
<ul className="divide-y text-sm">
|
||||
{entries.map((d) => (
|
||||
{shown.map((d) => (
|
||||
<li key={d.url} className="flex items-start gap-3 py-2">
|
||||
<div className="min-w-0 flex-1">
|
||||
<span className="font-medium">{d.name}</span>
|
||||
@@ -237,17 +305,8 @@ function RemovedList({ entries }: { entries: { url: string; name: string; reason
|
||||
{d.deleted_at.slice(0, 10)} · {d.reason}
|
||||
</span>
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
disabled={restore.isPending}
|
||||
onClick={() => {
|
||||
if (window.confirm(`Bring "${d.name}" back? It is re-scraped; if it is still paid or gone, it stays on the removed list.`)) {
|
||||
restore.mutate(d.url)
|
||||
}
|
||||
}}
|
||||
>
|
||||
Restore
|
||||
<Button size="sm" variant="outline" disabled={busy !== null} onClick={() => void restore(d)}>
|
||||
{busy === d.url ? 'Restoring…' : 'Restore'}
|
||||
</Button>
|
||||
</li>
|
||||
))}
|
||||
@@ -255,3 +314,58 @@ function RemovedList({ entries }: { entries: { url: string; name: string; reason
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** Paste up to 100 game URLs (any text containing them), see which are in the catalog, remove them in one commit. */
|
||||
function RemoveByList({ games, deleted }: { games: Game[]; deleted: { url: string }[] }) {
|
||||
const [text, setText] = useState('')
|
||||
const [open, setOpen] = useState<Picked[] | null>(null)
|
||||
const { removed } = useOverrides()
|
||||
const byUrl = useMemo(() => new Map(games.map((g) => [g.url, g])), [games])
|
||||
const gone = useMemo(() => new Set(deleted.map((d) => d.url)), [deleted])
|
||||
const { urls } = useMemo(() => extractGameUrls(text), [text])
|
||||
|
||||
const inCatalog = urls.filter((u) => byUrl.has(u) && !removed.has(u))
|
||||
const alreadyRemoved = urls.filter((u) => gone.has(u) || removed.has(u))
|
||||
const unknown = urls.filter((u) => !byUrl.has(u) && !gone.has(u) && !removed.has(u))
|
||||
const picked: Picked[] = inCatalog.slice(0, MAX_BULK).map((u) => ({ url: u, name: byUrl.get(u)?.name ?? u }))
|
||||
|
||||
return (
|
||||
<div className="max-w-3xl text-sm">
|
||||
<textarea
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
rows={8}
|
||||
spellCheck={false}
|
||||
placeholder="Game URLs to remove (any text containing them), up to 100 per removal"
|
||||
aria-label="Game URLs to remove"
|
||||
className="w-full rounded-md border bg-background p-2 font-mono text-sm"
|
||||
/>
|
||||
{urls.length > 0 && (
|
||||
<div className="mt-2 space-y-1">
|
||||
<p>
|
||||
<span className="font-medium">{plural(inCatalog.length, 'game')} in the catalog</span>
|
||||
{alreadyRemoved.length > 0 && <span className="text-muted-foreground"> · {alreadyRemoved.length} already removed</span>}
|
||||
{unknown.length > 0 && <span className="text-muted-foreground"> · {unknown.length} not in the catalog</span>}
|
||||
</p>
|
||||
{inCatalog.length > MAX_BULK && (
|
||||
<p className="text-destructive">
|
||||
One removal takes {MAX_BULK} games: the first {MAX_BULK} go now, paste the rest afterwards.
|
||||
</p>
|
||||
)}
|
||||
<ul className="max-h-60 overflow-y-auto rounded-md border px-3 py-2 text-xs">
|
||||
{picked.map((g) => (
|
||||
<li key={g.url} className="truncate">
|
||||
<span className="font-medium">{g.name}</span> <span className="text-muted-foreground">{g.url.replace('https://', '')}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
<Button className="mt-3" variant="destructive" disabled={picked.length === 0} onClick={() => setOpen(picked)}>
|
||||
Remove {plural(picked.length, 'game')}…
|
||||
</Button>
|
||||
{/* Removed games drop out of the list by themselves (this session remembers them). */}
|
||||
{open && <RemoveDialog games={open} onClose={() => setOpen(null)} onRemoved={() => {}} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { CheckResult } from './api'
|
||||
import { groupOf, groupResults } from './check-groups'
|
||||
|
||||
const r = (outcome: string, url: string | null, status: CheckResult['status'] = null): CheckResult => ({
|
||||
input: url ?? 'x',
|
||||
url,
|
||||
outcome,
|
||||
status,
|
||||
})
|
||||
|
||||
describe('check groups', () => {
|
||||
it('says what Add would do with each URL', () => {
|
||||
expect(groupOf(r('queued', 'u'))).toBe('new')
|
||||
expect(groupOf(r('duplicate_pending', 'u', 'pending'))).toBe('waiting')
|
||||
expect(groupOf(r('duplicate_pending', 'u', 'failed'))).toBe('waiting')
|
||||
expect(groupOf(r('duplicate_pending', 'u', 'queued'))).toBe('queued')
|
||||
expect(groupOf(r('duplicate_catalog', 'u'))).toBe('catalog')
|
||||
expect(groupOf(r('previously_deleted', 'u'))).toBe('removed')
|
||||
expect(groupOf(r('rejected', 'u'))).toBe('rejected')
|
||||
expect(groupOf(r('invalid_url', null))).toBe('invalid')
|
||||
})
|
||||
|
||||
it('groups results in input order and drops repeats', () => {
|
||||
const groups = groupResults([r('queued', 'a'), r('duplicate_in_request', 'a'), r('queued', 'b'), r('invalid_url', null)])
|
||||
expect(groups.new).toEqual(['a', 'b'])
|
||||
expect(groups.invalid).toEqual(['x'])
|
||||
expect(groups.catalog).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,42 @@
|
||||
/** The Add page's preview: each checked URL in the group that says what Add would do with it. */
|
||||
import type { CheckResult } from './api'
|
||||
|
||||
export type CheckGroup = 'new' | 'waiting' | 'queued' | 'catalog' | 'removed' | 'rejected' | 'invalid'
|
||||
|
||||
export const GROUPS: { id: CheckGroup; label: string; hint: string }[] = [
|
||||
{ id: 'new', label: 'New', hint: 'Added and queued for the ingest workflow.' },
|
||||
{ id: 'waiting', label: 'Waiting in the queue', hint: 'Pending or failed before: approved too.' },
|
||||
{ id: 'queued', label: 'Already queued', hint: 'Queued or just ingested: nothing to do.' },
|
||||
{ id: 'catalog', label: 'In the catalog', hint: 'Nothing to do.' },
|
||||
{ id: 'removed', label: 'Removed before', hint: 'Added only with override (it is re-checked; a still-paid game stays removed).' },
|
||||
{ id: 'rejected', label: 'Rejected before', hint: 'Added only with override.' },
|
||||
{ id: 'invalid', label: 'Not a game URL', hint: 'Skipped.' },
|
||||
]
|
||||
|
||||
export function groupOf(r: CheckResult): CheckGroup {
|
||||
switch (r.outcome) {
|
||||
case 'queued':
|
||||
return 'new'
|
||||
case 'duplicate_pending':
|
||||
return r.status === 'pending' || r.status === 'failed' ? 'waiting' : 'queued'
|
||||
case 'duplicate_catalog':
|
||||
case 'duplicate_in_request':
|
||||
return 'catalog'
|
||||
case 'previously_deleted':
|
||||
return 'removed'
|
||||
case 'rejected':
|
||||
return 'rejected'
|
||||
default:
|
||||
return 'invalid'
|
||||
}
|
||||
}
|
||||
|
||||
/** URLs per group, in input order; a URL listed twice counts once. */
|
||||
export function groupResults(results: CheckResult[]): Record<CheckGroup, string[]> {
|
||||
const groups = Object.fromEntries(GROUPS.map((g) => [g.id, [] as string[]])) as Record<CheckGroup, string[]>
|
||||
for (const r of results) {
|
||||
if (r.outcome === 'duplicate_in_request') continue
|
||||
groups[groupOf(r)].push(r.url ?? r.input)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* In-app confirmation (the browser's confirm() is blocking and unstyled; it
|
||||
* also does nothing in some webviews). `confirm()` resolves when the
|
||||
* maintainer answers the dialog that ConfirmDialog (admin/ui.tsx) shows.
|
||||
*/
|
||||
import { create } from 'zustand'
|
||||
|
||||
export interface ConfirmRequest {
|
||||
title: string
|
||||
body?: string
|
||||
action: string
|
||||
destructive?: boolean
|
||||
}
|
||||
|
||||
interface ConfirmState {
|
||||
request: (ConfirmRequest & { resolve: (ok: boolean) => void }) | null
|
||||
}
|
||||
|
||||
export const useConfirm = create<ConfirmState>(() => ({ request: null }))
|
||||
|
||||
export function confirm(request: ConfirmRequest): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
useConfirm.getState().request?.resolve(false)
|
||||
useConfirm.setState({ request: { ...request, resolve } })
|
||||
})
|
||||
}
|
||||
|
||||
export function answer(ok: boolean): void {
|
||||
const pending = useConfirm.getState().request
|
||||
useConfirm.setState({ request: null })
|
||||
pending?.resolve(ok)
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
/** The bulk actions on selected catalog games: remove (with a public reason) and edit. */
|
||||
import { useState } from 'react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Dialog, DialogContent, DialogDescription, DialogTitle } from '@/components/ui/dialog'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { REMOVAL_REASONS } from '@/lib/deleted-reason'
|
||||
import { MAX_REASON } from '../worker/limits'
|
||||
import { adminApi, shortSha, type GameEdit } from './api'
|
||||
import { rememberEdits, rememberRemoved } from './overrides'
|
||||
import { PUBLISH_NOTE, WARNING_TEXT, errorText, plural } from './texts'
|
||||
import { Notice, type NoticeState } from './ui'
|
||||
|
||||
export interface Picked {
|
||||
url: string
|
||||
name: string
|
||||
}
|
||||
|
||||
function GameList({ games }: { games: Picked[] }) {
|
||||
return (
|
||||
<ul className="max-h-40 overflow-y-auto rounded-md border px-3 py-2 text-xs">
|
||||
{games.map((g) => (
|
||||
<li key={g.url} className="truncate">
|
||||
<span className="font-medium">{g.name}</span> <span className="text-muted-foreground">{g.url.replace('https://', '')}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)
|
||||
}
|
||||
|
||||
const OTHER = 'other'
|
||||
|
||||
/**
|
||||
* Remove up to 100 games in one commit. The reason is public (the Removed page
|
||||
* shows it, translated for the standard ones) and the games stay blocked from
|
||||
* re-ingest until restored.
|
||||
*/
|
||||
export function RemoveDialog({
|
||||
games,
|
||||
onClose,
|
||||
onRemoved,
|
||||
}: {
|
||||
/** The games as they were when the dialog opened (mount it with a fresh list each time). */
|
||||
games: Picked[]
|
||||
onClose: () => void
|
||||
onRemoved: (urls: string[]) => void
|
||||
}) {
|
||||
const [preset, setPreset] = useState<string>(REMOVAL_REASONS[0])
|
||||
const [custom, setCustom] = useState('')
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
const reason = (preset === OTHER ? custom : preset).trim()
|
||||
|
||||
const remove = async () => {
|
||||
setBusy(true)
|
||||
setNotice(null)
|
||||
try {
|
||||
const r = await adminApi.removeMany(
|
||||
games.map((g) => g.url),
|
||||
reason,
|
||||
)
|
||||
rememberRemoved(r.removed)
|
||||
onRemoved(r.removed)
|
||||
const lines = [
|
||||
r.removed.length > 0
|
||||
? `Removed ${plural(r.removed.length, 'game')} (${shortSha(r.sha)}). ${PUBLISH_NOTE}`
|
||||
: 'Nothing was removed.',
|
||||
r.notFound.length > 0 ? `Not in the catalog any more: ${plural(r.notFound.length, 'game')}.` : '',
|
||||
...r.warnings.map((w) => `Note: ${WARNING_TEXT[w] ?? w}.`),
|
||||
]
|
||||
setNotice({ tone: 'info', text: lines.filter(Boolean).join('\n') })
|
||||
} catch (e) {
|
||||
setNotice({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(next) => !next && !busy && onClose()}>
|
||||
<DialogContent>
|
||||
<div className="space-y-1 pr-6">
|
||||
<DialogTitle className="text-lg font-semibold tracking-tight">
|
||||
Remove {plural(games.length, 'game')} from the catalog
|
||||
</DialogTitle>
|
||||
<DialogDescription className="text-sm text-muted-foreground">
|
||||
One commit. The reason is public (Removed page); the games stay blocked from every source until restored.
|
||||
</DialogDescription>
|
||||
</div>
|
||||
<GameList games={games} />
|
||||
<label className="grid gap-1 text-sm">
|
||||
<span className="text-xs text-muted-foreground">Reason</span>
|
||||
<select value={preset} onChange={(e) => setPreset(e.target.value)} className="h-9 rounded-md border bg-background px-2">
|
||||
{REMOVAL_REASONS.map((r) => (
|
||||
<option key={r} value={r}>
|
||||
{r}
|
||||
</option>
|
||||
))}
|
||||
<option value={OTHER}>Other…</option>
|
||||
</select>
|
||||
</label>
|
||||
{preset === OTHER && (
|
||||
<Input
|
||||
value={custom}
|
||||
maxLength={MAX_REASON}
|
||||
onChange={(e) => setCustom(e.target.value)}
|
||||
placeholder="Reason (public)"
|
||||
aria-label="Removal reason"
|
||||
autoFocus
|
||||
/>
|
||||
)}
|
||||
{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}
|
||||
<div className="flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
|
||||
<Button variant="ghost" disabled={busy} onClick={onClose}>
|
||||
{notice?.tone === 'info' ? 'Close' : 'Cancel'}
|
||||
</Button>
|
||||
{notice?.tone !== 'info' && (
|
||||
<Button variant="destructive" disabled={busy || !reason || games.length === 0} onClick={() => void remove()}>
|
||||
{busy ? 'Removing…' : `Remove ${plural(games.length, 'game')}`}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
const KEEP = 'keep'
|
||||
|
||||
/** Set the 18+ flag, the virus check and/or the notes of up to 100 games in one commit. */
|
||||
export function BulkEditDialog({ games, onClose }: { games: Picked[]; onClose: () => void }) {
|
||||
const [nsfw, setNsfw] = useState(KEEP)
|
||||
const [virus, setVirus] = useState(KEEP)
|
||||
const [notesMode, setNotesMode] = useState<'keep' | 'replace' | 'clear'>('keep')
|
||||
const [notes, setNotes] = useState('')
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
|
||||
const edit: GameEdit = {
|
||||
...(nsfw !== KEEP ? { nsfw } : {}),
|
||||
...(virus !== KEEP ? { safe_virus: virus } : {}),
|
||||
...(notesMode === 'replace' ? { notes } : notesMode === 'clear' ? { notes: '' } : {}),
|
||||
}
|
||||
const empty = Object.keys(edit).length === 0
|
||||
|
||||
const save = async () => {
|
||||
setBusy(true)
|
||||
setNotice(null)
|
||||
try {
|
||||
const edits = games.map((g) => ({ url: g.url, edit }))
|
||||
const r = await adminApi.editMany(edits)
|
||||
rememberEdits(edits.filter((e) => !r.notFound.includes(e.url)))
|
||||
const lines = [
|
||||
r.edited.length > 0 ? `Saved ${plural(r.edited.length, 'game')} (${shortSha(r.sha)}). ${PUBLISH_NOTE}` : 'Nothing changed.',
|
||||
r.unchanged.length > 0 ? `Already so: ${plural(r.unchanged.length, 'game')}.` : '',
|
||||
r.notFound.length > 0 ? `Not in the catalog any more: ${plural(r.notFound.length, 'game')}.` : '',
|
||||
]
|
||||
setNotice({ tone: 'info', text: lines.filter(Boolean).join('\n') })
|
||||
} catch (e) {
|
||||
setNotice({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
const select = (label: string, value: string, onChange: (v: string) => void, options: [string, string][]) => (
|
||||
<label className="grid gap-1 text-sm">
|
||||
<span className="text-xs text-muted-foreground">{label}</span>
|
||||
<select value={value} onChange={(e) => onChange(e.target.value)} className="h-9 rounded-md border bg-background px-2">
|
||||
<option value={KEEP}>Keep as it is</option>
|
||||
{options.map(([v, l]) => (
|
||||
<option key={v} value={v}>
|
||||
{l}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
)
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(next) => !next && !busy && onClose()}>
|
||||
<DialogContent>
|
||||
<div className="space-y-1 pr-6">
|
||||
<DialogTitle className="text-lg font-semibold tracking-tight">Edit {plural(games.length, 'game')}</DialogTitle>
|
||||
<DialogDescription className="text-sm text-muted-foreground">
|
||||
One commit. Only the fields you change are written.
|
||||
</DialogDescription>
|
||||
</div>
|
||||
<GameList games={games} />
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
{select('NSFW (18+)', nsfw, setNsfw, [
|
||||
['No', 'No'],
|
||||
['Yes', 'Yes'],
|
||||
])}
|
||||
{select('Virus check', virus, setVirus, [
|
||||
['?', 'Not checked (?)'],
|
||||
['Yes', 'Yes'],
|
||||
['No', 'No'],
|
||||
['Caution', 'Caution'],
|
||||
])}
|
||||
</div>
|
||||
<label className="grid gap-1 text-sm">
|
||||
<span className="text-xs text-muted-foreground">Notes (public)</span>
|
||||
<select
|
||||
value={notesMode}
|
||||
onChange={(e) => setNotesMode(e.target.value as typeof notesMode)}
|
||||
className="h-9 rounded-md border bg-background px-2"
|
||||
>
|
||||
<option value="keep">Keep as they are</option>
|
||||
<option value="replace">Replace with…</option>
|
||||
<option value="clear">Clear</option>
|
||||
</select>
|
||||
</label>
|
||||
{notesMode === 'replace' && (
|
||||
<textarea
|
||||
value={notes}
|
||||
maxLength={2000}
|
||||
rows={3}
|
||||
onChange={(e) => setNotes(e.target.value)}
|
||||
aria-label="New notes"
|
||||
className="rounded-md border bg-background p-2 text-sm"
|
||||
/>
|
||||
)}
|
||||
{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}
|
||||
<div className="flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
|
||||
<Button variant="ghost" disabled={busy} onClick={onClose}>
|
||||
{notice?.tone === 'info' ? 'Close' : 'Cancel'}
|
||||
</Button>
|
||||
{notice?.tone !== 'info' && (
|
||||
<Button disabled={busy || empty || games.length === 0} onClick={() => void save()}>
|
||||
{busy ? 'Saving…' : `Save ${plural(games.length, 'game')}`}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { canonicalize } from '../worker/canonical'
|
||||
import { extractGameUrls } from './extract'
|
||||
|
||||
describe('extractGameUrls', () => {
|
||||
it('finds game URLs in a plain list, in order, once each', () => {
|
||||
const { urls, duplicates } = extractGameUrls(
|
||||
'https://Alpha.itch.io/First-Light/\nbeta.itch.io/rock\nhttps://alpha.itch.io/first-light?ref=x\n',
|
||||
)
|
||||
expect(urls).toEqual(['https://alpha.itch.io/first-light', 'https://beta.itch.io/rock'])
|
||||
expect(duplicates).toBe(1)
|
||||
})
|
||||
|
||||
it('reads Markdown, CSV, HTML and JSON', () => {
|
||||
const text = [
|
||||
'- [Rock](https://beta.itch.io/rock) and <https://gamma.itch.io/tower>',
|
||||
'name,url\n"Delta",https://delta.itch.io/maze',
|
||||
'<a HREF="https://epsilon.itch.io/park/devlog/12">devlog</a>',
|
||||
JSON.stringify({ games: ['https://zeta.itch.io/walk'] }).replaceAll('/', '\\/'), // PHP-style escaped slashes
|
||||
].join('\n')
|
||||
expect(extractGameUrls(text).urls).toEqual([
|
||||
'https://beta.itch.io/rock',
|
||||
'https://gamma.itch.io/tower',
|
||||
'https://delta.itch.io/maze',
|
||||
'https://epsilon.itch.io/park',
|
||||
'https://zeta.itch.io/walk',
|
||||
])
|
||||
})
|
||||
|
||||
it('skips itch.io pages that are not games and look-alike hosts', () => {
|
||||
const text = 'https://itch.io/games/free https://www.itch.io/x https://static.itch.io/y https://creator.itch.io https://evil.com.itch.io.example/x https://img.itch.zone/abc'
|
||||
expect(extractGameUrls(text).urls).toEqual([])
|
||||
})
|
||||
|
||||
it('returns canonical URLs only (the same form the Worker stores)', () => {
|
||||
const { urls } = extractGameUrls('HTTP://My-Dev.ITCH.io/Game_2 my-dev.itch.io/7days')
|
||||
for (const url of urls) expect(canonicalize(url)).toBe(url)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* itch.io game URLs out of any pasted or uploaded text: a plain list, CSV,
|
||||
* Markdown, an HTML page or JSON (escaped "\/" slashes included). A link
|
||||
* deeper into a game page (devlog, purchase…) counts as the game. The result
|
||||
* is canonical (https://creator.itch.io/game, lower case), de-duplicated, in
|
||||
* the order the games first appear.
|
||||
*/
|
||||
|
||||
// The creator's subdomain, then the game's slug as the first path segment.
|
||||
const GAME_URL = /(?:^|[^a-z0-9.-])(?:https?:\/\/)?([a-z0-9][a-z0-9-]*)\.itch\.io\/([a-z0-9][a-z0-9_-]*)/gi
|
||||
// itch.io's own subdomains, never a creator.
|
||||
const NOT_CREATORS = new Set(['www', 'static', 'api'])
|
||||
|
||||
export interface Extracted {
|
||||
urls: string[]
|
||||
/** Further mentions of a game already listed. */
|
||||
duplicates: number
|
||||
}
|
||||
|
||||
export function extractGameUrls(text: string): Extracted {
|
||||
const seen = new Set<string>()
|
||||
let duplicates = 0
|
||||
for (const m of text.replace(/\\\//g, '/').matchAll(GAME_URL)) {
|
||||
const creator = m[1].toLowerCase()
|
||||
if (NOT_CREATORS.has(creator)) continue
|
||||
const url = `https://${creator}.itch.io/${m[2].toLowerCase()}`
|
||||
if (seen.has(url)) duplicates += 1
|
||||
else seen.add(url)
|
||||
}
|
||||
return { urls: [...seen], duplicates }
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<!-- The theme the site saved (same script and CSP hash as index.html). -->
|
||||
<script>try{var t=JSON.parse(localStorage.getItem('webapp.theme')||'null');t=t&&t.state&&t.state.theme;if(t==='dark'||t!=='light'&&matchMedia('(prefers-color-scheme: dark)').matches)document.documentElement.classList.add('dark')}catch(e){}</script>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<title>Admin · freeitchgames.win</title>
|
||||
</head>
|
||||
|
||||
@@ -8,12 +8,6 @@ import '@/index.css'
|
||||
import { AdminApp } from './app'
|
||||
import { SessionExpired, markSessionExpired } from './api'
|
||||
|
||||
// Follow the system theme (the admin has no settings of its own).
|
||||
const dark = window.matchMedia('(prefers-color-scheme: dark)')
|
||||
const applyTheme = () => document.documentElement.classList.toggle('dark', dark.matches)
|
||||
applyTheme()
|
||||
dark.addEventListener('change', applyTheme)
|
||||
|
||||
const onError = (error: unknown) => {
|
||||
if (error instanceof SessionExpired) markSessionExpired()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { useAllGames } from '@/hooks/useGames'
|
||||
import { cn } from '@/lib/utils'
|
||||
import type { Game } from '@/types/game'
|
||||
import { MAX_BULK } from '../worker/limits'
|
||||
import { BulkEditDialog, RemoveDialog, type Picked } from './dialogs'
|
||||
import { useOverrides, withEdits } from './overrides'
|
||||
import { useSelection } from './selection'
|
||||
import { plural } from './texts'
|
||||
import { Cover, SelectionBar, TabBar } from './ui'
|
||||
|
||||
const PAGE = 100
|
||||
const DAY_MS = 86_400_000
|
||||
|
||||
type Days = '7' | '30'
|
||||
type Choice = 'any' | string
|
||||
|
||||
/** Games the ingest added lately, newest first, to review their 18+ flag and virus check in bulk. */
|
||||
export function NewGamesPage() {
|
||||
const games = useAllGames()
|
||||
const [days, setDays] = useState<Days>('7')
|
||||
const [nsfw, setNsfw] = useState<Choice>('any')
|
||||
const [virus, setVirus] = useState<Choice>('any')
|
||||
const [query, setQuery] = useState('')
|
||||
const [page, setPage] = useState(0)
|
||||
const [dialog, setDialog] = useState<{ kind: 'edit' | 'remove'; games: Picked[] } | null>(null)
|
||||
const selection = useSelection()
|
||||
const { edits, removed } = useOverrides()
|
||||
|
||||
// "Now" is when the catalog was loaded (rendering must not read the clock).
|
||||
const loadedAt = games.dataUpdatedAt
|
||||
const recent = useMemo(() => {
|
||||
if (!games.data || !loadedAt) return []
|
||||
const cutoff = new Date(loadedAt - Number(days) * DAY_MS).toISOString()
|
||||
return games.data.games
|
||||
.filter((g) => (g.added_at ?? '') >= cutoff)
|
||||
.sort((a, b) => (b.added_at ?? '').localeCompare(a.added_at ?? '') || a.name.localeCompare(b.name))
|
||||
}, [games.data, loadedAt, days])
|
||||
const shown = useMemo(() => {
|
||||
const q = query.trim().toLowerCase()
|
||||
return recent
|
||||
.map((g) => withEdits(g, edits))
|
||||
.filter(
|
||||
(g) =>
|
||||
(nsfw === 'any' || g.nsfw === nsfw) &&
|
||||
(virus === 'any' || g.safe_virus === virus) &&
|
||||
(!q || g.name.toLowerCase().includes(q) || g.dev.toLowerCase().includes(q) || g.url.includes(q)),
|
||||
)
|
||||
}, [recent, edits, nsfw, virus, query])
|
||||
const pages = Math.max(1, Math.ceil(shown.length / PAGE))
|
||||
const current = Math.min(page, pages - 1)
|
||||
const rows = shown.slice(current * PAGE, (current + 1) * PAGE)
|
||||
const byUrl = new Map(recent.map((g) => [g.url, g]))
|
||||
const picked: Picked[] = [...selection.selected].map((url) => ({ url, name: byUrl.get(url)?.name ?? url }))
|
||||
const selectable = rows.filter((g) => !removed.has(g.url))
|
||||
|
||||
const filter = (label: string, value: Choice, onChange: (v: Choice) => void, options: string[]) => (
|
||||
<label className="flex items-center gap-2">
|
||||
<span className="text-muted-foreground">{label}</span>
|
||||
<select
|
||||
value={value}
|
||||
onChange={(e) => {
|
||||
onChange(e.target.value)
|
||||
setPage(0)
|
||||
}}
|
||||
className="h-8 rounded-md border bg-background px-2"
|
||||
>
|
||||
<option value="any">any</option>
|
||||
{options.map((o) => (
|
||||
<option key={o} value={o}>
|
||||
{o}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
)
|
||||
|
||||
return (
|
||||
<section>
|
||||
<h1 className="mb-1 text-lg font-semibold">New games</h1>
|
||||
<p className="mb-3 text-sm text-muted-foreground">
|
||||
Added by the ingest workflow lately (published data). Check the 18+ flag and the virus check, in bulk.
|
||||
</p>
|
||||
<TabBar
|
||||
label="Added within"
|
||||
value={days}
|
||||
onChange={(d) => {
|
||||
setDays(d)
|
||||
setPage(0)
|
||||
}}
|
||||
options={[
|
||||
{ id: '7', label: 'Last 7 days' },
|
||||
{ id: '30', label: 'Last 30 days' },
|
||||
]}
|
||||
/>
|
||||
<div className="mb-3 flex flex-wrap items-center gap-4 text-sm">
|
||||
<Input
|
||||
type="search"
|
||||
value={query}
|
||||
onChange={(e) => {
|
||||
setQuery(e.target.value)
|
||||
setPage(0)
|
||||
}}
|
||||
placeholder="Filter by name, developer or URL"
|
||||
aria-label="Filter new games"
|
||||
className="max-w-xs"
|
||||
/>
|
||||
{filter('18+', nsfw, setNsfw, ['No', 'Yes'])}
|
||||
{filter('Virus check', virus, setVirus, ['?', 'Yes', 'No', 'Caution'])}
|
||||
<span className="text-muted-foreground">
|
||||
{games.isPending ? 'Loading…' : `${plural(shown.length, 'game')} of ${recent.length.toLocaleString('en-GB')}`}
|
||||
</span>
|
||||
<button
|
||||
className="text-muted-foreground underline-offset-4 hover:underline disabled:opacity-50"
|
||||
disabled={selectable.length === 0 || selection.full}
|
||||
onClick={() => selection.setMany(selectable.map((g) => g.url), true)}
|
||||
>
|
||||
Select this page
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<SelectionBar count={selection.selected.size} max={MAX_BULK} onClear={selection.clear}>
|
||||
<Button size="sm" variant="outline" onClick={() => setDialog({ kind: 'edit', games: picked })}>
|
||||
Edit…
|
||||
</Button>
|
||||
<Button size="sm" variant="destructive" onClick={() => setDialog({ kind: 'remove', games: picked })}>
|
||||
Remove…
|
||||
</Button>
|
||||
</SelectionBar>
|
||||
|
||||
<ul className="divide-y text-sm">
|
||||
{rows.map((g) => (
|
||||
<NewGameRow
|
||||
key={g.url}
|
||||
game={g}
|
||||
removed={removed.has(g.url)}
|
||||
selected={selection.selected.has(g.url)}
|
||||
disabled={!selection.selected.has(g.url) && selection.full}
|
||||
onToggle={() => selection.toggle(g.url)}
|
||||
/>
|
||||
))}
|
||||
{!games.isPending && rows.length === 0 && <li className="py-3 text-muted-foreground">No game added in that period matches.</li>}
|
||||
</ul>
|
||||
|
||||
{pages > 1 && (
|
||||
<div className="mt-3 flex items-center gap-3 text-sm">
|
||||
<Button size="sm" variant="outline" disabled={current === 0} onClick={() => setPage(current - 1)}>
|
||||
Previous
|
||||
</Button>
|
||||
<span className="tabular-nums text-muted-foreground">
|
||||
Page {current + 1} of {pages}
|
||||
</span>
|
||||
<Button size="sm" variant="outline" disabled={current >= pages - 1} onClick={() => setPage(current + 1)}>
|
||||
Next
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{dialog?.kind === 'edit' && <BulkEditDialog games={dialog.games} onClose={() => setDialog(null)} />}
|
||||
{dialog?.kind === 'remove' && (
|
||||
<RemoveDialog games={dialog.games} onClose={() => setDialog(null)} onRemoved={(urls) => selection.setMany(urls, false)} />
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function NewGameRow({
|
||||
game,
|
||||
removed,
|
||||
selected,
|
||||
disabled,
|
||||
onToggle,
|
||||
}: {
|
||||
game: Game
|
||||
removed: boolean
|
||||
selected: boolean
|
||||
disabled: boolean
|
||||
onToggle: () => void
|
||||
}) {
|
||||
return (
|
||||
<li className={cn('flex items-center gap-3 py-2', removed && 'opacity-50')}>
|
||||
<input type="checkbox" aria-label={`Select ${game.name}`} checked={selected} disabled={removed || disabled} onChange={onToggle} />
|
||||
<Cover src={game.thumbnail} className="h-12 w-16 shrink-0" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<a href={game.url} target="_blank" rel="noreferrer noopener" className="font-medium hover:underline">
|
||||
{game.name}
|
||||
</a>
|
||||
<span className="ml-2 text-xs text-muted-foreground">{game.dev}</span>
|
||||
{removed && <span className="ml-2 text-xs text-destructive">removed (publishing…)</span>}
|
||||
<div className="mt-1 flex flex-wrap gap-1 text-xs">
|
||||
<Badge variant={game.nsfw === 'Yes' ? 'destructive' : 'outline'}>18+: {game.nsfw}</Badge>
|
||||
<Badge variant={game.safe_virus === '?' ? 'secondary' : 'outline'}>virus check: {game.safe_virus}</Badge>
|
||||
{game.genre && game.genre !== 'N/A' && <Badge variant="outline">{game.genre}</Badge>}
|
||||
</div>
|
||||
</div>
|
||||
<span className="shrink-0 text-xs tabular-nums text-muted-foreground">{game.added_at?.slice(0, 10)}</span>
|
||||
</li>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* What this session changed in the catalog. The published data the Catalog
|
||||
* and New games pages read lags the repository until the next build, so
|
||||
* edits and removals made here are layered on top of it until a reload.
|
||||
*/
|
||||
import { create } from 'zustand'
|
||||
import type { Game } from '@/types/game'
|
||||
import type { GameEdit } from './api'
|
||||
|
||||
interface OverridesState {
|
||||
edits: Map<string, GameEdit>
|
||||
removed: Set<string>
|
||||
}
|
||||
|
||||
export const useOverrides = create<OverridesState>(() => ({ edits: new Map(), removed: new Set() }))
|
||||
|
||||
export function rememberEdits(edits: { url: string; edit: GameEdit }[]): void {
|
||||
useOverrides.setState((s) => {
|
||||
const next = new Map(s.edits)
|
||||
for (const { url, edit } of edits) next.set(url, { ...next.get(url), ...edit })
|
||||
return { edits: next }
|
||||
})
|
||||
}
|
||||
|
||||
export function rememberRemoved(urls: string[]): void {
|
||||
useOverrides.setState((s) => ({ removed: new Set([...s.removed, ...urls]) }))
|
||||
}
|
||||
|
||||
/** A game as the repository holds it now, as far as this session knows. */
|
||||
export function withEdits(game: Game, edits: Map<string, GameEdit>): Game {
|
||||
const edit = edits.get(game.url)
|
||||
return edit ? { ...game, ...edit } : game
|
||||
}
|
||||
+104
-59
@@ -1,10 +1,14 @@
|
||||
import { useState } from 'react'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { adminApi, errorText, shortSha, type Candidate, type CandidateStatus } from './api'
|
||||
import { Notice, PUBLISH_NOTE } from './app'
|
||||
import { MAX_URLS } from '../worker/limits'
|
||||
import { adminApi, shortSha, type Candidate, type CandidateStatus, type Decision } from './api'
|
||||
import { chunk, runBatches } from './batch'
|
||||
import { confirm } from './confirm'
|
||||
import { INGEST_TEXT, PUBLISH_NOTE, SKIP_TEXT, errorText, plural } from './texts'
|
||||
import { useTracking } from './tracking'
|
||||
import { BatchProgress, Notice, TabBar, type NoticeState } from './ui'
|
||||
|
||||
const STATUSES: { id: CandidateStatus; label: string }[] = [
|
||||
{ id: 'pending', label: 'Pending' },
|
||||
@@ -14,7 +18,7 @@ const STATUSES: { id: CandidateStatus; label: string }[] = [
|
||||
{ id: 'ingested', label: 'Ingested' },
|
||||
]
|
||||
|
||||
const SOURCE: Record<Candidate['source'], string> = { rss: 'RSS', form: 'Suggest form', ext: 'Extension', admin: 'Admin' }
|
||||
const SOURCE: Record<Candidate['source'], string> = { rss: 'RSS', form: 'Suggest form', admin: 'Admin' }
|
||||
|
||||
const FLAG: Record<string, { label: string; variant: 'destructive' | 'secondary' | 'outline' }> = {
|
||||
previously_deleted: { label: 'previously removed', variant: 'destructive' },
|
||||
@@ -22,41 +26,78 @@ const FLAG: Record<string, { label: string; variant: 'destructive' | 'secondary'
|
||||
demo: { label: 'demo', variant: 'outline' },
|
||||
}
|
||||
|
||||
function source(c: Candidate): string {
|
||||
const collection = /^collection:(\d+)$/.exec(c.submitter ?? '')
|
||||
if (collection) return `Collection #${collection[1]}`
|
||||
return SOURCE[c.source]
|
||||
}
|
||||
|
||||
export function QueuePage() {
|
||||
const [status, setStatus] = useState<CandidateStatus>('pending')
|
||||
const [selected, setSelected] = useState<Set<string>>(new Set())
|
||||
const [notice, setNotice] = useState<{ text: string; tone: 'info' | 'error' } | null>(null)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
const [progress, setProgress] = useState<{ label: string; done: number; total: number } | null>(null)
|
||||
const client = useQueryClient()
|
||||
const track = useTracking((s) => s.track)
|
||||
const queue = useQuery({ queryKey: ['queue', status], queryFn: () => adminApi.queue(status) })
|
||||
|
||||
const done = () => {
|
||||
const finish = () => {
|
||||
setSelected(new Set())
|
||||
setProgress(null)
|
||||
void client.invalidateQueries({ queryKey: ['queue'] })
|
||||
}
|
||||
const approve = useMutation({
|
||||
mutationFn: ({ urls, override }: { urls: string[]; override: boolean }) => adminApi.approve(urls, override),
|
||||
onSuccess: (d) => {
|
||||
const skipped = d.skipped.map((s) => ` ${s.url} — ${s.reason.replace(/_/g, ' ')}`).join('\n')
|
||||
setNotice({
|
||||
tone: 'info',
|
||||
text:
|
||||
`Queued ${d.approved.length} for ingest (${shortSha(d.sha)}). ${PUBLISH_NOTE}` +
|
||||
(skipped ? `\nSkipped:\n${skipped}` : ''),
|
||||
|
||||
/** Approve in batches of 200 (one commit each); stops at the first error. */
|
||||
const approve = async (override: boolean) => {
|
||||
const urls = [...selected]
|
||||
if (override) {
|
||||
const ok = await confirm({
|
||||
title: `Approve ${plural(urls.length, 'game')} with override?`,
|
||||
body: 'Previously removed games are let through the ingest again. A game that is still paid or gone stays on the removed list.',
|
||||
action: 'Approve with override',
|
||||
})
|
||||
done()
|
||||
},
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
const reject = useMutation({
|
||||
mutationFn: (urls: string[]) => adminApi.reject(urls),
|
||||
onSuccess: (d) => {
|
||||
setNotice({ tone: 'info', text: `Rejected ${d.rejected.length} (they stay blocked from every source).` })
|
||||
done()
|
||||
},
|
||||
onError: (e) => setNotice({ tone: 'error', text: errorText(e) }),
|
||||
})
|
||||
if (!ok) return
|
||||
}
|
||||
setNotice(null)
|
||||
const run = await runBatches(
|
||||
chunk(urls, MAX_URLS),
|
||||
(batch) => adminApi.approve(batch, override),
|
||||
(done, total) => setProgress({ label: 'Approving', done, total }),
|
||||
)
|
||||
const decisions: Decision[] = run.results
|
||||
const approved = decisions.flatMap((d) => d.approved)
|
||||
const skipped = decisions.flatMap((d) => d.skipped)
|
||||
if (approved.length > 0 && decisions[0]) track(`Queue: approved ${plural(approved.length, 'game')}`, decisions[0].at, approved)
|
||||
const lines = [
|
||||
`Queued ${plural(approved.length, 'game')} for ingest (${decisions.map((d) => shortSha(d.sha)).join(', ') || 'no commit'}). ${PUBLISH_NOTE}`,
|
||||
...(skipped.length > 0 ? ['Skipped:', ...skipped.map((s) => ` ${s.url} — ${SKIP_TEXT[s.reason] ?? s.reason}`)] : []),
|
||||
...(run.failed ? [`Stopped: ${errorText(run.failed.error)}`, `${plural(run.remaining.flat().length, 'game')} not sent — select them again to retry.`] : []),
|
||||
]
|
||||
setNotice({ tone: run.failed ? 'error' : 'info', text: lines.join('\n') })
|
||||
finish()
|
||||
}
|
||||
|
||||
const reject = async () => {
|
||||
const urls = [...selected]
|
||||
setNotice(null)
|
||||
const run = await runBatches(
|
||||
chunk(urls, MAX_URLS),
|
||||
(batch) => adminApi.reject(batch),
|
||||
(done, total) => setProgress({ label: 'Rejecting', done, total }),
|
||||
)
|
||||
const rejected = run.results.flatMap((r) => r.rejected)
|
||||
setNotice({
|
||||
tone: run.failed ? 'error' : 'info',
|
||||
text: [
|
||||
`Rejected ${plural(rejected.length, 'game')} (they stay blocked from every source).`,
|
||||
...(run.failed ? [`Stopped: ${errorText(run.failed.error)}`] : []),
|
||||
].join('\n'),
|
||||
})
|
||||
finish()
|
||||
}
|
||||
|
||||
const items = queue.data?.items ?? []
|
||||
const total = queue.data?.counts[status] ?? 0
|
||||
const allSelected = items.length > 0 && items.every((c) => selected.has(c.url))
|
||||
const toggle = (url: string) =>
|
||||
setSelected((prev) => {
|
||||
@@ -65,39 +106,36 @@ export function QueuePage() {
|
||||
else next.add(url)
|
||||
return next
|
||||
})
|
||||
const urls = [...selected]
|
||||
const busy = approve.isPending || reject.isPending
|
||||
const busy = progress !== null
|
||||
const canApprove = status !== 'queued' && status !== 'ingested'
|
||||
|
||||
return (
|
||||
<section>
|
||||
<div className="mb-4 flex flex-wrap items-center gap-1" role="tablist" aria-label="Queue status">
|
||||
{STATUSES.map((s) => (
|
||||
<button
|
||||
key={s.id}
|
||||
role="tab"
|
||||
aria-selected={status === s.id}
|
||||
onClick={() => {
|
||||
setStatus(s.id)
|
||||
setSelected(new Set())
|
||||
}}
|
||||
className={cn(
|
||||
'rounded-md px-3 py-1.5 text-sm hover:bg-accent',
|
||||
status === s.id && 'bg-accent font-medium',
|
||||
)}
|
||||
>
|
||||
{s.label}
|
||||
<span className="ml-1.5 tabular-nums text-muted-foreground">{queue.data?.counts[s.id] ?? ''}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<TabBar
|
||||
label="Queue status"
|
||||
value={status}
|
||||
onChange={(s) => {
|
||||
setStatus(s)
|
||||
setSelected(new Set())
|
||||
}}
|
||||
options={STATUSES.map((s) => ({
|
||||
id: s.id,
|
||||
label: (
|
||||
<>
|
||||
{s.label}
|
||||
<span className="ml-1.5 tabular-nums text-muted-foreground">{queue.data?.counts[s.id] ?? ''}</span>
|
||||
</>
|
||||
),
|
||||
}))}
|
||||
/>
|
||||
|
||||
{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}
|
||||
{progress && <BatchProgress label={progress.label} done={progress.done} total={progress.total} />}
|
||||
|
||||
{canApprove && (
|
||||
<div className="mb-3 flex flex-wrap items-center gap-2 text-sm">
|
||||
<span className="text-muted-foreground">{selected.size} selected</span>
|
||||
<Button size="sm" disabled={!selected.size || busy} onClick={() => approve.mutate({ urls, override: false })}>
|
||||
<Button size="sm" disabled={!selected.size || busy} onClick={() => void approve(false)}>
|
||||
Approve
|
||||
</Button>
|
||||
<Button
|
||||
@@ -105,16 +143,12 @@ export function QueuePage() {
|
||||
variant="outline"
|
||||
disabled={!selected.size || busy}
|
||||
title="Also brings back games that were removed from the catalog before"
|
||||
onClick={() => {
|
||||
if (window.confirm(`Approve ${urls.length} game(s), including previously removed ones?`)) {
|
||||
approve.mutate({ urls, override: true })
|
||||
}
|
||||
}}
|
||||
onClick={() => void approve(true)}
|
||||
>
|
||||
Approve with override
|
||||
</Button>
|
||||
{status !== 'rejected' && (
|
||||
<Button size="sm" variant="destructive" disabled={!selected.size || busy} onClick={() => reject.mutate(urls)}>
|
||||
<Button size="sm" variant="destructive" disabled={!selected.size || busy} onClick={() => void reject()}>
|
||||
Reject
|
||||
</Button>
|
||||
)}
|
||||
@@ -124,6 +158,11 @@ export function QueuePage() {
|
||||
{queue.isPending && <p className="text-sm text-muted-foreground">Loading…</p>}
|
||||
{queue.error && <Notice text={errorText(queue.error)} tone="error" onClose={() => void queue.refetch()} />}
|
||||
{queue.data && items.length === 0 && <p className="text-sm text-muted-foreground">Nothing here.</p>}
|
||||
{items.length > 0 && total > items.length && (
|
||||
<p className="mb-2 text-xs text-muted-foreground">
|
||||
Showing the newest {items.length.toLocaleString('en-GB')} of {total.toLocaleString('en-GB')}.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{items.length > 0 && (
|
||||
<table className="w-full text-sm">
|
||||
@@ -177,12 +216,18 @@ export function QueuePage() {
|
||||
{FLAG[f]?.label ?? f}
|
||||
</Badge>
|
||||
))}
|
||||
{c.outcome && (
|
||||
<Badge variant={c.outcome === 'added' || c.outcome === 'duplicate' ? 'outline' : 'secondary'}>
|
||||
last ingest: {INGEST_TEXT[c.outcome] ?? c.outcome}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
{c.note && <p className="mt-1 whitespace-pre-line text-xs">“{c.note}”</p>}
|
||||
</td>
|
||||
<td className="py-2 text-xs text-muted-foreground">
|
||||
{SOURCE[c.source]}
|
||||
{c.submitter && c.source !== 'form' && c.source !== 'admin' && <div>{c.submitter}</div>}
|
||||
{source(c)}
|
||||
{c.submitter && c.source === 'rss' && <div>{c.submitter}</div>}
|
||||
{c.decided_by === 'auto' && <div>approved automatically</div>}
|
||||
</td>
|
||||
<td className="py-2 text-xs tabular-nums text-muted-foreground">{c.discovered_at.slice(0, 10)}</td>
|
||||
</tr>
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
/** A set of selected game URLs, capped at the bulk limit (one commit). */
|
||||
import { useCallback, useState } from 'react'
|
||||
import { MAX_BULK } from '../worker/limits'
|
||||
|
||||
export function useSelection(max = MAX_BULK) {
|
||||
const [selected, setSelected] = useState<ReadonlySet<string>>(new Set())
|
||||
const toggle = useCallback(
|
||||
(url: string) =>
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev)
|
||||
if (next.has(url)) next.delete(url)
|
||||
else if (next.size < max) next.add(url)
|
||||
return next
|
||||
}),
|
||||
[max],
|
||||
)
|
||||
/** Add (or, with `on` false, drop) several; adding stops at the cap. */
|
||||
const setMany = useCallback(
|
||||
(urls: string[], on: boolean) =>
|
||||
setSelected((prev) => {
|
||||
const next = new Set(prev)
|
||||
for (const url of urls) {
|
||||
if (!on) next.delete(url)
|
||||
else if (next.size < max) next.add(url)
|
||||
}
|
||||
return next
|
||||
}),
|
||||
[max],
|
||||
)
|
||||
const clear = useCallback(() => setSelected(new Set()), [])
|
||||
return { selected, toggle, setMany, clear, max, full: selected.size >= max }
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
import { useState } from 'react'
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { adminApi, shortSha, type Automation, type CandidateSource, type SettingsData, type WatchedCollection } from './api'
|
||||
import { confirm } from './confirm'
|
||||
import { errorText, plural, when } from './texts'
|
||||
import { Notice, type NoticeState } from './ui'
|
||||
|
||||
const SOURCES: { id: CandidateSource; label: string }[] = [
|
||||
{ id: 'rss', label: 'RSS discovery' },
|
||||
{ id: 'form', label: 'Suggest form' },
|
||||
]
|
||||
|
||||
export function SettingsPage() {
|
||||
const settings = useQuery({ queryKey: ['settings'], queryFn: adminApi.settings })
|
||||
return (
|
||||
<section className="max-w-4xl space-y-8">
|
||||
<h1 className="text-lg font-semibold">Settings</h1>
|
||||
{settings.error && <Notice text={errorText(settings.error)} tone="error" onClose={() => void settings.refetch()} />}
|
||||
{settings.isPending && <p className="text-sm text-muted-foreground">Loading…</p>}
|
||||
{settings.data && (
|
||||
<>
|
||||
{!settings.data.writes && (
|
||||
<Notice
|
||||
tone="error"
|
||||
text="The GitHub App is not configured: the cron cannot commit, so neither auto-approve nor followed collections run."
|
||||
/>
|
||||
)}
|
||||
<AutomationForm key={JSON.stringify(settings.data.automation)} data={settings.data} />
|
||||
<LastRun data={settings.data} />
|
||||
<Collections collections={settings.data.collections} />
|
||||
</>
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function AutomationForm({ data }: { data: SettingsData }) {
|
||||
const [form, setForm] = useState<Omit<Automation, 'since'>>({
|
||||
enabled: data.automation.enabled,
|
||||
sources: data.automation.sources,
|
||||
hours: data.automation.hours,
|
||||
})
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
const client = useQueryClient()
|
||||
const since = data.automation.since
|
||||
const everything = since <= '1970-01-01T00:00:00Z'
|
||||
|
||||
const save = async (includeOlder = false) => {
|
||||
if (includeOlder) {
|
||||
const ok = await confirm({
|
||||
title: 'Include older pending items?',
|
||||
body: `Every unflagged pending item from the chosen sources that has waited ${form.hours} hours qualifies, not only those found after ${when(since)}. The cron approves at most 100 per run (every 4 hours).`,
|
||||
action: 'Include them',
|
||||
})
|
||||
if (!ok) return
|
||||
}
|
||||
setBusy(true)
|
||||
try {
|
||||
await adminApi.saveAutomation({ ...form, ...(includeOlder ? { includeOlder: true } : {}) })
|
||||
setNotice({ tone: 'info', text: 'Saved. The next cron run (every 4 hours, at :23) uses it.' })
|
||||
void client.invalidateQueries({ queryKey: ['settings'] })
|
||||
} catch (e) {
|
||||
setNotice({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border p-4 text-sm">
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div>
|
||||
<h2 className="text-base font-semibold">Auto-approve the review queue</h2>
|
||||
<p className="text-muted-foreground">
|
||||
Pending items without a flag (NSFW?, demo, previously removed) that nobody decided on are queued for
|
||||
ingest after a while. Paid or dead games are still dropped by the ingest.
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={form.enabled}
|
||||
onCheckedChange={(enabled) => setForm({ ...form, enabled })}
|
||||
aria-label="Auto-approve"
|
||||
/>
|
||||
</div>
|
||||
<fieldset className="mt-4 flex flex-wrap gap-4" disabled={!form.enabled}>
|
||||
<legend className="mb-1 text-xs text-muted-foreground">Sources</legend>
|
||||
{SOURCES.map((s) => (
|
||||
<label key={s.id} className="flex items-center gap-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={form.sources.includes(s.id)}
|
||||
onChange={(e) =>
|
||||
setForm({
|
||||
...form,
|
||||
sources: e.target.checked ? [...form.sources, s.id] : form.sources.filter((x) => x !== s.id),
|
||||
})
|
||||
}
|
||||
/>
|
||||
{s.label}
|
||||
</label>
|
||||
))}
|
||||
</fieldset>
|
||||
<label className="mt-4 flex items-center gap-2">
|
||||
<span>After</span>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={720}
|
||||
value={form.hours}
|
||||
disabled={!form.enabled}
|
||||
onChange={(e) => setForm({ ...form, hours: Math.round(Number(e.target.value)) || 1 })}
|
||||
className="h-8 w-20"
|
||||
/>
|
||||
<span>hours without a decision</span>
|
||||
</label>
|
||||
<p className="mt-3 text-xs text-muted-foreground">
|
||||
{everything
|
||||
? 'Applies to every pending item.'
|
||||
: `Applies to items found after ${when(since)} (switching it on starts a new baseline, so a backlog is not approved all at once).`}
|
||||
</p>
|
||||
{notice && (
|
||||
<div className="mt-3">
|
||||
<Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />
|
||||
</div>
|
||||
)}
|
||||
<div className="mt-3 flex flex-wrap gap-2">
|
||||
<Button disabled={busy} onClick={() => void save()}>
|
||||
{busy ? 'Saving…' : 'Save'}
|
||||
</Button>
|
||||
{form.enabled && !everything && (
|
||||
<Button variant="outline" disabled={busy} onClick={() => void save(true)}>
|
||||
Include older pending items
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function LastRun({ data }: { data: SettingsData }) {
|
||||
const run = data.lastRun
|
||||
return (
|
||||
<div className="text-sm">
|
||||
<h2 className="mb-1 text-base font-semibold">Last automatic run</h2>
|
||||
{!run ? (
|
||||
<p className="text-muted-foreground">None yet (the cron runs every 4 hours, at :23).</p>
|
||||
) : (
|
||||
<div className="space-y-1">
|
||||
<p>
|
||||
{when(run.at)}: {plural(run.approved, 'item')} auto-approved, {plural(run.added, 'game')} from collections
|
||||
{run.review > 0 && `, ${plural(run.review, 'item')} sent to review`}
|
||||
{run.duplicates > 0 && `, ${plural(run.duplicates, 'item')} already in the catalog`} ·{' '}
|
||||
{plural(run.pages, 'collection page')} read{run.sha && ` · commit ${shortSha(run.sha)}`}
|
||||
</p>
|
||||
{run.errors.map((e) => (
|
||||
<p key={e} className="text-destructive">
|
||||
{e}
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const STATUS_TEXT: Record<number, string> = {
|
||||
0: 'itch.io unreachable',
|
||||
200: 'OK',
|
||||
404: 'not found',
|
||||
422: 'page unreadable (markup changed?)',
|
||||
429: 'rate limited',
|
||||
500: 'not found',
|
||||
503: 'rate limited',
|
||||
}
|
||||
|
||||
function Collections({ collections }: { collections: WatchedCollection[] }) {
|
||||
const [url, setUrl] = useState('')
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [notice, setNotice] = useState<NoticeState>(null)
|
||||
const client = useQueryClient()
|
||||
|
||||
const run = async (action: () => Promise<unknown>, done: string) => {
|
||||
setBusy(true)
|
||||
try {
|
||||
await action()
|
||||
setNotice({ tone: 'info', text: done })
|
||||
void client.invalidateQueries({ queryKey: ['settings'] })
|
||||
} catch (e) {
|
||||
setNotice({ tone: 'error', text: errorText(e) })
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="text-sm">
|
||||
<h2 className="mb-1 text-base font-semibold">Followed itch.io collections</h2>
|
||||
<p className="mb-3 text-muted-foreground">
|
||||
The cron reads them in turn (up to 30 pages per run, one a second) and queues their new free games directly.
|
||||
A game removed before, or with a flag, goes to the review queue instead; one the queue already knows is left
|
||||
to you.
|
||||
</p>
|
||||
{notice && <Notice text={notice.text} tone={notice.tone} onClose={() => setNotice(null)} />}
|
||||
<div className="mb-3 flex flex-wrap gap-2">
|
||||
<Input
|
||||
value={url}
|
||||
onChange={(e) => setUrl(e.target.value)}
|
||||
placeholder="https://itch.io/c/1234567/collection-name"
|
||||
aria-label="Collection to follow"
|
||||
className="max-w-xl flex-1"
|
||||
/>
|
||||
<Button
|
||||
disabled={busy || !url.trim()}
|
||||
onClick={() =>
|
||||
void run(async () => {
|
||||
await adminApi.watch(url)
|
||||
setUrl('')
|
||||
}, 'Following it: the next cron run reads it.')
|
||||
}
|
||||
>
|
||||
Follow
|
||||
</Button>
|
||||
</div>
|
||||
{collections.length === 0 ? (
|
||||
<p className="text-muted-foreground">None yet.</p>
|
||||
) : (
|
||||
<table className="w-full text-sm">
|
||||
<thead className="border-b text-left text-xs uppercase text-muted-foreground">
|
||||
<tr>
|
||||
<th className="py-2 pr-3">Collection</th>
|
||||
<th className="py-2 pr-3">On</th>
|
||||
<th className="py-2 pr-3">Last read</th>
|
||||
<th className="py-2 pr-3">Found / queued</th>
|
||||
<th className="py-2" />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{collections.map((c) => (
|
||||
<tr key={c.id} className="border-b align-top last:border-0">
|
||||
<td className="py-2 pr-3">
|
||||
<a href={`https://itch.io/c/${c.id}/${c.slug}`} target="_blank" rel="noreferrer noopener" className="font-medium hover:underline">
|
||||
#{c.id}
|
||||
</a>{' '}
|
||||
<span className="text-xs text-muted-foreground">{c.slug}</span>
|
||||
</td>
|
||||
<td className="py-2 pr-3">
|
||||
<Switch
|
||||
checked={c.enabled === 1}
|
||||
disabled={busy}
|
||||
aria-label={`Read collection ${c.id}`}
|
||||
onCheckedChange={(on) => void run(() => adminApi.toggleCollection(c.id, on), on ? 'Switched on.' : 'Switched off.')}
|
||||
/>
|
||||
</td>
|
||||
<td className="py-2 pr-3 text-xs">
|
||||
{c.last_polled_at ? when(c.last_polled_at) : 'not yet'}
|
||||
{c.last_status !== null && (
|
||||
<div className="text-muted-foreground">
|
||||
{STATUS_TEXT[c.last_status] ?? `HTTP ${c.last_status}`}
|
||||
{c.backoff_until && ` · paused until ${when(c.backoff_until)}`}
|
||||
{c.next_page > 1 && ` · continues at page ${c.next_page}`}
|
||||
</div>
|
||||
)}
|
||||
</td>
|
||||
<td className="py-2 pr-3 text-xs tabular-nums">
|
||||
{c.last_found ?? '—'} / {c.last_added ?? '—'}
|
||||
</td>
|
||||
<td className="py-2 text-right">
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
disabled={busy}
|
||||
onClick={async () => {
|
||||
const ok = await confirm({
|
||||
title: `Stop following collection #${c.id}?`,
|
||||
body: 'Games already queued or added stay.',
|
||||
action: 'Stop following',
|
||||
destructive: true,
|
||||
})
|
||||
if (ok) void run(() => adminApi.unwatch(c.id), 'Stopped following it.')
|
||||
}}
|
||||
>
|
||||
Remove
|
||||
</Button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { ApiError, SessionExpired } from './api'
|
||||
import { ERROR_TEXT, errorText } from './texts'
|
||||
|
||||
// The sources of every error the admin API answers with.
|
||||
const SOURCES = import.meta.glob<string>(['../worker/admin.ts', '../worker/http.ts'], {
|
||||
query: '?raw',
|
||||
import: 'default',
|
||||
eager: true,
|
||||
})
|
||||
|
||||
describe('admin error texts', () => {
|
||||
it('has wording for every error code of the admin API', () => {
|
||||
const codes = new Set(
|
||||
Object.values(SOURCES).flatMap((src) => [...src.matchAll(/HttpError\(\d{3}, '([a-z_]+)'/g)].map((m) => m[1])),
|
||||
)
|
||||
expect(codes.size).toBeGreaterThan(15)
|
||||
expect([...codes].filter((c) => !(c in ERROR_TEXT))).toEqual([])
|
||||
})
|
||||
|
||||
it('adds the server message on a second line, and never loses an unknown code', () => {
|
||||
expect(errorText(new ApiError(400, 'invalid_edit', 'nsfw must be one of No, Yes'))).toBe(
|
||||
`${ERROR_TEXT.invalid_edit}\nnsfw must be one of No, Yes`,
|
||||
)
|
||||
expect(errorText(new ApiError(418, 'teapot'))).toBe('Error: teapot.')
|
||||
expect(errorText(new SessionExpired())).toMatch(/expired/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,81 @@
|
||||
/** The admin's wording for API errors, skip reasons and outcomes (the admin is English only). */
|
||||
import { ApiError, SessionExpired } from './api'
|
||||
|
||||
export const PUBLISH_NOTE = 'The public site shows it after the next build (a few minutes).'
|
||||
|
||||
/** Every error code the Worker's admin API can answer (admin/texts.test.ts checks that none is missing). */
|
||||
export const ERROR_TEXT: Record<string, string> = {
|
||||
forbidden: 'This account is not allowed to use the admin.',
|
||||
not_configured: 'The admin is not configured yet (Cloudflare Access / D1).',
|
||||
github_not_configured: 'Writes are off: the GitHub App is not configured.',
|
||||
github_error: 'GitHub refused the change.',
|
||||
network_error: 'The server could not be reached. Check the connection and try again.',
|
||||
internal_error: 'Something went wrong on the server.',
|
||||
not_found: 'Not found.',
|
||||
invalid_body: 'The request was malformed.',
|
||||
invalid_json: 'The request was malformed.',
|
||||
unsupported_media_type: 'The request was malformed.',
|
||||
payload_too_large: 'Too much at once: send fewer games per request.',
|
||||
too_many_urls: 'Too many URLs in one request.',
|
||||
too_many_edits: 'Too many games in one bulk edit.',
|
||||
invalid_url: 'Not an itch.io game URL.',
|
||||
invalid_status: 'Unknown queue status.',
|
||||
invalid_action: 'Unknown queue action.',
|
||||
invalid_edit: 'Only the virus check, 18+ and notes can be changed, with allowed values.',
|
||||
invalid_reason: 'A reason is required (200 characters at most).',
|
||||
not_in_catalog: 'That game is not in the catalog (any more).',
|
||||
needs_override: 'Previously removed: approve with override.',
|
||||
invalid_settings: 'Those settings are not valid.',
|
||||
invalid_collection_url: 'Not an itch.io collection URL (https://itch.io/c/<id>/<name>).',
|
||||
collection_unavailable: 'itch.io has no such collection, or it is not public.',
|
||||
collection_unreadable: 'itch.io answered, but its page could not be read: its markup may have changed.',
|
||||
collection_exists: 'That collection is already followed.',
|
||||
too_many_collections: 'No more collections can be followed.',
|
||||
itch_rate_limited: 'itch.io asks to slow down.',
|
||||
itch_error: 'itch.io could not be reached.',
|
||||
}
|
||||
|
||||
export function errorText(e: unknown): string {
|
||||
if (e instanceof SessionExpired) return e.message
|
||||
if (e instanceof ApiError) {
|
||||
const known = ERROR_TEXT[e.code] ?? `Error: ${e.code.replace(/_/g, ' ')}.`
|
||||
// The server's message adds the specifics (which field, how long to wait…).
|
||||
return e.message && e.message !== e.code ? `${known}\n${e.message}` : known
|
||||
}
|
||||
return e instanceof Error ? e.message : String(e)
|
||||
}
|
||||
|
||||
/** Why an approval skipped a URL. */
|
||||
export const SKIP_TEXT: Record<string, string> = {
|
||||
not_in_queue: 'not in the queue',
|
||||
already_queued: 'already queued',
|
||||
already_ingested: 'already in the catalog',
|
||||
needs_override: 'removed before: needs override',
|
||||
}
|
||||
|
||||
/** How a queued game's ingest ended (the pipeline's outcomes, or the cron's timeout). */
|
||||
export const INGEST_TEXT: Record<string, string> = {
|
||||
added: 'added',
|
||||
duplicate: 'already in the catalog',
|
||||
paid: 'paid: not added',
|
||||
dead: 'page gone',
|
||||
private: 'password-protected',
|
||||
deleted: 'removed before: blocked',
|
||||
gave_up: 'failed 3 times: given up',
|
||||
timeout: 'no result within 3 days',
|
||||
}
|
||||
|
||||
export const WARNING_TEXT: Record<string, string> = {
|
||||
covers_failed: 'the resized covers could not be deleted (they expire from the cache on their own)',
|
||||
audit_failed: 'the activity log entry could not be written',
|
||||
}
|
||||
|
||||
export function plural(n: number, one: string, many = `${one}s`): string {
|
||||
return `${n.toLocaleString('en-GB')} ${n === 1 ? one : many}`
|
||||
}
|
||||
|
||||
export function when(iso: string | null | undefined): string {
|
||||
if (!iso) return '—'
|
||||
const date = new Date(iso)
|
||||
return Number.isNaN(date.getTime()) ? iso : date.toLocaleString('en-GB', { dateStyle: 'medium', timeStyle: 'short' })
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { capBatches, summarize, type TrackedBatch } from './tracking'
|
||||
|
||||
const batch = (n: number, size: number): TrackedBatch => ({
|
||||
id: String(n),
|
||||
label: `b${n}`,
|
||||
at: '2026-10-13T00:00:00Z',
|
||||
urls: Array.from({ length: size }, (_, i) => `https://dev.itch.io/g${n}-${i}`),
|
||||
})
|
||||
|
||||
describe('tracked batches', () => {
|
||||
it('keeps the newest 20 batches and at most 10,000 URLs', () => {
|
||||
expect(capBatches(Array.from({ length: 25 }, (_, i) => batch(i, 1))).map((b) => b.id)).toEqual(
|
||||
Array.from({ length: 20 }, (_, i) => String(i)),
|
||||
)
|
||||
expect(capBatches([batch(1, 6000), batch(2, 5000), batch(3, 10)]).map((b) => b.id)).toEqual(['1'])
|
||||
})
|
||||
|
||||
it('counts games per state', () => {
|
||||
expect(
|
||||
summarize([
|
||||
{ url: 'a', state: 'added' },
|
||||
{ url: 'b', state: 'added' },
|
||||
{ url: 'c', state: 'dropped', outcome: 'paid' },
|
||||
{ url: 'd', state: 'waiting' },
|
||||
]),
|
||||
).toEqual({ added: 2, waiting: 1, dropped: 1, unknown: 0 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Batches the maintainer queued (Add, Queue approvals, Restore), kept in this
|
||||
* browser so their ingest can be followed after a reload: the Add page asks
|
||||
* /api/admin/track how each game ended.
|
||||
*/
|
||||
import { create } from 'zustand'
|
||||
import { persist } from 'zustand/middleware'
|
||||
import type { TrackItem } from './api'
|
||||
|
||||
export interface TrackedBatch {
|
||||
id: string
|
||||
label: string
|
||||
/** When it was queued (server time): only outcomes after it count. */
|
||||
at: string
|
||||
urls: string[]
|
||||
}
|
||||
|
||||
const MAX_BATCHES = 20
|
||||
const MAX_URLS = 10_000
|
||||
|
||||
/** Newest first, at most 20 batches and 10,000 URLs in all. */
|
||||
export function capBatches(batches: TrackedBatch[]): TrackedBatch[] {
|
||||
const out: TrackedBatch[] = []
|
||||
let urls = 0
|
||||
for (const b of batches) {
|
||||
if (out.length >= MAX_BATCHES || urls + b.urls.length > MAX_URLS) break
|
||||
out.push(b)
|
||||
urls += b.urls.length
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
interface TrackingState {
|
||||
batches: TrackedBatch[]
|
||||
track: (label: string, at: string, urls: string[]) => void
|
||||
forget: (id: string) => void
|
||||
}
|
||||
|
||||
export const useTracking = create<TrackingState>()(
|
||||
persist(
|
||||
(set) => ({
|
||||
batches: [],
|
||||
track: (label, at, urls) =>
|
||||
set((s) =>
|
||||
urls.length === 0
|
||||
? s
|
||||
: { batches: capBatches([{ id: `${at}-${Math.random().toString(36).slice(2, 8)}`, label, at, urls }, ...s.batches]) },
|
||||
),
|
||||
forget: (id) => set((s) => ({ batches: s.batches.filter((b) => b.id !== id) })),
|
||||
}),
|
||||
{ name: 'admin.batches' },
|
||||
),
|
||||
)
|
||||
|
||||
export interface TrackSummary {
|
||||
added: number
|
||||
waiting: number
|
||||
dropped: number
|
||||
unknown: number
|
||||
}
|
||||
|
||||
export function summarize(items: TrackItem[]): TrackSummary {
|
||||
const out: TrackSummary = { added: 0, waiting: 0, dropped: 0, unknown: 0 }
|
||||
for (const i of items) out[i.state] += 1
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/** Small building blocks shared by the admin pages. */
|
||||
import { useState, type ReactNode } from 'react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Dialog, DialogContent, DialogDescription, DialogTitle } from '@/components/ui/dialog'
|
||||
import { THEME_ICON } from '@/components/theme-icon'
|
||||
import { thumbUrl } from '@/lib/thumbnail'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { useThemeStore, type Theme } from '@/stores/theme'
|
||||
import { answer, useConfirm } from './confirm'
|
||||
|
||||
/** A line reporting what the last action did; dismissible when `onClose` is given. */
|
||||
export function Notice({ text, tone = 'info', onClose }: { text: string; tone?: 'info' | 'error'; onClose?: () => void }) {
|
||||
return (
|
||||
<div
|
||||
role={tone === 'error' ? 'alert' : 'status'}
|
||||
className={cn(
|
||||
'mb-4 flex items-start gap-3 rounded-md border px-3 py-2 text-sm',
|
||||
tone === 'error' ? 'border-destructive/50 text-destructive' : 'bg-muted/50',
|
||||
)}
|
||||
>
|
||||
<span className="flex-1 whitespace-pre-line">{text}</span>
|
||||
{onClose && (
|
||||
<button aria-label="Dismiss" className="text-muted-foreground hover:text-foreground" onClick={onClose}>
|
||||
×
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export type NoticeState = { text: string; tone: 'info' | 'error' } | null
|
||||
|
||||
/** A row of tab-like buttons (the admin's sub-pages and filters). */
|
||||
export function TabBar<T extends string>({
|
||||
label,
|
||||
value,
|
||||
options,
|
||||
onChange,
|
||||
}: {
|
||||
label: string
|
||||
value: T
|
||||
options: { id: T; label: ReactNode }[]
|
||||
onChange: (id: T) => void
|
||||
}) {
|
||||
return (
|
||||
<div className="mb-3 flex flex-wrap gap-1 text-sm" role="tablist" aria-label={label}>
|
||||
{options.map((o) => (
|
||||
<button
|
||||
key={o.id}
|
||||
role="tab"
|
||||
aria-selected={value === o.id}
|
||||
onClick={() => onChange(o.id)}
|
||||
className={cn('rounded-md px-3 py-1.5 hover:bg-accent', value === o.id && 'bg-accent font-medium')}
|
||||
>
|
||||
{o.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** "N selected" and the actions on the selection, kept in view while scrolling a long list. */
|
||||
export function SelectionBar({ count, max, onClear, children }: { count: number; max: number; onClear: () => void; children: ReactNode }) {
|
||||
if (count === 0) return null
|
||||
return (
|
||||
<div className="sticky top-12 z-[5] mb-3 flex flex-wrap items-center gap-2 rounded-md border bg-background/95 px-3 py-2 text-sm shadow-xs backdrop-blur-sm">
|
||||
<span className="tabular-nums">
|
||||
{count} selected{count >= max && <span className="text-muted-foreground"> (the most one action takes)</span>}
|
||||
</span>
|
||||
<button className="text-muted-foreground underline-offset-4 hover:underline" onClick={onClear}>
|
||||
Clear
|
||||
</button>
|
||||
<div className="ml-auto flex flex-wrap gap-2">{children}</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export function BatchProgress({ done, total, label }: { done: number; total: number; label: string }) {
|
||||
if (total <= 1) return null
|
||||
return (
|
||||
<div className="my-3 text-sm" role="status">
|
||||
<div className="mb-1 text-muted-foreground">
|
||||
{label}: batch {Math.min(done + 1, total)} of {total}
|
||||
</div>
|
||||
<div className="h-1.5 overflow-hidden rounded-full bg-muted">
|
||||
<div className="h-full bg-primary transition-[width]" style={{ width: `${(done / total) * 100}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const NEXT_THEME: Record<Theme, Theme> = { light: 'dark', dark: 'system', system: 'light' }
|
||||
const THEME_NAME: Record<Theme, string> = { light: 'Light', dark: 'Dark', system: 'System' }
|
||||
|
||||
/** The site's theme choice (shared with the public site: same browser storage). */
|
||||
export function ThemeToggle() {
|
||||
const theme = useThemeStore((s) => s.theme)
|
||||
const setTheme = useThemeStore((s) => s.setTheme)
|
||||
const Icon = THEME_ICON[theme]
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setTheme(NEXT_THEME[theme])}
|
||||
title={`Theme: ${THEME_NAME[theme]} (click for ${THEME_NAME[NEXT_THEME[theme]]})`}
|
||||
aria-label={`Theme: ${THEME_NAME[theme]}. Switch to ${THEME_NAME[NEXT_THEME[theme]]}`}
|
||||
className="flex size-8 items-center justify-center rounded-md text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<Icon className="size-4" aria-hidden="true" />
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
/** The dialog behind confirm() (admin/confirm.ts); mounted once in the app shell. */
|
||||
export function ConfirmDialog() {
|
||||
const request = useConfirm((s) => s.request)
|
||||
return (
|
||||
<Dialog open={request !== null} onOpenChange={(open) => !open && answer(false)}>
|
||||
<DialogContent className="max-w-md">
|
||||
<div className="space-y-2 pr-6">
|
||||
<DialogTitle className="text-lg font-semibold tracking-tight">{request?.title}</DialogTitle>
|
||||
{request?.body && (
|
||||
<DialogDescription className="text-sm whitespace-pre-line text-muted-foreground">{request.body}</DialogDescription>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
|
||||
<Button variant="ghost" onClick={() => answer(false)}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant={request?.destructive ? 'destructive' : 'default'} onClick={() => answer(true)}>
|
||||
{request?.action}
|
||||
</Button>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
/** A game's cover: the site's resized copy, or itch.io's original if that fails (e.g. no verification pass). */
|
||||
export function Cover({ src, className }: { src: string | null | undefined; className?: string }) {
|
||||
const [failed, setFailed] = useState(false)
|
||||
if (!src || !src.startsWith('https://')) return <div className={cn('rounded bg-muted', className)} />
|
||||
return (
|
||||
<img
|
||||
src={failed ? src : thumbUrl(src, 160)}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
onError={() => setFailed(true)}
|
||||
className={cn('rounded object-cover', className)}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -6,6 +6,9 @@
|
||||
<meta name="theme-color" content="#0b0b0c" media="(prefers-color-scheme: dark)" />
|
||||
<meta name="theme-color" content="#fafaf9" media="(prefers-color-scheme: light)" />
|
||||
<meta name="color-scheme" content="dark light" />
|
||||
<!-- Before the first paint: the saved theme (webapp.theme), so a dark page never flashes light.
|
||||
Its sha256 is in the CSP of public/_headers, src-tauri/tauri.conf.json and worker/admin.ts. -->
|
||||
<script>try{var t=JSON.parse(localStorage.getItem('webapp.theme')||'null');t=t&&t.state&&t.state.theme;if(t==='dark'||t!=='light'&&matchMedia('(prefers-color-scheme: dark)').matches)document.documentElement.classList.add('dark')}catch(e){}</script>
|
||||
|
||||
<title>Free Itch Games — free games on itch.io, sorted and kept fresh</title>
|
||||
<meta name="description" content="A curated, auto-updating catalog of 2,600+ free games on itch.io. Search, filter and explore the dataset." />
|
||||
|
||||
Generated
+1173
-786
File diff suppressed because it is too large.
Load diff
Loaded 100 of 215 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user