Merge pull request #226 from poli0981/feat/all-languages
v4.6.0 "Polyglot": games in every language
This commit is contained in:
52 files changed
+1345
-105
No files matched your search
@@ -3,10 +3,15 @@ name: Backfill Discovery (TEMPORARY)
|
||||
# >>> TEMPORARY. DELETE THIS FILE AND scripts/backfill_discover.py ONCE THE
|
||||
# >>> HISTORICAL BACKLOG HAS BEEN WORKED THROUGH.
|
||||
#
|
||||
# Measured 2026-09-10: Steam lists 16,668 free games matching the catalogue's
|
||||
# own filters; data/ holds 3,424. This workflow chews through that gap one
|
||||
# release window at a time. `Discover New Games` covers everything from today
|
||||
# forward and is permanent -- this one is not.
|
||||
# Measured 2026-09-10: Steam listed 16,668 free games that support English;
|
||||
# data/ held 3,424. This workflow chews through that gap one release window at
|
||||
# a time. `Discover New Games` covers everything from today forward and is
|
||||
# permanent -- this one is not.
|
||||
#
|
||||
# Since v4.6.0 discovery takes games in every language (18,790 free games on
|
||||
# 2026-10-30, 1,592 of them without English), so a window swept before then
|
||||
# needs one more pass. A whole recent year is ~30 search pages; a window is
|
||||
# done only when the log says "walked past the window".
|
||||
#
|
||||
# Same propose-only contract as the daily job: `contents: read`, and the same
|
||||
# guard step asserting no dataset-writing helper is reachable.
|
||||
@@ -29,6 +34,10 @@ on:
|
||||
description: 'Max candidates to propose in one run'
|
||||
required: false
|
||||
default: '250'
|
||||
max_pages:
|
||||
description: 'Search pages to walk once the window is found (100 rows each; a whole recent year is ~30)'
|
||||
required: false
|
||||
default: '40'
|
||||
dry_run:
|
||||
description: 'Find candidates but do not POST them'
|
||||
type: boolean
|
||||
@@ -51,9 +60,12 @@ permissions:
|
||||
concurrency:
|
||||
# Shares the `discovery` group with the daily job on purpose: two sweeps
|
||||
# hammering Steam's search from the same runner pool is the one way to earn a
|
||||
# rate limit that hurts both.
|
||||
# rate limit that hurts both. `queue: max`, like the daily job: windows are
|
||||
# dispatched several at a time, and the default single waiting slot would
|
||||
# cancel all but the last.
|
||||
group: discovery
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
backfill:
|
||||
@@ -102,6 +114,7 @@ jobs:
|
||||
YEAR: ${{ inputs.year }}
|
||||
QUARTER: ${{ inputs.quarter }}
|
||||
LIMIT: ${{ inputs.limit || '250' }}
|
||||
MAX_PAGES: ${{ inputs.max_pages || '40' }}
|
||||
DRY: ${{ inputs.dry_run && '--dry-run' || '' }}
|
||||
run: |
|
||||
# An absent YEAR means this is the scheduled run, so both env defaults
|
||||
@@ -119,7 +132,7 @@ jobs:
|
||||
fi
|
||||
# A blank quarter must be omitted from argv, not passed as an empty
|
||||
# string: argparse rejects '' as an invalid choice.
|
||||
args="--year $YEAR --limit $LIMIT"
|
||||
args="--year $YEAR --limit $LIMIT --max-pages $MAX_PAGES"
|
||||
if [ -n "$QUARTER" ]; then args="$args --quarter $QUARTER"; fi
|
||||
echo "backfill_discover.py $args $DRY"
|
||||
python scripts/backfill_discover.py $args $DRY
|
||||
@@ -34,11 +34,17 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
# Its own group, NOT `data-write`: this job writes nothing to the repo, so
|
||||
# queueing it behind the ~115-minute data jobs would delay discovery for no
|
||||
# queueing it behind the long data jobs would delay discovery for no
|
||||
# benefit. Two overlapping discovery runs are the only real hazard, and this
|
||||
# covers that.
|
||||
#
|
||||
# `queue: max` keeps every waiting run (up to 100, first in, first out).
|
||||
# Without it the group keeps ONE: a third run - a backfill dispatched while
|
||||
# another waits - cancels the one already waiting. Every member of the group
|
||||
# says the same (Backfill Discovery too); web/shared/pipeline.test.ts holds it.
|
||||
group: discovery
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
discover:
|
||||
|
||||
@@ -4,6 +4,72 @@ All notable changes to this awesome noob repo will be documented here.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [v4.6.0] – 2026-10-30 (The "Polyglot" Edition)
|
||||
|
||||
The catalogue now takes free games in every language, not only the ones
|
||||
that support English: about 1,300 more, most of them Chinese, Japanese or
|
||||
Korean, arrive through the review queue over the next days. The site and
|
||||
the apps get a language filter, search that matches across scripts and
|
||||
widths, and CSV files that Excel opens correctly; and 1,357 games filed under
|
||||
"Free To Play" get a real genre. Desktop and Android
|
||||
bumped `2.3.0` → `2.4.0`; repo public-facing version `4.5.1` → `4.6.0`.
|
||||
The desktop app offers 2.4.0 through its updater once the release is
|
||||
published; on Android, install the new APK over the old one.
|
||||
|
||||
### ✨ Added
|
||||
|
||||
- **Games in every language.** Discovery used to search Steam only for free
|
||||
games that list English, which left out 1,592 of them: mostly Simplified
|
||||
Chinese (784), Japanese (317), Korean (165) and Traditional Chinese (164)
|
||||
only, the free Japanese visual novel `//トロイメライ・ノクターン` among them.
|
||||
It now proposes every free game whatever its languages, and every release
|
||||
window swept before gets one more pass. The pipeline still asks Steam in English, so
|
||||
tags, genres and language names stay in English; a name or description the
|
||||
developer never wrote in English stays in their language.
|
||||
- **A language filter on Games**, counted like the others and part of the
|
||||
link (`?lang=Japanese`), so a reader can keep to the languages they read.
|
||||
The About page says the catalogue now covers every language.
|
||||
- **Search across scripts and widths.** Every search box (Games, the command
|
||||
palette, developers, publishers and the anti-cheat list) now matches
|
||||
half-width and full-width forms and hiragana and katakana as one: `トロイ`,
|
||||
`とろい` and `トロイ` find the same game, and `4` finds `4`. One kanji, kana
|
||||
or Hangul syllable is enough in the command palette.
|
||||
- `/admin` shows a candidate's languages, with a **No English** badge when
|
||||
English is not among them. Such games are decided like any other.
|
||||
|
||||
### 🐛 Fixed
|
||||
|
||||
- The header search no longer jumps to Games in the middle of a word typed
|
||||
with a Japanese, Chinese or Korean input method; it waits until the word is
|
||||
chosen. And it keeps its focus when it does jump: until now the next letter
|
||||
typed after the jump went nowhere, whatever the language.
|
||||
- A CSV export opens in Excel with its Japanese, Chinese and Korean names
|
||||
intact (it now starts with a byte order mark).
|
||||
- A long Chinese or Japanese description is cut at the end of a sentence
|
||||
where it can be, and never ends in `、…` or `。…`.
|
||||
- **"Free To Play" is no longer a genre.** Steam spells it with a capital T,
|
||||
the list of names never to use as a genre had a small one, and 1,357 games
|
||||
were filed under "Free To Play" - the third most common genre, and one that
|
||||
says nothing about a catalogue of free games. New games get their real
|
||||
genre, and the daily store refresh gives the 1,357 theirs from Steam, 150 a
|
||||
day (about nine days). A genre someone set by hand is never touched.
|
||||
- A queued store link written in full-width digits (`5118700`) is read
|
||||
as the appid it is; digits of any other script are refused. Such an appid
|
||||
used to slip through, and the next daily refresh would have stopped on it.
|
||||
The blocklist now accepts exactly the lines the site's own reader does.
|
||||
|
||||
### 🧰 Pipeline
|
||||
|
||||
- Discovery runs wait in turn: backfill windows dispatched together no longer
|
||||
cancel each other, and a window can be swept a whole year at a time
|
||||
(`max_pages`). Each run says how many candidates do not list English.
|
||||
|
||||
### 📚 Docs
|
||||
|
||||
- `docs/plan/19-all-languages.md`: what kept these games out, what every
|
||||
workflow does with them, and what is left (games not sold in the US, CJK
|
||||
fonts).
|
||||
|
||||
## [v4.5.1] – 2026-10-27 (The "Speed Bump" Edition)
|
||||
|
||||
The site's Worker now limits how fast one address can call it, behind
|
||||
|
||||
@@ -15,7 +15,13 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
|
||||
- **`appid` is not a stored field.** It is parsed out of `link` with
|
||||
`/\/app\/(\d+)/` (`web/src/lib/data-store.ts`, `extract_appid` in
|
||||
`scripts/core/data_store.py`). Never add it as a column; derive it.
|
||||
`scripts/core/data_store.py`). Never add it as a column; derive it. It is
|
||||
ASCII digits on both sides: JavaScript's `\d` is ASCII, Python's is not, so
|
||||
`extract_appid` refuses a run of digits with any non-ASCII one in it
|
||||
(`/app/730٤` is `None`, never 730) and `normalize_link()` reads a queued line
|
||||
through NFKC first, which turns a Japanese keyboard's `5118700` into
|
||||
5118700. A non-ASCII appid would miss every comparison and make
|
||||
`rotation.slice_of()` (`.encode("ascii")`) stop every refresh.
|
||||
|
||||
- **Numeric-looking fields are formatted strings, not numbers.**
|
||||
`current_players: "492,197"`, `reviews: "86% (Very Positive)"`,
|
||||
@@ -67,6 +73,13 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
field an active override covers, or a retired entry's value, would be undone
|
||||
by `apply_overrides()` and bump `last_updated` on every run.
|
||||
|
||||
One stored genre is not a judgement: a value in `NOT_GENRES` ("Free To
|
||||
Play", "Early Access"; `constants.py`), a price model or a release state.
|
||||
Until v4.6.0 the skip list matched appdetails' names case-sensitively and
|
||||
1,357 games were stored with "Free To Play". `fetcher.pick_genre()` compares
|
||||
casefolded and never falls back to one, and Refresh Store Data replaces a
|
||||
stored one (`is_stray_genre()`) from appdetails, guarded like any writer.
|
||||
|
||||
- **`data/overrides/<appid>.json` is human intent, and it is permanent.**
|
||||
`apply_overrides()` is called from inside `save_main()` — the single choke
|
||||
point through which all 15 data-writing scripts pass, and the only code that
|
||||
@@ -230,11 +243,15 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
The slice is visited in `rotation.daily_order()`, so a run that meets its
|
||||
time budget leaves a different tail each day. It
|
||||
replaces values only with NON-EMPTY ones, never writes `MANUAL_FIELDS` or
|
||||
`notes`, trusts `has_paid_dlc` only from a real store page (an age gate has
|
||||
`notes` - except a stray genre (above), read from appdetails every day,
|
||||
first after the page repairs, at most `GENRE_DAILY_CAP` a day, and left
|
||||
alone under an override - trusts `has_paid_dlc` only from a real store page (an age gate has
|
||||
no language table or tags) or a gated game's DLC prices, compares every
|
||||
value as stored (normalised, `description` truncated - and a blurb merely
|
||||
cut off later, now that `"` is one character, is written without
|
||||
counting as news), and bumps `last_updated` only on a real change. A game
|
||||
cut off somewhere else is written without counting as news: later, now
|
||||
that `"` is one character, or earlier, at a CJK sentence end since
|
||||
v4.6.0; `_fuller_cut()` takes two cuts of one text, never a rewrite), and
|
||||
bumps `last_updated` only on a real change. A game
|
||||
with empty tags, languages or `language_details` is in the page group EVERY
|
||||
day, first, capped at `MISSING_DAILY_CAP`; `--missing-only` runs just those.
|
||||
A game whose appdetails no longer call it free is LISTED in a `::warning::`,
|
||||
@@ -372,6 +389,32 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
rule a reviewer turned on there (off by default, at most 500 a day). Either
|
||||
way it is a REQUEST in `temp_info.jsonl` that `ingest_new.py` still checks
|
||||
game by game.
|
||||
- **Discovery takes games in every language** (since v4.6.0). The store search
|
||||
(`SteamClient.fetch_search_page()`) sends no `supportedlang`; pinned to
|
||||
english it left out 1,592 of 18,790 free games (2026-10-30), mostly Chinese,
|
||||
Japanese or Korean only, and without it Steam filters on no language at all.
|
||||
Rules that are easy to break:
|
||||
- The search sends `l=english`. It fixes the language of the row TEXT, never
|
||||
which rows come back: discovery skips a row whose release reads "Coming
|
||||
soon" and the backfill parses English month names.
|
||||
- Every other request still asks Steam in English, so tags, genre, language
|
||||
names, review labels and dates stay English; a name or description the
|
||||
developer never wrote in English stays in their script. That is the
|
||||
record, not a parsing error.
|
||||
- A candidate's payload carries `languages` (appdetails'
|
||||
`supported_languages`, read by `appdetails_languages()`), which `/admin`
|
||||
shows with a "No English" badge. Auto-approve treats such a game like any
|
||||
other. Rows queued earlier have no list: the Worker never rewrites a
|
||||
queued row's `payload_json`.
|
||||
- Both workflows' guard step greps the two discovery scripts for
|
||||
`save_main|save_jsonl|clear_temp|fetch_full` as WORDS, comments and
|
||||
docstrings included: none of the four may appear in them at all.
|
||||
- The `discovery` group queues (`queue: max`) in both of its workflows, so
|
||||
backfill windows dispatched together wait in turn instead of cancelling
|
||||
each other; `pipeline.test.ts` holds a group to one setting.
|
||||
- A game that declares no language (29 of them) has empty `languages` once
|
||||
ingested: Refresh Store Data's repair group takes it every day and the
|
||||
admin Quality page flags it. Accepted.
|
||||
- **Auto-approve (`worker/lib/auto-approve.ts`, run by the cron after
|
||||
reconcile) commits a batch only when every gate says yes**, and a gate it
|
||||
cannot evaluate says no: enabled (or "Run one batch now"), its D1 lease,
|
||||
@@ -1076,6 +1119,31 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
`url.search` in markup or a load - prerender throws on it. The page number
|
||||
lives in the filter store, keyed to the criteria (`filters.pageFor`), so Back
|
||||
from a game keeps the page and a changed filter starts again at page 1.
|
||||
The language filter (`?lang=`) offers the names stored in `languages`,
|
||||
Steam's English ones: data, never i18n keys.
|
||||
- **Search folds; storage never does.** `lib/search-fold.ts` `fold()` is NFKC
|
||||
plus hiragana to katakana, applied to BOTH sides of every search box
|
||||
(`/games`' Fuse index, the palette, /developers, /publishers, the anti-cheat
|
||||
list), so `トロイ`, `とろい` and `トロイ` find one game and a full-width `4`
|
||||
or `:` its ASCII twin. Never apply it to stored text: NFKC rewrites U+3000
|
||||
and half-width kana, which `save_main()` keeps as spelling. Rules:
|
||||
- Fuse gets one `getFn` per KEY. A wrapped `Fuse.config.getFn` hands back
|
||||
`{v, i}` objects for array fields (`tags`, `developer`, `publisher`), not
|
||||
strings, and folding those type-checks and then breaks.
|
||||
- Fold the query, THEN trim: NFKC turns U+3000 into a plain space.
|
||||
- Not Fuse's `ignoreDiacritics`: it strips the dakuten, making ガ equal カ.
|
||||
And no `includeMatches`: its positions would point into folded text.
|
||||
- The header search jumps to /games only once an input method has finished
|
||||
composing (`lib/layout/header-search.ts`); the value is still written on
|
||||
every keystroke, because Android keyboards compose Latin words too. The
|
||||
jump is `goto("/games", {reset: false})`: the default reset moved focus to
|
||||
`<body>` and the next keystroke was lost. The page scrolls inside `<main>`,
|
||||
which no reset touches, so focus is all `reset` changes there.
|
||||
- The palette takes one kanji, kana or Hangul syllable where Latin needs two.
|
||||
- A CSV download starts with a byte order mark (`csvForExcel()`), or Excel
|
||||
opens CJK names as mojibake. It is the escape `"\uFEFF"` in the source;
|
||||
the editing tools turn a typed escape into the raw character, so write it
|
||||
with a script and check the bytes.
|
||||
- **`safe` is classified by `lib/safety.ts`** (`y`, `n`, `?` = not reviewed,
|
||||
`""` = never set). `/games`, `/stats` and `/health` all use it; `?` is a
|
||||
to-do, not missing data, so /health reports it as its own group.
|
||||
|
||||
@@ -78,7 +78,7 @@ npm run tauri:build # release binary into web/src-tauri/target/release/bundle
|
||||
<details>
|
||||
<summary><b>The admin at <code>/admin</code></b></summary>
|
||||
|
||||
`Discover New Games` sweeps the Steam store daily and proposes candidates into a Cloudflare D1 queue. The maintainer reviews them in a small Svelte app embedded in the Worker (never shipped to visitors): approving commits the store link to `scripts/temp_info.jsonl`, and the pipeline does the rest, checking every game again (free, a game, reachable) before it is published. Approval can also run on its own, in batches with a daily cap, once the maintainer turns it on. Decided and already-published rows are read-only. Corrections to published games are saved as override files that survive every pipeline run, and a removed game goes into `data/blocklist.jsonl`, which keeps it out of the catalogue until it is restored. An overview, the pipeline's runs and a data-quality report complete it. See [`docs/ADMIN.md`](docs/ADMIN.md).
|
||||
`Discover New Games` sweeps the Steam store daily for free games in every language, not only the ones that support English, and proposes candidates into a Cloudflare D1 queue. The maintainer reviews them in a small Svelte app embedded in the Worker (never shipped to visitors): approving commits the store link to `scripts/temp_info.jsonl`, and the pipeline does the rest, checking every game again (free, a game, reachable) before it is published. Approval can also run on its own, in batches with a daily cap, once the maintainer turns it on. Decided and already-published rows are read-only. Corrections to published games are saved as override files that survive every pipeline run, and a removed game goes into `data/blocklist.jsonl`, which keeps it out of the catalogue until it is restored. An overview, the pipeline's runs and a data-quality report complete it. See [`docs/ADMIN.md`](docs/ADMIN.md).
|
||||
|
||||
</details>
|
||||
|
||||
@@ -177,7 +177,7 @@ scripts/
|
||||
| Workflow | Trigger | What it does |
|
||||
| --------------------- | -------------------------------- | --------------------------------------------------------- |
|
||||
| Auto Update JSON | Daily 00:00 UTC | Fills in incomplete records (API + HTML scrape) |
|
||||
| Discover New Games | Daily 05:00 UTC | Propose new free games into the `/admin` review queue |
|
||||
| Discover New Games | Daily 05:00 UTC | Propose new free games, any language, to the `/admin` queue |
|
||||
| Ingest New | On `scripts/temp_info.jsonl` or `data/blocklist.jsonl` push, and every 3 h | Apply removals, then add approved games |
|
||||
| Top Online | Sun/Wed/Fri 03:00 UTC | Live player leaderboard |
|
||||
| Top Offline | 1st + 15th of month 05:00 UTC | Leaderboard for single-player F2P concurrents |
|
||||
@@ -197,7 +197,7 @@ The site itself is deployed by Cloudflare Workers Builds on every push to `main`
|
||||
### Contribute
|
||||
|
||||
New games arrive on their own: `Discover New Games` sweeps the Steam store
|
||||
daily into a review queue, and the maintainer approves them at `/admin` — by
|
||||
daily, in every language, into a review queue, and the maintainer approves them at `/admin` — by
|
||||
hand, or with an auto-approve rule that has a daily cap and is off until turned
|
||||
on. Discovery only proposes, and every approved game is checked again by the
|
||||
pipeline (free, a game, reachable) before it is published. Missing a free game
|
||||
|
||||
+22
-4
@@ -329,9 +329,11 @@ account higher up this page.
|
||||
|
||||
## 8. Working through the backlog
|
||||
|
||||
Measured 2026-09-10: Steam lists **16,668** free games matching the
|
||||
catalogue's own filters (`category1=998`, `maxprice=free`,
|
||||
`supportedlang=english`); `data/` holds **3,424**.
|
||||
Measured 2026-09-10: Steam listed **16,668** free games that support English
|
||||
(`category1=998`, `maxprice=free`, `supportedlang=english`); `data/` held
|
||||
**3,424**. Since v4.6.0 discovery takes games in every language, with no
|
||||
`supportedlang`: **18,790** free games on 2026-10-30, 1,592 of them without
|
||||
English ([docs/plan/19-all-languages.md](plan/19-all-languages.md)).
|
||||
|
||||
Two jobs close that gap, and they are not the same kind of thing:
|
||||
|
||||
@@ -360,6 +362,19 @@ a re-run reports `candidates: 0`; the scheduled run's window is the
|
||||
`SWEEP_YEAR`/`SWEEP_QUARTER` block at the top of the workflow file, which you
|
||||
bump as you go.
|
||||
|
||||
Every window swept before v4.6.0 found only games that list English, so it
|
||||
needs one more pass. A whole year at a time works once `max_pages` covers it (a
|
||||
recent year is about 30 pages of 100):
|
||||
|
||||
```bash
|
||||
gh workflow run "Backfill Discovery (TEMPORARY)" -f year=2024 -f limit=1000 -f max_pages=60
|
||||
```
|
||||
|
||||
A window is done when its log says `walked past the window`; `stopping: hit
|
||||
--max-pages` means it was cut short. Runs dispatched together wait in turn
|
||||
(`queue: max`). Each log counts the candidates without English, and a dry run
|
||||
lists their languages.
|
||||
|
||||
When the backlog is closed, delete `scripts/backfill_discover.py`, delete
|
||||
`.github/workflows/backfill-discover.yml`, and drop
|
||||
`"Backfill Discovery (TEMPORARY)"` from the allowlist in
|
||||
@@ -421,7 +436,10 @@ a dialog is open, and never interrupt IME composition.
|
||||
|
||||
Click a game (or press `e`) for its details: the payload as submitted, the
|
||||
durable decision for the game, and every other row the queue holds for the same
|
||||
appid. A `rejected` row has **Reopen** there. It moves that one row back to
|
||||
appid. **Languages** lists what the game supports, as Steam names them, with a
|
||||
**No English** badge when English is not among them; such a game is decided
|
||||
like any other, and auto-approve treats it the same. Rows queued before v4.6.0
|
||||
show "—": discovery did not record languages then. A `rejected` row has **Reopen** there. It moves that one row back to
|
||||
`pending` and deletes the rejection, but only if no other row for the game is
|
||||
open or committed and the game has no approved decision; otherwise the dialog
|
||||
says which of those stopped it. There is no bulk reopen, on purpose. A game
|
||||
|
||||
@@ -667,6 +667,7 @@ workflow_dispatch: mode=backfill, start_offset=0, max_pages=9, budget=700
|
||||
4. **Playtest apps' `type` value.** `--strict-names` covers the case either way, but the frequency is unmeasured.
|
||||
5. **Region-locked `unavailable` rate from a GitHub runner IP.** This determines whether the optional negative cache is worth building. The first backfill run's counters measure it — that is why `Stats` breaks the drop reasons out individually rather than reporting one "skipped" number.
|
||||
6. **`supportedlang=english`.** Keeping it excludes 1,537 titles (8.4%) **[measured]**. Matching the existing dataset's character is the v1 default; it is a one-line knob if you later decide the tracker should be language-neutral.
|
||||
*Decided 2026-10-30, v4.6.0: language-neutral. The search sends no `supportedlang` (1,592 titles were excluded by then); see [19-all-languages.md](19-all-languages.md).*
|
||||
|
||||
### Files this design touches
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# Every language — games whose store page has no English
|
||||
|
||||
## Why this note exists
|
||||
|
||||
On 2026-10-30 the maintainer asked why a free Japanese game,
|
||||
[`//トロイメライ・ノクターン`](https://store.steampowered.com/app/5118700/)
|
||||
(5118700), was nowhere in the catalogue, and for an assessment of every
|
||||
workflow a game like it passes through, a plan, and the change. This is that
|
||||
assessment and the decisions it led to. Every number below was measured on
|
||||
the day with read-only requests to Steam and the repository's own data
|
||||
(`main` at 60b710a6, 12,854 games).
|
||||
|
||||
The short version: one parameter kept them out. Discovery searched Steam with
|
||||
`supportedlang=english`, a choice made when discovery was designed
|
||||
([11-design-discovery.md](11-design-discovery.md) §6, "a one-line knob if you
|
||||
later decide the tracker should be language-neutral"). Everything after
|
||||
discovery already handled these games: 275 of them were in the catalogue,
|
||||
added before discovery existed, and were displayed, reviewed, refreshed and
|
||||
purged like any other.
|
||||
|
||||
## The example game
|
||||
|
||||
| Question | Answer |
|
||||
| --- | --- |
|
||||
| Free, a game? | `is_free: true`, `type: game` (appdetails, US, JP and VN) |
|
||||
| Sold in the US? | Yes: GetItems (US) `visible`, free, 4 reviews, 100 % |
|
||||
| Languages | `Japanese` only |
|
||||
| Store text | An English description exists; the name is Japanese only |
|
||||
| In the repository? | No: not in `data/`, the queue, `removed_games.jsonl` or the blocklist |
|
||||
|
||||
## What Steam lists
|
||||
|
||||
Steam's search (`category1=998`, `maxprice=free`), walked to the end both ways:
|
||||
|
||||
| Search | Free games |
|
||||
| --- | --- |
|
||||
| `supportedlang=english` | 17,198 |
|
||||
| No language filter | 18,790 |
|
||||
| No filter, with `l=english` or `Accept-Language: ja` | 18,790 |
|
||||
|
||||
Every game of the first list is in the second, so the difference is exactly
|
||||
**1,592 games that do not list English**. Without the parameter Steam applies
|
||||
no language filter of its own. These counts were taken from Vietnam; search
|
||||
results depend on the country (GitHub's US runners saw 17,259 English-listing
|
||||
games the same morning).
|
||||
|
||||
Supported languages of those 1,592, from `supportedlang=english,<X>` minus the
|
||||
English count (a game can count under several):
|
||||
|
||||
| Language | Games |
|
||||
| --- | --- |
|
||||
| Simplified Chinese | 784 |
|
||||
| Japanese | 317 |
|
||||
| Korean | 165 |
|
||||
| Traditional Chinese | 164 |
|
||||
| French / Spanish / Russian / German / Portuguese (Brazil) | 51 / 45 / 40 / 32 / 31 |
|
||||
| Any other single language | 23 or fewer |
|
||||
| No language declared at all | 29 |
|
||||
|
||||
Of the 1,592, **275 were already in the catalogue**; the other **1,317** were
|
||||
not. By release year:
|
||||
|
||||
| ≤2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026 |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 62 | 63 | 70 | 83 | 139 | 175 | 229 | 270 | 226 |
|
||||
|
||||
The English backlog had been swept back to 2019 (the last Backfill Discovery
|
||||
run that morning was 2019 Q4), so every window from 2019 to 2026 needed one
|
||||
more pass; older windows get these games as they are swept for the first time.
|
||||
|
||||
## What each workflow does with such a game
|
||||
|
||||
| Workflow | Before this change | Change |
|
||||
| --- | --- | --- |
|
||||
| Discover New Games (daily, ≤5 newest pages) | Never sees it | Search every language |
|
||||
| Backfill Discovery (by year or quarter) | Never sees it | Same search; re-sweep 2019-2026 |
|
||||
| `/admin` queue, auto-approve | No language rule, but nothing tells a reviewer which languages a game supports | Candidates carry their languages; "No English" badge |
|
||||
| Ingest (`ingest_new.py`, `fetch_full`) | Any link form (percent-encoded slug, raw Japanese slug, `?l=`) becomes `/app/<id>/`. Name, description and developer keep their script; tags, genre, language names, review labels and dates are English, because every request asks Steam in English | None, apart from the two points below |
|
||||
| Update Reviews, anti-cheat, Peaks, Purge | Nothing depends on the language. Reviews are GetItems' `summary_filtered`, which counts every language, so a Japanese game with Japanese reviews has its full score | None |
|
||||
| Refresh Store Data | As above. A game that declares no language keeps empty `languages`, so the daily repair group takes it every day and the admin Quality page flags it | None; accepted for the 29 such games |
|
||||
| Web app | Names and descriptions display (CJK through the system's fonts); search finds exact substrings | Language filter, CJK-aware search, CSV for Excel |
|
||||
| Region | Every request goes out as the US (runners, `country_code: US`): a game not sold there is refused as `unavailable` | Out of scope |
|
||||
|
||||
Two smaller problems turned up in the pipeline:
|
||||
|
||||
- **Digits that are not ASCII.** Python's `\d` also matches full-width
|
||||
(`5118700`, what a Japanese keyboard types) and other scripts' digits.
|
||||
A queued link in such digits, which only a pull request could add, gave an
|
||||
appid that missed every comparison and made `rotation.slice_of()` raise on
|
||||
`.encode("ascii")`, stopping Refresh Store Data, Peaks and Refetch for every
|
||||
game. The Worker and `web/shared/removal.ts` already took ASCII only;
|
||||
`scripts/core/blocklist.py` did not, and its `$` also let `"730\n"` through.
|
||||
- **Cutting a CJK blurb.** `truncate_description()` backed off only to an
|
||||
ASCII space and trimmed only Latin punctuation before the ellipsis, so a
|
||||
Chinese or Japanese blurb was cut mid-phrase, and two stored ones end in
|
||||
`。…`.
|
||||
|
||||
And in the web app:
|
||||
|
||||
- `/games` had no language filter, and `languages` is not a search key, so a
|
||||
reader of English had no way to leave out the games they cannot play.
|
||||
- Every search box only lowercased, so `トロイ` (half-width), `とろい` (hiragana)
|
||||
and `トロイ` were three different searches, as were `4` and `4`.
|
||||
- The header search jumped to `/games` on the first keystroke an input method
|
||||
had not finished composing.
|
||||
- The command palette waited for two characters; one kanji found nothing.
|
||||
- The CSV download had no byte order mark, so Excel, Japanese Windows
|
||||
included, opened every CJK name as mojibake.
|
||||
|
||||
## What changed (v4.6.0)
|
||||
|
||||
- **Discovery searches every language.** `fetch_search_page()` sends no
|
||||
`supportedlang`, and sends `l=english`, which fixes the language of the row
|
||||
text (discovery skips "Coming soon" rows, the backfill reads English month
|
||||
names) without changing which rows come back.
|
||||
- **Candidates carry their languages.** `appdetails_languages()` reads
|
||||
appdetails' `supported_languages` HTML into a list in the payload; dry runs
|
||||
print "no English: …" and each run counts the candidates without English.
|
||||
`/admin`'s candidate dialog shows the list and a "No English" badge.
|
||||
Auto-approve is unchanged: the maintainer chose to treat these games like
|
||||
any other.
|
||||
- **The `discovery` group queues** (`queue: max` in both workflows), and the
|
||||
backfill takes a `max_pages` input. `pipeline.test.ts` now holds every
|
||||
shared group to one queue setting.
|
||||
- **ASCII appids.** `extract_appid` refuses a run of digits with a non-ASCII
|
||||
one in it; `normalize_link()` reads NFKC first, so `5118700` is
|
||||
5118700; the blocklist uses ASCII digits and `fullmatch`.
|
||||
- **CJK blurbs** are cut just after the last `。!?` when that is later than
|
||||
the last space and in the last 40 %, and `、。,.:;` are trimmed before
|
||||
the ellipsis. Stored descriptions (all within the cap) never change;
|
||||
`_fuller_cut()` lets Refresh Store Data write a re-cut without counting it
|
||||
as news.
|
||||
- **Web:** a language filter on `/games` (`?lang=`, counted like the others),
|
||||
`fold()` (NFKC plus hiragana to katakana) on both sides of every search box,
|
||||
the header search waiting for the input method, one CJK character enough in
|
||||
the palette, and a byte order mark on CSV downloads. The About page says
|
||||
the catalogue covers every language.
|
||||
|
||||
The legal documents were not edited. DISCLAIMER §2 and ToS item 2 say the
|
||||
pipeline asks Steam in English and parses English answers, which is still
|
||||
true; an edit would change their hash and reopen the consent gate for
|
||||
everyone. No filter is stored under an `f2p:` key, so the privacy policy is
|
||||
unchanged too.
|
||||
|
||||
## The re-sweep
|
||||
|
||||
After the merge, Backfill Discovery was dispatched once per year from 2019 to
|
||||
2026, `limit=1000`, `max_pages=60` (a whole recent year is about 30 pages).
|
||||
A window counts as done only when its log says "walked past the window"; a
|
||||
run that reaches its limit is run again. Windows before 2019 get their
|
||||
non-English games when they are first swept.
|
||||
|
||||
## Found on the way: genre "Free To Play"
|
||||
|
||||
`SKIP_GENRE_TAGS` held `"Free to Play"`, but appdetails writes `"Free To
|
||||
Play"` and the comparison was case-sensitive, and the fallback took the first
|
||||
name anyway: 1,357 of 12,854 games had it as their genre. Not a language
|
||||
problem, but fixed in the same release:
|
||||
|
||||
- `fetcher.pick_genre()` compares casefolded and never falls back to a price
|
||||
model or a release state (`NOT_GENRES`); with nothing else it says
|
||||
"Uncategorized".
|
||||
- Refresh Store Data replaces a stored "Free To Play" from appdetails, 150
|
||||
games a day, first after the page repairs, never under an override. `genre`
|
||||
is a human-judgement field, and this is its one scheduled writer: no person
|
||||
chose "Free To Play".
|
||||
|
||||
## Left for later
|
||||
- **Games not sold in the US.** Every request goes out as the US; covering
|
||||
region-locked games needs another country or runner.
|
||||
- **CJK fonts and `lang`.** CJK text renders in the system's font (a Linux
|
||||
machine without one shows boxes), and no element marks a game's name with
|
||||
its language, so shared Han characters may take Chinese shapes in Japanese
|
||||
names. A self-hosted CJK font is megabytes; guessing a name's language from
|
||||
its script would be wrong for many Han-only names.
|
||||
@@ -23,10 +23,12 @@ appid. That is the reason for windowing at all -- ~12,000 unknown appids at the
|
||||
store throttle is several hours, far past a job timeout. One quarter is a
|
||||
sitting most of the way through.
|
||||
|
||||
Language filter: fetch_search_page pins supportedlang=english, which is known to
|
||||
exclude ~1,537 titles. That is deliberately left as-is so this sweep and the
|
||||
daily one agree on what the catalogue is supposed to contain; widening it is a
|
||||
separate decision, not a backfill detail.
|
||||
Languages: since v4.6.0 fetch_search_page lists free games in every language,
|
||||
for this sweep and the daily one alike. Until then it pinned
|
||||
supportedlang=english, which left out 1,592 titles (2026-10-30), most of them
|
||||
Chinese, Japanese or Korean only - so every window swept before v4.6.0 needs
|
||||
one more pass to find them. Each candidate carries its languages, and a dry
|
||||
run says which ones list no English.
|
||||
|
||||
Usage:
|
||||
python scripts/backfill_discover.py --year 2025 --quarter 3 --dry-run
|
||||
@@ -41,8 +43,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.steam_client import get_client
|
||||
from discover_new import (
|
||||
PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, parse_search_rows,
|
||||
safe_stdout, worker_get, worker_post,
|
||||
PAGE_SIZE, POST_BATCH, catalogue_appids, enrich, language_note,
|
||||
parse_search_rows, safe_stdout, without_english, worker_get, worker_post,
|
||||
)
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(
|
||||
@@ -242,12 +244,15 @@ def main():
|
||||
client.close()
|
||||
|
||||
print(f"\ncandidates: {len(candidates)}")
|
||||
if candidates:
|
||||
print(f" of which without English: {without_english(candidates)}")
|
||||
if rejected:
|
||||
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
|
||||
|
||||
if args.dry_run:
|
||||
for c in candidates[:20]:
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
|
||||
note = language_note(c.payload["languages"])
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
|
||||
if len(candidates) > 20:
|
||||
print(f" ... and {len(candidates) - 20} more")
|
||||
return 0
|
||||
|
||||
@@ -51,10 +51,13 @@ ATTRIBUTION = "admin"
|
||||
|
||||
STATUS_CODE = "admin_removed"
|
||||
|
||||
_APPID = re.compile(r"^\d{1,10}$")
|
||||
# ASCII digits and the whole string (fullmatch): Python's digit class also
|
||||
# takes full-width and other scripts' digits, and `$` a trailing newline, all
|
||||
# of which web/shared/removal.ts refuses. One line must mean one thing to both.
|
||||
_APPID = re.compile(r"\d{1,10}", re.ASCII)
|
||||
# now_iso()'s shape exactly. dedup_removed() compares removed_at as a string,
|
||||
# so a millisecond suffix would sort differently from the pipeline's own rows.
|
||||
_ISO_SECONDS = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
|
||||
_ISO_SECONDS = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", re.ASCII)
|
||||
|
||||
|
||||
class BlocklistError(RuntimeError):
|
||||
@@ -70,10 +73,10 @@ def entry_problem(entry: object) -> Optional[str]:
|
||||
if not isinstance(entry, dict):
|
||||
return "not an object"
|
||||
appid = entry.get("appid")
|
||||
if not isinstance(appid, str) or not _APPID.match(appid):
|
||||
if not isinstance(appid, str) or not _APPID.fullmatch(appid):
|
||||
return "appid must be 1-10 digits, as a string"
|
||||
blocked_at = entry.get("blocked_at")
|
||||
if not isinstance(blocked_at, str) or not _ISO_SECONDS.match(blocked_at):
|
||||
if not isinstance(blocked_at, str) or not _ISO_SECONDS.fullmatch(blocked_at):
|
||||
# Without a fixed timestamp the log line would be stamped anew on every
|
||||
# write and rewritten forever.
|
||||
return "blocked_at must look like 2026-10-13T12:00:00Z"
|
||||
|
||||
@@ -47,6 +47,11 @@ SKIP_GENRE_TAGS = frozenset({
|
||||
"Cross-Platform Multiplayer", "In-App Purchases",
|
||||
"Massively Multiplayer", "Competitive",
|
||||
})
|
||||
# Not genres at all: a price model and a release state. Every game here is
|
||||
# free, so "Free To Play" tells a reader nothing. Both are on the skip list
|
||||
# above; this is what a genre may never FALL BACK to (fetcher.pick_genre), and
|
||||
# what Refresh Store Data replaces when it finds one stored.
|
||||
NOT_GENRES = frozenset({"Free to Play", "Early Access"})
|
||||
|
||||
# ──────────── Anti-Cheat patterns ────────────
|
||||
# A key or a pattern matches a name only as whole words (core/anticheat.py
|
||||
|
||||
@@ -11,6 +11,7 @@ import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
@@ -30,16 +31,27 @@ _EMPTY_STRINGS = frozenset({"", "N/A", "-", "?"})
|
||||
|
||||
|
||||
def extract_appid(link: str) -> Optional[str]:
|
||||
"""Extract appid from any Steam URL or bare number. O(1) regex."""
|
||||
"""Extract appid from any Steam URL or bare number. O(1) regex.
|
||||
|
||||
ASCII digits only. The regex digit class also takes full-width (5) and
|
||||
other scripts' digits; such an id would miss every comparison, and
|
||||
rotation.slice_of() encodes it as ASCII, which would stop every refresh.
|
||||
The whole run of digits is read first, so "/app/730٤" is refused rather
|
||||
than read as 730.
|
||||
"""
|
||||
if not link:
|
||||
return None
|
||||
m = _APPID_RE.search(link)
|
||||
return m.group(1) if m else None
|
||||
return m.group(1) if m and m.group(1).isascii() else None
|
||||
|
||||
|
||||
def normalize_link(raw: str) -> Optional[str]:
|
||||
raw = raw.strip().rstrip("/")
|
||||
if raw.isdigit():
|
||||
"""The canonical store link for a queued line: a store URL in any form
|
||||
(slug, query string, no scheme) or a bare appid. NFKC first, so an appid
|
||||
typed in full-width digits, a Japanese keyboard's default, is read as
|
||||
ASCII; any other kind of digit is refused."""
|
||||
raw = unicodedata.normalize("NFKC", raw).strip().rstrip("/")
|
||||
if raw.isascii() and raw.isdigit():
|
||||
return f"https://store.steampowered.com/app/{raw}/"
|
||||
appid = extract_appid(raw)
|
||||
return f"https://store.steampowered.com/app/{appid}/" if appid else None
|
||||
@@ -221,6 +233,14 @@ def load_main() -> list[dict]:
|
||||
ELLIPSIS = "\u2026"
|
||||
|
||||
|
||||
# Sentence ends a cut may back off to, as to a space (。!?).
|
||||
_SENTENCE_ENDS = "\u3002\uff01\uff1f"
|
||||
# Trimmed before the ellipsis: the Latin set, then the CJK comma, full stop,
|
||||
# full-width comma, full-width full stop, colon and semicolon (、。,.:;).
|
||||
# Like ! and ?, the CJK ! and ? stay.
|
||||
_TRAILING = " ,;:.\u2014\u2013-\u3001\u3002\uff0c\uff0e\uff1a\uff1b"
|
||||
|
||||
|
||||
def truncate_description(text: str) -> str:
|
||||
"""Shorten one blurb to DESCRIPTION_MAX, on a word boundary where possible.
|
||||
|
||||
@@ -229,18 +249,26 @@ def truncate_description(text: str) -> str:
|
||||
single write to data/ -- if it were not idempotent it would nibble a
|
||||
character off every description on every run and rewrite all five shards
|
||||
forever.
|
||||
|
||||
Only a blurb longer than the cap is cut, so every stored description
|
||||
(all <= DESCRIPTION_MAX) comes back unchanged whatever the rule: a new
|
||||
rule reaches only blurbs read fresh from Steam.
|
||||
"""
|
||||
t = (text or "").strip()
|
||||
if len(t) <= DESCRIPTION_MAX:
|
||||
return t
|
||||
# -1 leaves room for the ellipsis, so the STORED value never exceeds the cap.
|
||||
cut = t[:DESCRIPTION_MAX - 1]
|
||||
# A word boundary, or just after a CJK sentence end: Chinese and Japanese
|
||||
# blurbs have no spaces to back off to. Whichever comes later.
|
||||
space = cut.rfind(" ")
|
||||
# Only back off to a word boundary if one is reasonably near the end;
|
||||
sentence = max(cut.rfind(c) for c in _SENTENCE_ENDS) + 1
|
||||
boundary = max(space, sentence)
|
||||
# Only back off to a boundary if one is reasonably near the end;
|
||||
# otherwise a blurb with no spaces in its tail would lose most of itself.
|
||||
if space >= int(DESCRIPTION_MAX * 0.6):
|
||||
cut = cut[:space]
|
||||
return cut.rstrip(" ,;:.\u2014\u2013-") + ELLIPSIS
|
||||
if boundary >= int(DESCRIPTION_MAX * 0.6):
|
||||
cut = cut[:boundary]
|
||||
return cut.rstrip(_TRAILING) + ELLIPSIS
|
||||
|
||||
|
||||
def truncate_descriptions(records: list[dict]) -> int:
|
||||
|
||||
+35
-4
@@ -14,7 +14,7 @@ from typing import Optional
|
||||
from .anticheat import apply_anti_cheat, from_categories
|
||||
from .budget import Deadline, report
|
||||
from .constants import (
|
||||
STEAM_API_KEY, SKIP_GENRE_TAGS,
|
||||
STEAM_API_KEY, SKIP_GENRE_TAGS, NOT_GENRES,
|
||||
BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX,
|
||||
)
|
||||
from .steam_client import get_client
|
||||
@@ -58,6 +58,39 @@ def _infer_type_game(categories: list[dict]) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
# ──────────── Genre ────────────
|
||||
|
||||
_SKIP_GENRES = frozenset(t.casefold() for t in SKIP_GENRE_TAGS)
|
||||
_NOT_GENRES = frozenset(t.casefold() for t in NOT_GENRES)
|
||||
|
||||
|
||||
def is_stray_genre(genre) -> bool:
|
||||
"""A stored genre that is not a genre ("Free To Play"): the scraper's
|
||||
leftover, never a choice a person made."""
|
||||
return isinstance(genre, str) and genre.strip().casefold() in _NOT_GENRES
|
||||
|
||||
|
||||
def pick_genre(genres: list) -> str:
|
||||
"""The genre for appdetails' `genres` names: the first not on the skip
|
||||
list, else the first that is a genre at all, else "Uncategorized".
|
||||
|
||||
Case-insensitive. appdetails writes "Free To Play" and the skip list said
|
||||
"Free to Play"; the exact match let 1,357 games be stored with it as their
|
||||
genre (2026-10-30), and the fallback to the first name could pick it too.
|
||||
"""
|
||||
names = [g.strip() for g in genres if isinstance(g, str) and g.strip()]
|
||||
kept = [g for g in names if g.casefold() not in _SKIP_GENRES]
|
||||
if kept:
|
||||
return kept[0]
|
||||
real = [g for g in names if g.casefold() not in _NOT_GENRES]
|
||||
return real[0] if real else "Uncategorized"
|
||||
|
||||
|
||||
def appdetails_genres(data: dict) -> list:
|
||||
"""The genre names appdetails lists, in its order."""
|
||||
return [g.get("description", "") for g in data.get("genres") or [] if isinstance(g, dict)]
|
||||
|
||||
|
||||
# ──────────── Apply functions ────────────
|
||||
|
||||
def apply_details(game: dict, data: dict) -> dict:
|
||||
@@ -77,9 +110,7 @@ def apply_details(game: dict, data: dict) -> dict:
|
||||
game["header_image"] = img
|
||||
|
||||
if is_empty(game.get("genre")):
|
||||
raw = [g["description"] for g in data.get("genres", [])]
|
||||
filt = [g for g in raw if g not in SKIP_GENRE_TAGS]
|
||||
game["genre"] = filt[0] if filt else (raw[0] if raw else "Uncategorized")
|
||||
game["genre"] = pick_genre(appdetails_genres(data))
|
||||
|
||||
if is_empty(game.get("developer")):
|
||||
devs = data.get("developers", [])
|
||||
|
||||
@@ -314,8 +314,8 @@ class SteamClient:
|
||||
return self._tag_names
|
||||
|
||||
def fetch_search_page(self, start: int, count: int = 100) -> Optional[dict]:
|
||||
"""Store search: free games, newest first. Returns the parsed JSON
|
||||
({'success', 'results_html', 'total_count', 'start'}) or None.
|
||||
"""Store search: free games in every language, newest first. Returns the
|
||||
parsed JSON ({'success', 'results_html', 'total_count', 'start'}) or None.
|
||||
|
||||
This is the ONLY discovery-capable method on this client - every other
|
||||
one needs an appid you already have. It deliberately shares
|
||||
@@ -325,6 +325,17 @@ class SteamClient:
|
||||
Measured: the server caps page size at 100 regardless of `count`, and a
|
||||
`start` past the end returns zero rows rather than an error, so a paging
|
||||
loop terminates naturally.
|
||||
|
||||
No `supportedlang`. Until v4.6.0 it was pinned to english, which left out
|
||||
every game that does not list English: 1,592 of 18,790 free games on
|
||||
2026-10-30, most of them Chinese, Japanese or Korean only. Without it
|
||||
Steam filters on no language at all (the same 18,790 with `l=english`
|
||||
and with an `Accept-Language: ja` header).
|
||||
|
||||
`l=english` sets the language of the row TEXT, never which rows come
|
||||
back: discovery skips a row whose release reads "Coming soon", and the
|
||||
backfill reads English month names. Before, only the session's
|
||||
Accept-Language header kept that text English.
|
||||
"""
|
||||
resp = self._get(
|
||||
"https://store.steampowered.com/search/results/",
|
||||
@@ -334,9 +345,9 @@ class SteamClient:
|
||||
"count": count,
|
||||
"maxprice": "free",
|
||||
"category1": 998, # "Games" - excludes DLC/soundtracks/videos/software
|
||||
"supportedlang": "english",
|
||||
"sort_by": "Released_DESC",
|
||||
"infinite": 1,
|
||||
"l": "english",
|
||||
},
|
||||
throttle_fn=self._throttle_store,
|
||||
timeout=20,
|
||||
|
||||
+47
-4
@@ -32,13 +32,17 @@ from core.steam_client import get_client
|
||||
|
||||
# Steam caps the search page at 100 rows regardless of what we ask for.
|
||||
PAGE_SIZE = 100
|
||||
# A page spans roughly 11 days of releases at the current rate (~8-9/day), so
|
||||
# five pages is about eight weeks of insurance. If the cron silently stops for a
|
||||
# month, the next successful run still catches everything with no intervention.
|
||||
# A page spans about a week of releases (13 a day in October 2026, every
|
||||
# language), so five pages is over a month of insurance. If the cron silently
|
||||
# stops for a few weeks, the next successful run still catches everything with
|
||||
# no intervention.
|
||||
DEFAULT_MAX_PAGES = 5
|
||||
DEFAULT_STOP_AFTER_KNOWN = 2
|
||||
# The Worker's own cap; batches larger than this are rejected.
|
||||
POST_BATCH = 100
|
||||
# Steam knows about 30 languages; the cap only bounds what a publisher's
|
||||
# string can put into the payload.
|
||||
MAX_LANGUAGES = 40
|
||||
|
||||
_RE_APPID = re.compile(r'data-ds-appid="([^"]+)"')
|
||||
_RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
|
||||
@@ -48,6 +52,7 @@ _RE_TITLE = re.compile(r'<span class="title">(.*?)</span>', re.S)
|
||||
# "Coming soon" skip below silently never fired.
|
||||
_RE_RELEASED = re.compile(r'<div class="search_released[^"]*">(.*?)</div>', re.S)
|
||||
_RE_TAGS = re.compile(r"<[^>]+>")
|
||||
_RE_BR = re.compile(r"<br\s*/?>", re.I)
|
||||
|
||||
# Types that are not games. `category1=998` already filters most of these out,
|
||||
# but appdetails is the authoritative source and occasionally disagrees.
|
||||
@@ -73,6 +78,9 @@ class Candidate:
|
||||
"platforms": data.get("platforms") or {},
|
||||
"developers": (data.get("developers") or [])[:6],
|
||||
"publishers": (data.get("publishers") or [])[:6],
|
||||
# Discovery proposes games in every language; this is what tells a
|
||||
# reviewer which ones a reader of English cannot play.
|
||||
"languages": appdetails_languages(data.get("supported_languages")),
|
||||
}
|
||||
|
||||
def to_wire(self):
|
||||
@@ -162,6 +170,38 @@ def parse_search_rows(results_html):
|
||||
return rows
|
||||
|
||||
|
||||
def appdetails_languages(text):
|
||||
"""The language names in appdetails' `supported_languages`, in its order.
|
||||
|
||||
Steam sends one HTML string, named in the request's language (English
|
||||
here): "English<strong>*</strong>, Japanese<br><strong>*</strong>languages
|
||||
with full audio support". The asterisks mark full audio and the line break
|
||||
starts that footnote. A game that declares no language has no key at all:
|
||||
an empty list, which is not the same thing as "no English".
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
return []
|
||||
first = _RE_BR.split(text, maxsplit=1)[0]
|
||||
plain = html.unescape(_RE_TAGS.sub("", first)).replace("*", "")
|
||||
return [n.strip() for n in plain.split(",") if n.strip()][:MAX_LANGUAGES]
|
||||
|
||||
|
||||
def language_note(languages):
|
||||
"""What a dry run says about a candidate's languages: nothing for a game
|
||||
that lists English."""
|
||||
if "English" in languages:
|
||||
return ""
|
||||
if not languages:
|
||||
return "no languages declared"
|
||||
more = ", ..." if len(languages) > 3 else ""
|
||||
return "no English: " + ", ".join(languages[:3]) + more
|
||||
|
||||
|
||||
def without_english(candidates):
|
||||
"""How many candidates do not list English, for the run's summary."""
|
||||
return sum(1 for c in candidates if "English" not in c.payload["languages"])
|
||||
|
||||
|
||||
def catalogue_appids():
|
||||
"""Appids already in data/, plus the ones the maintainer removed.
|
||||
|
||||
@@ -395,12 +435,15 @@ def main():
|
||||
client.close()
|
||||
|
||||
print(f"\ncandidates: {len(candidates)}")
|
||||
if candidates:
|
||||
print(f" of which without English: {without_english(candidates)}")
|
||||
if rejected:
|
||||
print("rejected:", ", ".join(f"{k}={v}" for k, v in sorted(rejected.items())))
|
||||
|
||||
if args.dry_run:
|
||||
for c in candidates[:20]:
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]}")
|
||||
note = language_note(c.payload["languages"])
|
||||
print(f" {c.appid:>8} {c.release_date:<14} {c.name[:50]} {note}".rstrip())
|
||||
if len(candidates) > 20:
|
||||
print(f" ... and {len(candidates) - 20} more")
|
||||
return 0
|
||||
|
||||
@@ -41,7 +41,12 @@ The rules, because these fields were fill-if-empty until now:
|
||||
- A value is only ever replaced by a NON-EMPTY one. A failed request, an age
|
||||
gate, a delisted app or a changed page layout leaves the stored value alone.
|
||||
- MANUAL_FIELDS are never written (genre, type_game, anti_cheat, safe, ...),
|
||||
and neither is `notes`: health checks and "no longer free" belong to
|
||||
with one exception: a stored genre that is not a genre ("Free To Play",
|
||||
left by a case-sensitive skip list until v4.6.0) is replaced by the one
|
||||
appdetails gives (fetcher.pick_genre), unless an override owns the field.
|
||||
No person chose it. Such games are in the details group every day, first
|
||||
after the page repairs, at most GENRE_DAILY_CAP a day.
|
||||
Nor is `notes` written: health checks and "no longer free" belong to
|
||||
purge_unhealthy.py and mark_dead_games.py. A game appdetails no longer
|
||||
calls free is only listed, in a ::warning:: - purge removes a game only
|
||||
when Steam's store API flags it too, and this is where the opposite case
|
||||
@@ -78,11 +83,14 @@ from core.data_store import (
|
||||
ELLIPSIS, extract_appid, is_empty, load_main, migrate_record, now_iso, save_main,
|
||||
truncate_description,
|
||||
)
|
||||
from core.fetcher import process_batch, scrape_store_fields
|
||||
from core.fetcher import (
|
||||
appdetails_genres, is_stray_genre, pick_genre, process_batch, scrape_store_fields,
|
||||
)
|
||||
from core.health_checker import free_per_appdetails
|
||||
from core.normalize import (
|
||||
clean_list, clean_text, decode_entities, normalize_records, normalize_release_date,
|
||||
)
|
||||
from core.overrides import OverrideGuard
|
||||
# The rotation (crc32 slices) is shared with refresh_peaks.py.
|
||||
from core.rotation import cycle_for, daily_order, day_number, due
|
||||
from core.steam_client import get_client
|
||||
@@ -97,6 +105,10 @@ DETAILS_DAILY = 500
|
||||
# 2026-10-10; the cap is for the day something breaks for every game at once,
|
||||
# so a two-hour job cannot turn into a whole-catalogue crawl.
|
||||
MISSING_DAILY_CAP = 150
|
||||
# Games with a stray genre ("Free To Play") whose appdetails are read a day on
|
||||
# top of the rotation: 1,357 on 2026-10-30, so about nine days, then only the
|
||||
# odd one a new rule lets through.
|
||||
GENRE_DAILY_CAP = 150
|
||||
|
||||
PAGE_FIELDS = ("tags", "languages", "language_details", "has_paid_dlc")
|
||||
DETAILS_FIELDS = (
|
||||
@@ -194,32 +206,49 @@ def details_values(data: dict) -> dict:
|
||||
|
||||
|
||||
def _fuller_cut(stored, fresh: str) -> bool:
|
||||
"""Whether `fresh` is the stored blurb, only cut off later.
|
||||
"""Whether `fresh` is the stored blurb, only cut off somewhere else.
|
||||
|
||||
A blurb stored HTML-escaped was truncated while """ still took six
|
||||
characters; decoded, the same blurb fits further under DESCRIPTION_MAX.
|
||||
That is not news about the game, so it is written without counting as a
|
||||
change. Only a blurb that grew from the stored one passes; one the
|
||||
And since v4.6.0 a CJK blurb is cut just after a sentence end where it can
|
||||
be, a little EARLIER than the hard cut it was stored with. Neither is news
|
||||
about the game, so it is written without counting as a change. Only two
|
||||
cuts of one text pass - one stem a prefix of the other; a blurb the
|
||||
publisher rewrote is still a change.
|
||||
"""
|
||||
if not isinstance(stored, str) or not stored.endswith(ELLIPSIS):
|
||||
return False
|
||||
stem = stored[:-1].rstrip()
|
||||
return bool(stem) and len(fresh) > len(stored) and fresh.startswith(stem)
|
||||
if not stem:
|
||||
return False
|
||||
if len(fresh) > len(stored):
|
||||
return fresh.startswith(stem)
|
||||
fresh_stem = fresh[:-1].rstrip() if fresh.endswith(ELLIPSIS) else ""
|
||||
return bool(fresh_stem) and fresh != stored and stem.startswith(fresh_stem)
|
||||
|
||||
|
||||
def apply_app_details(game: dict, data: dict) -> list[str]:
|
||||
def apply_app_details(game: dict, data: dict, guard: Optional[OverrideGuard] = None) -> list[str]:
|
||||
changed: list[str] = []
|
||||
for field, value in details_values(data).items():
|
||||
if field == "description" and _fuller_cut(game.get("description"), value):
|
||||
game["description"] = value
|
||||
continue
|
||||
_replace(game, field, value, changed)
|
||||
# The one manual field this job writes, and only a value no person chose:
|
||||
# a genre that is not a genre. Left alone when appdetails names no genre,
|
||||
# and when an override owns the field.
|
||||
genres = appdetails_genres(data)
|
||||
if is_stray_genre(game.get("genre")) and genres:
|
||||
genre = pick_genre(genres)
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if guard is None or guard.allows(appid, "genre", genre):
|
||||
_replace(game, "genre", genre, changed)
|
||||
return changed
|
||||
|
||||
|
||||
def refresh_game(game: dict, client, page: bool, details: bool,
|
||||
not_free: Optional[list] = None) -> list[str]:
|
||||
not_free: Optional[list] = None,
|
||||
guard: Optional[OverrideGuard] = None) -> list[str]:
|
||||
"""Refresh one game in place. Returns the fields that changed. A game
|
||||
whose appdetails no longer call it free is added to `not_free`."""
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
@@ -230,7 +259,7 @@ def refresh_game(game: dict, client, page: bool, details: bool,
|
||||
if details:
|
||||
data = client.fetch_app_details(appid)
|
||||
if data:
|
||||
changed += apply_app_details(game, data)
|
||||
changed += apply_app_details(game, data, guard)
|
||||
if not_free is not None and not free_per_appdetails(data):
|
||||
not_free.append(f"{appid} {game.get('name', '')[:40]}")
|
||||
if page:
|
||||
@@ -254,23 +283,34 @@ def build_plan(games: list[dict], day: int, want_page: bool = True,
|
||||
page_cycle: int = PAGE_CYCLE_DAYS,
|
||||
details_cycle: int = DETAILS_CYCLE_DAYS) -> list[tuple]:
|
||||
"""[(game, page, details)] for one day: the games with empty store fields
|
||||
first (page group, every day), then the day's rotation slices.
|
||||
first (page group, every day), then the games with a stray genre
|
||||
(appdetails, every day), then the day's rotation slices.
|
||||
|
||||
Both parts come in an order that changes daily (rotation.daily_order):
|
||||
the repairs are capped at MISSING_DAILY_CAP, so a residue Steam never
|
||||
fills cannot hold the same places forever, and a run that meets its time
|
||||
budget leaves a different part of the slice for the next one.
|
||||
missing_only keeps just the repairs, store page only.
|
||||
All of it comes in an order that changes daily (rotation.daily_order):
|
||||
the repairs are capped (MISSING_DAILY_CAP, GENRE_DAILY_CAP), so a residue
|
||||
Steam never fills cannot hold the same places forever, and a run that
|
||||
meets its time budget leaves a different part of the slice for the next one.
|
||||
missing_only keeps just the store-field repairs, store page only.
|
||||
"""
|
||||
order = daily_order(games, day)
|
||||
repairs: list[dict] = []
|
||||
if want_page:
|
||||
repairs = daily_order([g for g in games if missing_store_fields(g)], day)[:MISSING_DAILY_CAP]
|
||||
repairs = [g for g in order if missing_store_fields(g)][:MISSING_DAILY_CAP]
|
||||
if missing_only:
|
||||
return [(g, True, False) for g in repairs]
|
||||
|
||||
plan = [(g, True, want_details and due(g, details_cycle, day)) for g in repairs]
|
||||
strays: list[dict] = []
|
||||
if want_details:
|
||||
strays = [g for g in order if is_stray_genre(g.get("genre"))][:GENRE_DAILY_CAP]
|
||||
stray_ids = {id(g) for g in strays}
|
||||
plan = [(g, True, want_details and (due(g, details_cycle, day) or id(g) in stray_ids))
|
||||
for g in repairs]
|
||||
taken = {id(g) for g in repairs}
|
||||
for g in daily_order(games, day):
|
||||
for g in strays:
|
||||
if id(g) not in taken:
|
||||
plan.append((g, want_page and due(g, page_cycle, day), True))
|
||||
taken.add(id(g))
|
||||
for g in order:
|
||||
if id(g) in taken:
|
||||
continue
|
||||
page = want_page and due(g, page_cycle, day)
|
||||
@@ -314,13 +354,16 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
n_page = sum(1 for _, p, _ in plan if p)
|
||||
n_details = sum(1 for _, _, d in plan if d)
|
||||
n_missing = sum(1 for g, p, _ in plan if p and missing_store_fields(g))
|
||||
n_stray = sum(1 for g, _, d in plan if d and is_stray_genre(g.get("genre")))
|
||||
print(f"{len(games)} games; day {day}: store page for {n_page} "
|
||||
f"(slice {day % page_cycle}/{page_cycle}, {n_missing} with empty store fields), "
|
||||
f"appdetails for {n_details} (slice {day % details_cycle}/{details_cycle})")
|
||||
f"appdetails for {n_details} (slice {day % details_cycle}/{details_cycle}, "
|
||||
f"{n_stray} with a genre that is not one)")
|
||||
if not plan:
|
||||
return
|
||||
|
||||
client = get_client()
|
||||
guard = OverrideGuard.load()
|
||||
flags = {id(g): (page, details) for g, page, details in plan}
|
||||
fields: Counter = Counter()
|
||||
not_free: list[str] = []
|
||||
@@ -329,7 +372,7 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
def refresh(game: dict) -> None:
|
||||
nonlocal touched
|
||||
page, details = flags[id(game)]
|
||||
changed = refresh_game(game, client, page=page, details=details, not_free=not_free)
|
||||
changed = refresh_game(game, client, page=page, details=details, not_free=not_free, guard=guard)
|
||||
if changed:
|
||||
touched += 1
|
||||
fields.update(changed)
|
||||
|
||||
@@ -13,7 +13,7 @@ import os
|
||||
import pytest
|
||||
|
||||
from core import blocklist, data_store
|
||||
from core.blocklist import BlocklistError, load_blocklist, make_entry, serialize_entry
|
||||
from core.blocklist import BlocklistError, entry_problem, load_blocklist, make_entry, serialize_entry
|
||||
|
||||
GOLDEN = os.path.join(os.path.dirname(__file__), "fixtures", "blocklist_golden.jsonl")
|
||||
|
||||
@@ -147,6 +147,22 @@ def test_damaged_and_incomplete_lines_block_nothing(workdir, capsys):
|
||||
assert "ignored" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("appid, blocked_at", [
|
||||
("5118700", "2026-10-13T12:00:00Z"), # full-width digits
|
||||
("٥١١٨٧٠٠", "2026-10-13T12:00:00Z"), # Arabic-Indic digits
|
||||
("730\n", "2026-10-13T12:00:00Z"), # `$` used to allow this
|
||||
("730", "2026-10-13T12:00:00Z"),
|
||||
("730", "2026-10-13T12:00:00Z\n"),
|
||||
])
|
||||
def test_a_line_removal_ts_would_refuse_blocks_nothing(appid, blocked_at):
|
||||
# web/shared/removal.ts reads these with JavaScript's ASCII-only digits
|
||||
# and an anchored `$`; both sides must refuse the same lines.
|
||||
entry = {"appid": appid, "link": "", "name": "", "reason": "not_free", "note": "",
|
||||
"blocked_at": blocked_at, "by": "admin"}
|
||||
assert entry_problem(entry) is not None
|
||||
assert entry_problem({**entry, "appid": "730", "blocked_at": "2026-10-13T12:00:00Z"}) is None
|
||||
|
||||
|
||||
def test_blocking_again_later_logs_the_new_removal(workdir):
|
||||
_block(_entry("2", at="2026-10-13T12:00:00Z"))
|
||||
data_store.save_main([_game("1"), _game("2")], apply_human_overrides=False)
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
"""How discover_new reads the redirect Cloudflare Access answers with."""
|
||||
"""How discover_new reads the redirect Cloudflare Access answers with, the
|
||||
search rows, and the languages a candidate carries to the review queue."""
|
||||
import pytest
|
||||
|
||||
import discover_new
|
||||
|
||||
|
||||
@@ -72,3 +75,50 @@ def test_appids_outside_anchors_are_still_recovered():
|
||||
def test_a_page_cut_off_inside_a_tag_keeps_the_rows_before_it():
|
||||
html = '<a data-ds-appid="570"><span class="title">Dota 2</span></a><a data-ds-appid="730"'
|
||||
assert discover_new.parse_search_rows(html) == [("570", "Dota 2", "")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, expected", [
|
||||
# 5118700 as appdetails answered it on 2026-10-30.
|
||||
("Japanese", ["Japanese"]),
|
||||
("English<strong>*</strong>, Simplified Chinese<strong>*</strong>, Japanese"
|
||||
"<br><strong>*</strong>languages with full audio support",
|
||||
["English", "Simplified Chinese", "Japanese"]),
|
||||
("Korean<br/><strong>*</strong>languages with full audio support", ["Korean"]),
|
||||
("Spanish - Spain, Portuguese - Brazil<BR />footnote", ["Spanish - Spain", "Portuguese - Brazil"]),
|
||||
("Français , , Deutsch", ["Français", "Deutsch"]),
|
||||
("", []),
|
||||
(None, []),
|
||||
(["English"], []), # not the shape Steam sends
|
||||
])
|
||||
def test_appdetails_languages(text, expected):
|
||||
assert discover_new.appdetails_languages(text) == expected
|
||||
|
||||
|
||||
def test_appdetails_languages_are_capped():
|
||||
text = ", ".join(f"Lang{i}" for i in range(60))
|
||||
assert len(discover_new.appdetails_languages(text)) == discover_new.MAX_LANGUAGES
|
||||
|
||||
|
||||
def test_a_candidate_carries_its_languages():
|
||||
data = {"name": "//トロイメライ・ノクターン", "type": "game", "is_free": True,
|
||||
"supported_languages": "Japanese", "release_date": {"date": "Oct 16, 2026"}}
|
||||
cand = discover_new.Candidate("5118700", data)
|
||||
assert cand.link == "https://store.steampowered.com/app/5118700/"
|
||||
assert cand.to_wire()["payload"]["languages"] == ["Japanese"]
|
||||
assert discover_new.Candidate("1", {}).payload["languages"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("languages, note", [
|
||||
(["English", "Japanese"], ""),
|
||||
(["Japanese"], "no English: Japanese"),
|
||||
(["Korean", "Japanese", "Thai", "Russian"], "no English: Korean, Japanese, Thai, ..."),
|
||||
([], "no languages declared"),
|
||||
])
|
||||
def test_language_note(languages, note):
|
||||
assert discover_new.language_note(languages) == note
|
||||
|
||||
|
||||
def test_candidates_without_english_are_counted():
|
||||
cands = [discover_new.Candidate(str(i), {"supported_languages": s})
|
||||
for i, s in enumerate(["English", "Japanese", "", "English, Korean"])]
|
||||
assert discover_new.without_english(cands) == 2
|
||||
@@ -3,11 +3,13 @@ fetch_full() builds every new game (ingest_new.py) and refills incomplete ones
|
||||
(update_data.py, refetch_all.py). These pin how it treats the anti-cheat
|
||||
fields: filled from the developer's disclosure on the page it already fetched,
|
||||
else from appdetails' VAC category, never replaced, and never where an
|
||||
override would undo it.
|
||||
override would undo it. And how it picks a genre: never a price model.
|
||||
|
||||
Steam is never called: the client is a fake. Run from the repository root:
|
||||
python -m pytest scripts/tests
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from core import fetcher
|
||||
from core.data_store import make_skeleton
|
||||
from core.overrides import OverrideGuard
|
||||
@@ -124,6 +126,37 @@ def test_apply_details_writes_no_anti_cheat():
|
||||
assert (game["anti_cheat"], game["anti_cheat_note"], game["is_kernel_ac"]) == ("-", "", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("genres, genre", [
|
||||
(["Action", "Free To Play"], "Action"),
|
||||
# appdetails capitalises the T; the skip list used to miss it.
|
||||
(["Casual", "Free To Play", "Indie"], "Casual"),
|
||||
(["Free To Play", "Massively Multiplayer"], "Massively Multiplayer"),
|
||||
(["Free To Play", "Early Access"], "Uncategorized"),
|
||||
(["Free To Play"], "Uncategorized"),
|
||||
(["Indie", "Simulation"], "Simulation"),
|
||||
([], "Uncategorized"),
|
||||
([" RPG ", None, ""], "RPG"),
|
||||
])
|
||||
def test_pick_genre_never_picks_a_price_model(genres, genre):
|
||||
assert fetcher.pick_genre(genres) == genre
|
||||
|
||||
|
||||
def test_apply_details_reads_genres_case_insensitively():
|
||||
game = make_skeleton(LINK)
|
||||
game["genre"] = ""
|
||||
fetcher.apply_details(game, {"genres": [{"id": "37", "description": "Free To Play"},
|
||||
{"id": "4", "description": "Casual"}]})
|
||||
assert game["genre"] == "Casual"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("genre, stray", [
|
||||
("Free To Play", True), ("free to play", True), (" Early Access ", True),
|
||||
("Casual", False), ("", False), (None, False), (["Free To Play"], False),
|
||||
])
|
||||
def test_a_stray_genre_is_one_no_person_chose(genre, stray):
|
||||
assert fetcher.is_stray_genre(genre) is stray
|
||||
|
||||
|
||||
def test_the_guard_reaches_fetch_full():
|
||||
# A restored game keeps its override, here "nobody has verified the kernel
|
||||
# flag": filling it would be undone by save_main() on every run.
|
||||
|
||||
@@ -90,6 +90,40 @@ def test_a_numeric_link_no_longer_crashes_the_run(repo):
|
||||
assert open(os.path.join("scripts", "temp_info.jsonl"), encoding="utf-8").read() == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw, link", [
|
||||
# A non-English store page: the slug is the name, percent-encoded or raw.
|
||||
("https://store.steampowered.com/app/5118700/%E3%83%88%E3%83%AD%E3%82%A4/",
|
||||
"https://store.steampowered.com/app/5118700/"),
|
||||
("https://store.steampowered.com/app/5118700/トロイメライノクターン/?l=japanese",
|
||||
"https://store.steampowered.com/app/5118700/"),
|
||||
# A Japanese keyboard types full-width digits; NFKC reads them as ASCII.
|
||||
("5118700", "https://store.steampowered.com/app/5118700/"),
|
||||
("store.steampowered.com/app/5118700/", "https://store.steampowered.com/app/5118700/"),
|
||||
# Any other script's digits are refused, never stored: a non-ASCII appid
|
||||
# would stop every refresh (rotation.slice_of encodes it as ASCII).
|
||||
("٥١١٨٧٠٠", None),
|
||||
("https://store.steampowered.com/app/٥١١٨٧٠٠/", None),
|
||||
("https://store.steampowered.com/app/730٤/", None),
|
||||
])
|
||||
def test_a_link_in_any_script_becomes_the_canonical_one_or_nothing(raw, link):
|
||||
assert data_store.normalize_link(raw) == link
|
||||
|
||||
|
||||
def test_extract_appid_reads_ascii_digits_only():
|
||||
assert data_store.extract_appid("https://store.steampowered.com/app/730/") == "730"
|
||||
assert data_store.extract_appid("https://store.steampowered.com/app/5/") is None
|
||||
assert data_store.extract_appid("https://store.steampowered.com/app/730٤/") is None
|
||||
|
||||
|
||||
def test_a_link_in_other_digits_is_refused_and_the_run_goes_on(repo):
|
||||
_queue({"link": "٥١١٨٧٠٠"}, {"link": "5118700"})
|
||||
ingest_new.main()
|
||||
assert _appids() == ["1", "5118700"]
|
||||
assert repo == ["5118700"]
|
||||
# Not logged: a row without an appid is dropped by dedup_removed().
|
||||
assert data_store.load_jsonl(os.path.join("scripts", "removed_games.jsonl")) == []
|
||||
|
||||
|
||||
def test_a_blocked_game_is_skipped_before_any_steam_request(repo):
|
||||
with open(os.path.join("data", "blocklist.jsonl"), "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(serialize_entry(make_entry("440", "Blocked", "not_free", "", "2026-10-13T12:00:00Z")) + "\n")
|
||||
|
||||
@@ -203,6 +203,21 @@ def test_save_main_writes_every_record_normalised_and_none_as_news(workdir):
|
||||
assert stored["last_updated"] == "2026-01-01T00:00:00Z"
|
||||
|
||||
|
||||
def test_a_cjk_blurb_already_stored_is_never_cut_again(workdir):
|
||||
# Exactly the cap, ending in "、…" as the old rule could leave it: the new
|
||||
# rule must not touch it, or every such record would change on the next save.
|
||||
blurb = ("星の稗史を記録する本屋で、" * 14)[:data_store.DESCRIPTION_MAX - 2] + "、…"
|
||||
assert len(blurb) == data_store.DESCRIPTION_MAX
|
||||
game = {"link": "https://store.steampowered.com/app/5118700/", "name": "//トロイメライ・ノクターン",
|
||||
"description": blurb, "last_updated": "2026-01-01T00:00:00Z"}
|
||||
data_store.save_main([game], apply_human_overrides=False)
|
||||
before = _shards()
|
||||
[stored] = data_store.load_main()
|
||||
assert stored["description"] == blurb and stored["name"] == "//トロイメライ・ノクターン"
|
||||
data_store.save_main(data_store.load_main(), apply_human_overrides=False)
|
||||
assert _shards() == before
|
||||
|
||||
|
||||
def test_saving_what_was_loaded_changes_no_byte(workdir):
|
||||
data_store.save_main([_messy(1), _messy(2)], apply_human_overrides=False)
|
||||
before = _shards()
|
||||
|
||||
@@ -344,6 +344,22 @@ def test_missing_only_takes_just_the_repairs():
|
||||
assert [(g["link"], p, d) for g, p, d in plan] == [(games[1]["link"], True, False)]
|
||||
|
||||
|
||||
def test_stray_genres_are_read_every_day_capped(monkeypatch):
|
||||
monkeypatch.setattr(rsd, "GENRE_DAILY_CAP", 2)
|
||||
games = [_game(str(10 * i), genre="Free To Play") for i in range(1, 6)] + [_game("730")]
|
||||
plan = rsd.build_plan(games, day=3)
|
||||
strays = [(g, d) for g, _, d in plan if g["genre"] == "Free To Play" and d and not rsd.due(
|
||||
g, rsd.DETAILS_CYCLE_DAYS, 3)]
|
||||
assert len(strays) <= 2
|
||||
assert all(d for _, d in strays)
|
||||
assert sum(1 for g, _, _ in plan if g["link"].endswith("/730/")) <= 1
|
||||
|
||||
|
||||
def test_a_page_repair_with_a_stray_genre_reads_its_details_too():
|
||||
plan = rsd.build_plan([_game("4156010", tags=[], genre="Free To Play")], day=0)
|
||||
assert [(p, d) for _, p, d in plan] == [(True, True)]
|
||||
|
||||
|
||||
def test_a_details_only_run_repairs_no_page():
|
||||
plan = rsd.build_plan([_game("4156010", tags=[])], day=0, want_page=False)
|
||||
assert not any(p for _, p, _ in plan)
|
||||
@@ -388,11 +404,68 @@ def test_a_blurb_that_now_fits_further_is_written_but_is_not_news():
|
||||
assert game["description"] == rsd.truncate_description(blurb)
|
||||
|
||||
|
||||
GENRES = {"genres": [{"description": "Casual"}, {"description": "Free To Play"},
|
||||
{"description": "Indie"}]}
|
||||
|
||||
|
||||
def test_a_stray_genre_is_replaced_by_the_one_appdetails_gives():
|
||||
game = _game("4156010", genre="Free To Play")
|
||||
assert "genre" in rsd.apply_app_details(game, GENRES)
|
||||
assert game["genre"] == "Casual"
|
||||
|
||||
|
||||
def test_a_genre_a_person_could_have_chosen_is_never_touched():
|
||||
game = _game("4156010", genre="Puzzle")
|
||||
assert "genre" not in rsd.apply_app_details(game, GENRES)
|
||||
assert game["genre"] == "Puzzle"
|
||||
|
||||
|
||||
def test_a_stray_genre_stays_without_genres_or_under_an_override():
|
||||
game = _game("4156010", genre="Free To Play")
|
||||
assert "genre" not in rsd.apply_app_details(game, {"genres": []})
|
||||
guard = rsd.OverrideGuard({"4156010": {"fields": {"genre": {"value": "Free To Play"}}}})
|
||||
assert "genre" not in rsd.apply_app_details(game, GENRES, guard)
|
||||
assert game["genre"] == "Free To Play"
|
||||
|
||||
|
||||
def test_a_rewritten_blurb_is_a_change():
|
||||
game = _game(description="Old words, cut short…")
|
||||
assert "description" in rsd.apply_app_details(game, {"short_description": "New words entirely."})
|
||||
|
||||
|
||||
# Two sentences (76 and 64 characters), then a third that runs past the cap.
|
||||
CJK_BLURB = "星の稗史を記録する本屋で物語を紡ぐノベルゲームです" * 3 + "。" \
|
||||
+ "貴方の選択は果たして何を意味していましたか" * 3 + "。" + "静かな夜の湖畔で" * 10
|
||||
|
||||
|
||||
def test_a_cjk_blurb_is_cut_after_a_sentence():
|
||||
cut = rsd.truncate_description(CJK_BLURB)
|
||||
# Just after the second "。", which is trimmed like a Latin full stop.
|
||||
assert cut == CJK_BLURB[:CJK_BLURB.index("。", 80)] + "…"
|
||||
assert len(cut) <= DESCRIPTION_MAX
|
||||
assert rsd.truncate_description(cut) == cut
|
||||
|
||||
|
||||
def test_a_cjk_comma_never_comes_before_the_ellipsis():
|
||||
# No sentence end and no space: a hard cut, landing just after "、".
|
||||
blurb = "物" * (DESCRIPTION_MAX - 2) + "、" + "語" * 50
|
||||
assert rsd.truncate_description(blurb) == "物" * (DESCRIPTION_MAX - 2) + "…"
|
||||
|
||||
|
||||
def test_a_cjk_blurb_cut_earlier_than_stored_is_written_but_is_not_news():
|
||||
hard = CJK_BLURB[:DESCRIPTION_MAX - 1] + "…" # how the old rule cut it
|
||||
game = _game(description=hard)
|
||||
assert "description" not in rsd.apply_app_details(game, {"short_description": CJK_BLURB})
|
||||
assert game["description"] == rsd.truncate_description(CJK_BLURB)
|
||||
|
||||
|
||||
def test_a_shorter_rewrite_is_still_a_change():
|
||||
game = _game(description=CJK_BLURB[:DESCRIPTION_MAX - 1] + "…")
|
||||
assert "description" in rsd.apply_app_details(game, {"short_description": "新しい説明です。"})
|
||||
game = _game(description="Old words, cut short…")
|
||||
assert "description" in rsd.apply_app_details(game, {"short_description": "Other words, cut…"})
|
||||
|
||||
|
||||
# ──────────── A growing catalogue ────────────
|
||||
|
||||
def test_the_page_cycle_grows_with_the_catalogue_and_keeps_its_weekday():
|
||||
|
||||
@@ -92,6 +92,22 @@ def test_appdetails_asks_for_english():
|
||||
assert session.requests[0]["cookies"] is None # age cookies: store pages only
|
||||
|
||||
|
||||
def test_search_asks_for_every_language_and_english_text():
|
||||
body = {"success": 1, "results_html": "", "total_count": 0, "start": 0}
|
||||
client, session = _client(_Resp("https://store.steampowered.com/search/results/", body=body))
|
||||
assert client.fetch_search_page(0) == body
|
||||
[req] = session.requests
|
||||
assert "supportedlang" not in req["params"] # games in every language
|
||||
assert req["params"]["l"] == "english" # "Coming soon" and month names
|
||||
assert req["params"]["maxprice"] == "free" and req["params"]["category1"] == 998
|
||||
assert req["cookies"] is None # age cookies: store pages only
|
||||
|
||||
|
||||
def test_a_failed_search_page_is_none():
|
||||
client, _ = _client(_Resp("https://store.steampowered.com/search/results/", body={"success": 0}))
|
||||
assert client.fetch_search_page(0) is None
|
||||
|
||||
|
||||
def test_store_items_are_keyed_by_the_id_asked_for():
|
||||
body = {"response": {"store_items": [
|
||||
{"id": 2279510, "appid": 2279510, "success": 1, "tags": []},
|
||||
|
||||
+15
-1
@@ -18,6 +18,7 @@ interface Rec {
|
||||
release_date?: string;
|
||||
reviews?: string;
|
||||
current_players?: string;
|
||||
languages?: string[];
|
||||
}
|
||||
|
||||
const HOUR = 3_600_000;
|
||||
@@ -65,7 +66,14 @@ export function seed(sqlite: SqliteD1, repo: FakeRepo, records: Rec[], now: Date
|
||||
pct(r),
|
||||
r.current_players ?? "N/A",
|
||||
players(r),
|
||||
JSON.stringify({ link: r.link, name: r.name, reviews: r.reviews, current_players: r.current_players }),
|
||||
JSON.stringify({
|
||||
link: r.link,
|
||||
name: r.name,
|
||||
reviews: r.reviews,
|
||||
current_players: r.current_players,
|
||||
// Discovery records every candidate's languages since v4.6.0.
|
||||
languages: r.languages,
|
||||
}),
|
||||
extra.source ?? (n % 5 === 0 ? "backfill:2025" : "discover"),
|
||||
status,
|
||||
extra.by ?? null,
|
||||
@@ -93,6 +101,12 @@ export function seed(sqlite: SqliteD1, repo: FakeRepo, records: Rec[], now: Date
|
||||
};
|
||||
|
||||
for (let k = 0; k < 45; k++) row(hide(next()), "pending");
|
||||
// A game whose store page lists no English, for the dialog's "No English" badge.
|
||||
// Outside the pool, so it never meets another open row for the same game.
|
||||
const foreign = records.find(
|
||||
(r) => !pool.includes(r) && r.name && r.header_image && r.languages?.length && !r.languages.includes("English"),
|
||||
);
|
||||
if (foreign) row(hide(foreign), "pending", { seenAgo: HOUR / 2 });
|
||||
for (let k = 0; k < 6; k++) row(hide(next()), "deferred", { seenAgo: (60 + k) * HOUR });
|
||||
row(hide(next()), "failed", { reason: "never appeared in data/ or removed_games.jsonl - approve again to retry", by: reviewer, at: iso(30 * HOUR) });
|
||||
row(hide(next()), "failed", { reason: "rejected by ingest pipeline", by: reviewer, at: iso(50 * HOUR) });
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { exactTime, plural, prettyJson, rangeLabel, relativeTime, shortSha, showValue } from "./format";
|
||||
import {
|
||||
exactTime,
|
||||
lacksEnglish,
|
||||
languagesLabel,
|
||||
payloadLanguages,
|
||||
plural,
|
||||
prettyJson,
|
||||
rangeLabel,
|
||||
relativeTime,
|
||||
shortSha,
|
||||
showValue,
|
||||
} from "./format";
|
||||
|
||||
const NOW = Date.parse("2026-09-17T12:00:00Z");
|
||||
|
||||
@@ -40,4 +51,28 @@ describe("format", () => {
|
||||
expect(prettyJson('{"a":1}')).toBe('{\n "a": 1\n}');
|
||||
expect(prettyJson("{broken")).toBe("{broken");
|
||||
});
|
||||
|
||||
it("payloadLanguages tells no list from an empty one", () => {
|
||||
expect(payloadLanguages('{"languages":["Japanese"]}')).toEqual(["Japanese"]);
|
||||
expect(payloadLanguages('{"languages":[]}')).toEqual([]);
|
||||
expect(payloadLanguages('{"languages":["English",3,null,"Korean"]}')).toEqual(["English", "Korean"]);
|
||||
// Queued before v4.6.0, or not a payload at all.
|
||||
expect(payloadLanguages('{"short_description":"x"}')).toBeNull();
|
||||
expect(payloadLanguages('{"languages":"Japanese"}')).toBeNull();
|
||||
expect(payloadLanguages("{broken")).toBeNull();
|
||||
expect(payloadLanguages("null")).toBeNull();
|
||||
expect(payloadLanguages("7")).toBeNull();
|
||||
expect(payloadLanguages("")).toBeNull();
|
||||
expect(payloadLanguages(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it("languagesLabel and lacksEnglish", () => {
|
||||
expect(languagesLabel(["Japanese", "Korean"])).toBe("Japanese, Korean");
|
||||
expect(languagesLabel([])).toBe("none declared");
|
||||
expect(languagesLabel(null)).toBe("—");
|
||||
expect(lacksEnglish(["Japanese"])).toBe(true);
|
||||
expect(lacksEnglish([])).toBe(true);
|
||||
expect(lacksEnglish(["English", "Japanese"])).toBe(false);
|
||||
expect(lacksEnglish(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -61,6 +61,35 @@ export function prettyJson(text: string | null | undefined): string {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The languages a candidate's payload lists: English names, as discovery read
|
||||
* them from appdetails. `null` when there is no list - rows queued before
|
||||
* discovery took games in every language (v4.6.0), or a payload that does not
|
||||
* parse - which is not the same as `[]`, a game that declares no language.
|
||||
*/
|
||||
export function payloadLanguages(text: string | null | undefined): string[] | null {
|
||||
if (!text) return null;
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const list = (parsed as { languages?: unknown } | null)?.languages;
|
||||
if (!Array.isArray(list)) return null;
|
||||
return list.filter((l): l is string => typeof l === "string");
|
||||
}
|
||||
|
||||
/** The languages as the candidate dialog shows them. */
|
||||
export function languagesLabel(languages: string[] | null): string {
|
||||
if (languages === null) return "—";
|
||||
return languages.length ? languages.join(", ") : "none declared";
|
||||
}
|
||||
|
||||
/** A known list without English: a reader of English cannot play the game. */
|
||||
export const lacksEnglish = (languages: string[] | null): boolean =>
|
||||
languages !== null && !languages.includes("English");
|
||||
|
||||
/** "1-50 of 646" for a pager. */
|
||||
export function rangeLabel(offset: number, count: number, total: number): string {
|
||||
if (!total || !count) return `0 of ${formatInt(total)}`;
|
||||
|
||||
@@ -21,7 +21,15 @@
|
||||
import StatusBadge from "../components/StatusBadge.svelte";
|
||||
import { api, errorMessage, SessionExpiredError } from "../lib/api";
|
||||
import { isAbort, reportError } from "../lib/notify";
|
||||
import { exactTime, prettyJson, relativeTime, steamUrl } from "../lib/format";
|
||||
import {
|
||||
exactTime,
|
||||
lacksEnglish,
|
||||
languagesLabel,
|
||||
payloadLanguages,
|
||||
prettyJson,
|
||||
relativeTime,
|
||||
steamUrl,
|
||||
} from "../lib/format";
|
||||
import { LOCK_HELP, REOPEN_TEXT } from "../lib/labels";
|
||||
import { lockReason } from "../../../shared/queue-rules";
|
||||
import { REMOVED_DECISION_PREFIX } from "../../../shared/removal";
|
||||
@@ -67,6 +75,8 @@
|
||||
});
|
||||
|
||||
const lock = $derived(data ? lockReason(data) : null);
|
||||
// Discovery takes games in every language since v4.6.0; older rows have no list.
|
||||
const languages = $derived(data ? payloadLanguages(data.payload_json) : null);
|
||||
|
||||
async function reopen() {
|
||||
if (!data || reopening) return;
|
||||
@@ -119,6 +129,7 @@
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<StatusBadge status={data.status} />
|
||||
{#if data.published}<Badge variant="success">in the catalogue</Badge>{/if}
|
||||
{#if lacksEnglish(languages)}<Badge variant="info">No English</Badge>{/if}
|
||||
<span class="font-mono text-xs text-muted-foreground">appid {data.appid}</span>
|
||||
</div>
|
||||
<div class="flex flex-wrap gap-x-4 gap-y-1 text-sm">
|
||||
@@ -145,6 +156,7 @@
|
||||
{@render fact("Type", data.app_type)}
|
||||
{@render fact("Free to play", data.is_free ? "yes" : "no")}
|
||||
{@render fact("Store health", data.health_status)}
|
||||
{@render fact("Languages", languagesLabel(languages), languagesLabel(languages))}
|
||||
{@render fact("First seen", relativeTime(data.first_seen_at), exactTime(data.first_seen_at))}
|
||||
{@render fact("Last seen", relativeTime(data.last_seen_at), exactTime(data.last_seen_at))}
|
||||
{@render fact("Times seen", String(data.seen_count))}
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "free-steam-games-web",
|
||||
"version": "2.3.0",
|
||||
"version": "2.4.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "free-steam-games-web",
|
||||
"version": "2.3.0",
|
||||
"version": "2.4.0",
|
||||
"dependencies": {
|
||||
"@fontsource-variable/bricolage-grotesque": "^5.3.0",
|
||||
"@fontsource-variable/ibm-plex-sans": "^5.3.0",
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "free-steam-games-web",
|
||||
"private": true,
|
||||
"version": "2.3.0",
|
||||
"version": "2.4.0",
|
||||
"type": "module",
|
||||
"imports": {
|
||||
"#lib/*": "./src/lib/*"
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
* rebase conflict in data/index.json;
|
||||
* - declare `timeout-minutes`, and a JOB_BUDGET_MINUTES at least 20 below
|
||||
* it (scripts/core/budget.py): the script stops in time to save and push.
|
||||
*
|
||||
* The same holds for any shared group (`discovery`): its members queue, or
|
||||
* none of them does.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
@@ -78,6 +81,19 @@ describe("every data-write workflow", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("shares its queue setting with every workflow in its group", () => {
|
||||
// A group named by an expression is one run's own; only fixed names are shared.
|
||||
const byGroup = new Map<string, boolean[]>();
|
||||
for (const f of readdirSync(DIR).filter((name) => name.endsWith(".yml"))) {
|
||||
const text = code(`.github/workflows/${f}`);
|
||||
const group = /^\s*group\s*:\s*([\w-]+)\s*$/m.exec(text)?.[1];
|
||||
if (!group) continue;
|
||||
byGroup.set(group, [...(byGroup.get(group) ?? []), /\bqueue\s*:\s*max\b/.test(text)]);
|
||||
}
|
||||
expect(byGroup.get("discovery")).toEqual([true, true]);
|
||||
for (const [group, queues] of byGroup) expect(new Set(queues).size, group).toBe(1);
|
||||
});
|
||||
|
||||
it("stops at its budget in time to save before its timeout", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
|
||||
Generated
+1
-1
@@ -952,7 +952,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "f2p-tracker"
|
||||
version = "2.3.0"
|
||||
version = "2.4.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "f2p-tracker"
|
||||
version = "2.3.0"
|
||||
version = "2.4.0"
|
||||
description = "Desktop wrapper around the Steam F2P Tracker web app"
|
||||
authors = ["poli0981"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "F2P Tracker",
|
||||
"version": "2.3.0",
|
||||
"version": "2.4.0",
|
||||
"identifier": "io.github.poli0981.f2p-tracker",
|
||||
"build": {
|
||||
"beforeBuildCommand": "npm run build:desktop",
|
||||
|
||||
@@ -159,6 +159,7 @@
|
||||
"filterAllGenres": "All genres",
|
||||
"filterAllTypes": "All types",
|
||||
"filterAllPlatforms": "All platforms",
|
||||
"filterAllLanguages": "All languages",
|
||||
"filterAllStatus": "All status",
|
||||
"hideDead": "Hide dead",
|
||||
"colName": "Name",
|
||||
@@ -336,6 +337,7 @@
|
||||
"rawData": "Raw data (JSONL)",
|
||||
"headsUpGenre": "Genre is best-effort. The pipeline picks one genre per game from Steam tags and the description, and for hybrids it is often wrong. Corrections are made by hand and survive every later scrape.",
|
||||
"headsUpEnglish": "The scraper assumes Steam pages in English. Regional storefront overrides can break parsing, so names, genres and DLC counts may be wrong for some records.",
|
||||
"headsUpLanguages": "The catalogue lists free games in every language, not only the ones that support English. A name or description the developer never wrote in English stays in their language; the language filter on Games narrows the list to the languages you read.",
|
||||
"headsUpValve": "This is not a Valve or Steam product. Independent project, no affiliation, no support contract.",
|
||||
"aiDisclosureBody": "About 70% of the code here was written with help from large-language-model assistants - Grok for the first Python pipeline, Claude for the rewrite, this web app, the Worker and the legal docs. Everything is reviewed before it is committed, and no user data is sent to any model at runtime: those calls happen on the maintainer's machine, never from your browser.",
|
||||
"maintainerBlurb": "One Vietnamese developer, running this in spare time.",
|
||||
|
||||
@@ -159,6 +159,7 @@
|
||||
"filterAllGenres": "Mọi thể loại",
|
||||
"filterAllTypes": "Mọi loại",
|
||||
"filterAllPlatforms": "Mọi nền tảng",
|
||||
"filterAllLanguages": "Mọi ngôn ngữ",
|
||||
"filterAllStatus": "Mọi trạng thái",
|
||||
"hideDead": "Ẩn dead",
|
||||
"colName": "Tên",
|
||||
@@ -336,6 +337,7 @@
|
||||
"rawData": "Dữ liệu thô (JSONL)",
|
||||
"headsUpGenre": "Thể loại chỉ mang tính tương đối. Pipeline chọn một thể loại cho mỗi game từ tag Steam và mô tả, với game lai thì thường sai. Các sửa đổi thủ công được giữ qua mọi lần quét sau.",
|
||||
"headsUpEnglish": "Trình quét giả định trang Steam bằng tiếng Anh. Storefront theo khu vực có thể làm sai kết quả phân tích, nên tên, thể loại và số DLC có thể không chính xác ở một số bản ghi.",
|
||||
"headsUpLanguages": "Danh sách có game free ở mọi ngôn ngữ, không chỉ game hỗ trợ tiếng Anh. Tên hay mô tả mà nhà phát triển không viết bằng tiếng Anh được giữ nguyên ngôn ngữ gốc; bộ lọc ngôn ngữ ở trang Trò chơi thu hẹp danh sách theo ngôn ngữ bạn đọc được.",
|
||||
"headsUpValve": "Đây không phải sản phẩm của Valve hay Steam. Dự án độc lập, không liên kết, không hợp đồng hỗ trợ.",
|
||||
"aiDisclosureBody": "Khoảng 70% mã nguồn ở đây được viết với sự hỗ trợ của các trợ lý mô hình ngôn ngữ lớn - Grok cho pipeline Python đầu tiên, Claude cho bản viết lại, ứng dụng web này, Worker và tài liệu pháp lý. Mọi thứ đều được xem lại trước khi commit, và không dữ liệu người dùng nào được gửi tới mô hình lúc chạy.",
|
||||
"maintainerBlurb": "Một lập trình viên Việt Nam, làm dự án này lúc rảnh.",
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
import { headerToCapsule, thumbFallback } from "../image";
|
||||
import { imgFallback } from "../img-fallback";
|
||||
import { cn } from "../utils";
|
||||
import { longEnough, searchKey } from "../search-fold";
|
||||
|
||||
const t = i18n.t;
|
||||
|
||||
@@ -23,22 +24,31 @@
|
||||
*/
|
||||
const routes = paletteRoutes(CHART_PAGES);
|
||||
|
||||
const q = $derived(palette.query.trim().toLowerCase());
|
||||
// Folded (lib/search-fold.ts): a name typed in another width or kana script
|
||||
// still matches. Folded BEFORE trimming, which turns U+3000 into a space.
|
||||
const q = $derived(searchKey(palette.query).trim());
|
||||
|
||||
const matchedRoutes = $derived(
|
||||
q ? routes.filter((r) => t(r.i18n).toLowerCase().includes(q) || r.to.includes(q)) : routes,
|
||||
q ? routes.filter((r) => searchKey(t(r.i18n)).includes(q) || r.to.includes(q)) : routes,
|
||||
);
|
||||
|
||||
// Once per record set, not per keystroke; derived lazily, so only once the
|
||||
// palette is first searched.
|
||||
const gameKeys = $derived((games.data?.records ?? []).map((g) => searchKey(g.name ?? "")));
|
||||
|
||||
// Capped at 8: the palette is for jumping somewhere, not for browsing the
|
||||
// catalogue - /games is for that.
|
||||
// catalogue - /games is for that. Two characters, or one kanji, kana or
|
||||
// Hangul syllable, which is already a word.
|
||||
const matchedGames = $derived.by(() => {
|
||||
if (q.length < 2) return [];
|
||||
if (!longEnough(q, 2)) return [];
|
||||
const all = games.data?.records ?? [];
|
||||
const keys = gameKeys;
|
||||
const out = [];
|
||||
for (const g of all) {
|
||||
for (let i = 0; i < all.length; i++) {
|
||||
const g = all[i];
|
||||
// Only rows with an appid: a game is reachable by /games/<appid> alone,
|
||||
// and `items` below indexes games by position.
|
||||
if (g.name?.toLowerCase().includes(q) && appidOf(g)) out.push(g);
|
||||
if (keys[i].includes(q) && appidOf(g)) out.push(g);
|
||||
if (out.length >= 8) break;
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { csvForExcel, recordsToCsv } from "./export";
|
||||
import type { GameRecord } from "./schema";
|
||||
|
||||
describe("CSV export", () => {
|
||||
it("starts with a byte order mark, so Excel reads it as UTF-8", () => {
|
||||
const out = csvForExcel("name\nトロイ\n");
|
||||
expect(out.codePointAt(0)).toBe(0xfeff);
|
||||
expect(out.slice(1)).toBe("name\nトロイ\n");
|
||||
expect([...new TextEncoder().encode(out).slice(0, 3)]).toEqual([0xef, 0xbb, 0xbf]);
|
||||
});
|
||||
|
||||
it("keeps CJK names as written and quotes only an ASCII comma", () => {
|
||||
const game = { link: "https://store.steampowered.com/app/5118700/", name: "//トロイメライ・ノクターン" };
|
||||
const other = { link: "https://store.steampowered.com/app/1/", name: "无主之地,银河, again" };
|
||||
const [, first, second] = recordsToCsv([game, other] as unknown as GameRecord[]).split("\n");
|
||||
expect(first.startsWith("https://store.steampowered.com/app/5118700/,//トロイメライ・ノクターン,")).toBe(true);
|
||||
expect(second.startsWith('https://store.steampowered.com/app/1/,"无主之地,银河, again",')).toBe(true);
|
||||
});
|
||||
});
|
||||
+11
-1
@@ -75,8 +75,18 @@ export function downloadBlob(content: string, filename: string, mime: string) {
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* The CSV as a spreadsheet should open it. Without a byte order mark Excel
|
||||
* reads a CSV in the system's legacy code page and turns every Japanese,
|
||||
* Chinese or Korean name into mojibake - on Japanese Windows too, whose code
|
||||
* page is not UTF-8 either. Other programs skip the mark; JSON needs none.
|
||||
*/
|
||||
export function csvForExcel(csv: string): string {
|
||||
return "\uFEFF" + csv;
|
||||
}
|
||||
|
||||
export function exportCsv(records: GameRecord[], filename = "f2p-games.csv") {
|
||||
downloadBlob(recordsToCsv(records), filename, "text/csv");
|
||||
downloadBlob(csvForExcel(recordsToCsv(records)), filename, "text/csv");
|
||||
}
|
||||
|
||||
export function exportJson(records: GameRecord[], filename = "f2p-games.json") {
|
||||
|
||||
@@ -26,6 +26,8 @@ class Filters {
|
||||
genre = $state<string | null>(null);
|
||||
typeGame = $state<"online" | "offline" | null>(null);
|
||||
platform = $state<string | null>(null);
|
||||
/** One of the languages Steam lists for the game, by its English name. */
|
||||
language = $state<string | null>(null);
|
||||
safe = $state<SafeClass | null>(null);
|
||||
status = $state<"active" | "delisted" | null>(null);
|
||||
hasAntiCheat = $state<boolean | null>(null);
|
||||
@@ -49,6 +51,7 @@ class Filters {
|
||||
this.genre !== null ||
|
||||
this.typeGame !== null ||
|
||||
this.platform !== null ||
|
||||
this.language !== null ||
|
||||
this.safe !== null ||
|
||||
this.status !== null ||
|
||||
this.hasAntiCheat !== null ||
|
||||
@@ -63,6 +66,7 @@ class Filters {
|
||||
this.genre,
|
||||
this.typeGame,
|
||||
this.platform,
|
||||
this.language,
|
||||
this.safe,
|
||||
this.status,
|
||||
this.hasAntiCheat,
|
||||
@@ -96,6 +100,7 @@ class Filters {
|
||||
this.genre = null;
|
||||
this.typeGame = null;
|
||||
this.platform = null;
|
||||
this.language = null;
|
||||
this.safe = null;
|
||||
this.status = null;
|
||||
this.hasAntiCheat = null;
|
||||
@@ -111,6 +116,7 @@ class Filters {
|
||||
if (this.genre !== null) q.set("genre", this.genre);
|
||||
if (this.typeGame !== null) q.set("type", this.typeGame);
|
||||
if (this.platform !== null) q.set("platform", this.platform);
|
||||
if (this.language !== null) q.set("lang", this.language);
|
||||
if (this.safe !== null) q.set("safe", SAFE_PARAM[this.safe]);
|
||||
if (this.status !== null) q.set("status", this.status);
|
||||
if (this.hasAntiCheat !== null) q.set("ac", this.hasAntiCheat ? "yes" : "no");
|
||||
@@ -135,6 +141,7 @@ class Filters {
|
||||
const type = q.get("type");
|
||||
this.typeGame = type === "online" || type === "offline" ? type : null;
|
||||
this.platform = q.get("platform") || null;
|
||||
this.language = q.get("lang") || null;
|
||||
const safe = Object.entries(SAFE_PARAM).find(([, param]) => param === q.get("safe"));
|
||||
this.safe = safe ? (safe[0] as SafeClass) : null;
|
||||
const status = q.get("status");
|
||||
@@ -157,4 +164,4 @@ class Filters {
|
||||
export const filters = new Filters();
|
||||
|
||||
/** Query parameters the /games page owns. */
|
||||
export const FILTER_PARAMS = new Set(["q", "genre", "type", "platform", "safe", "status", "ac", "dead", "sort", "dir", "page"]);
|
||||
export const FILTER_PARAMS = new Set(["q", "genre", "type", "platform", "lang", "safe", "status", "ac", "dead", "sort", "dir", "page"]);
|
||||
@@ -0,0 +1,33 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { filters, FILTER_PARAMS } from "./filters.svelte";
|
||||
|
||||
describe("the language filter in the store", () => {
|
||||
beforeEach(() => filters.reset());
|
||||
|
||||
it("is a filter: active, part of the signature, cleared by reset", () => {
|
||||
const before = filters.signature;
|
||||
filters.language = "Japanese";
|
||||
// Without `active`, a language-only filter had no "Clear filters" button.
|
||||
expect(filters.active).toBe(true);
|
||||
expect(filters.signature).not.toBe(before);
|
||||
filters.reset();
|
||||
expect(filters.language).toBeNull();
|
||||
expect(filters.active).toBe(false);
|
||||
});
|
||||
|
||||
it("round-trips through ?lang=", () => {
|
||||
filters.language = "Simplified Chinese";
|
||||
const q = filters.toQuery();
|
||||
expect(q.get("lang")).toBe("Simplified Chinese");
|
||||
filters.reset();
|
||||
filters.fromQuery(q);
|
||||
expect(filters.language).toBe("Simplified Chinese");
|
||||
expect(FILTER_PARAMS.has("lang")).toBe(true);
|
||||
});
|
||||
|
||||
it("reads an empty ?lang= as no filter", () => {
|
||||
filters.fromQuery(new URLSearchParams("lang="));
|
||||
expect(filters.language).toBeNull();
|
||||
expect(filters.toQuery().has("lang")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyFilters, buildSearchIndex, facets, type FilterCriteria } from "./filtering";
|
||||
import type { GameRecord } from "../schema";
|
||||
|
||||
const NONE: FilterCriteria = {
|
||||
search: "",
|
||||
genre: null,
|
||||
typeGame: null,
|
||||
platform: null,
|
||||
language: null,
|
||||
safe: null,
|
||||
status: null,
|
||||
hasAntiCheat: null,
|
||||
hideDead: false,
|
||||
};
|
||||
|
||||
function game(name: string, languages: string[], extra: Partial<GameRecord> = {}): GameRecord {
|
||||
return {
|
||||
name,
|
||||
languages,
|
||||
description: "",
|
||||
tags: [],
|
||||
developer: [],
|
||||
publisher: [],
|
||||
platforms: [],
|
||||
...extra,
|
||||
} as unknown as GameRecord;
|
||||
}
|
||||
|
||||
const records = [
|
||||
game("//トロイメライ・ノクターン", ["Japanese"], { developer: ["shiori-project"] }),
|
||||
game("Apex Legends™", ["English", "Japanese", "Korean"], { tags: ["Battle Royale"] }),
|
||||
game("无主之地:银河", ["Simplified Chinese"], { tags: ["シューティング"] }),
|
||||
game("Untitled", []),
|
||||
];
|
||||
const fuse = buildSearchIndex(records);
|
||||
const names = (c: Partial<FilterCriteria>) => applyFilters(records, fuse, { ...NONE, ...c }).map((g) => g.name);
|
||||
|
||||
describe("the language filter", () => {
|
||||
it("keeps the games that list the language", () => {
|
||||
expect(names({ language: "Japanese" })).toEqual(["//トロイメライ・ノクターン", "Apex Legends™"]);
|
||||
expect(names({ language: "Simplified Chinese" })).toEqual(["无主之地:银河"]);
|
||||
expect(names({ language: null })).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("combines with search", () => {
|
||||
expect(names({ language: "Korean", search: "legends" })).toEqual(["Apex Legends™"]);
|
||||
expect(names({ language: "Korean", search: "とろい" })).toEqual([]);
|
||||
});
|
||||
|
||||
it("counts each language once per game, most common first", () => {
|
||||
const withDupe = [...records, game("Twice", ["Japanese", "Japanese"])];
|
||||
expect(facets(withDupe).language).toEqual([
|
||||
["Japanese", 3],
|
||||
["English", 1],
|
||||
["Korean", 1],
|
||||
["Simplified Chinese", 1],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("search", () => {
|
||||
it("matches across width and kana script, in every key", () => {
|
||||
expect(names({ search: "とろい" })).toEqual(["//トロイメライ・ノクターン"]);
|
||||
expect(names({ search: "トロイ" })).toEqual(["//トロイメライ・ノクターン"]);
|
||||
expect(names({ search: "しゅーてぃんぐ" })).toEqual(["无主之地:银河"]);
|
||||
expect(names({ search: "SHIORI" })).toEqual(["//トロイメライ・ノクターン"]);
|
||||
});
|
||||
|
||||
it("trims an ideographic space once it is folded", () => {
|
||||
expect(names({ search: " とろい " })).toEqual(["//トロイメライ・ノクターン"]);
|
||||
expect(names({ search: " " })).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("still finds English by name", () => {
|
||||
expect(names({ search: "legends" })).toEqual(["Apex Legends™"]);
|
||||
});
|
||||
});
|
||||
@@ -4,11 +4,12 @@
|
||||
* Kept as plain functions rather than a component or a rune so the page can
|
||||
* compose them in one `$derived` chain and so they are testable without a DOM.
|
||||
*/
|
||||
import Fuse from "fuse.js";
|
||||
import Fuse, { type IFuseOptions } from "fuse.js";
|
||||
import type { GameRecord } from "../schema";
|
||||
import { COLS } from "./columns";
|
||||
import type { SortDir } from "../filters.svelte";
|
||||
import { safeClass, type SafeClass } from "../safety";
|
||||
import { fold } from "../search-fold";
|
||||
|
||||
/**
|
||||
* Fuse options, carried over verbatim.
|
||||
@@ -17,9 +18,21 @@ import { safeClass, type SafeClass } from "../safety";
|
||||
* tight enough that a three-letter query does not return a third of the
|
||||
* catalogue. The key list matters as much: searching `description` is what
|
||||
* lets "battle royale" find games whose name says nothing of the sort.
|
||||
*
|
||||
* Every key is read through fold(), and so is the query (applyFilters), so a
|
||||
* name matches in any width or kana script. One getFn per key, not a wrapped
|
||||
* `Fuse.config.getFn`: for an array field that one hands back `{v, i}`
|
||||
* objects, not strings. No `includeMatches`: its positions would point into
|
||||
* the folded text.
|
||||
*/
|
||||
const FUSE_OPTIONS = {
|
||||
keys: ["name", "description", "tags", "developer", "publisher"],
|
||||
const FUSE_OPTIONS: IFuseOptions<GameRecord> = {
|
||||
keys: [
|
||||
{ name: "name", getFn: (g) => fold(g.name ?? "") },
|
||||
{ name: "description", getFn: (g) => fold(g.description ?? "") },
|
||||
{ name: "tags", getFn: (g) => (g.tags ?? []).map(fold) },
|
||||
{ name: "developer", getFn: (g) => (g.developer ?? []).map(fold) },
|
||||
{ name: "publisher", getFn: (g) => (g.publisher ?? []).map(fold) },
|
||||
],
|
||||
threshold: 0.32,
|
||||
ignoreLocation: true,
|
||||
};
|
||||
@@ -29,6 +42,7 @@ export interface FilterCriteria {
|
||||
genre: string | null;
|
||||
typeGame: "online" | "offline" | null;
|
||||
platform: string | null;
|
||||
language: string | null;
|
||||
safe: SafeClass | null;
|
||||
status: "active" | "delisted" | null;
|
||||
hasAntiCheat: boolean | null;
|
||||
@@ -48,12 +62,15 @@ export function applyFilters(
|
||||
c: FilterCriteria,
|
||||
): GameRecord[] {
|
||||
// Search first: it is the most selective, so every later predicate runs over
|
||||
// a smaller list.
|
||||
let out = c.search.trim() ? fuse.search(c.search.trim()).map((r) => r.item) : records;
|
||||
// a smaller list. Folded BEFORE trimming: NFKC turns U+3000, the space a
|
||||
// Japanese keyboard types, into a plain one.
|
||||
const query = fold(c.search).trim();
|
||||
let out = query ? fuse.search(query).map((r) => r.item) : records;
|
||||
|
||||
if (c.genre) out = out.filter((g) => g.genre === c.genre);
|
||||
if (c.typeGame) out = out.filter((g) => g.type_game === c.typeGame);
|
||||
if (c.platform) out = out.filter((g) => (g.platforms ?? []).includes(c.platform!));
|
||||
if (c.language) out = out.filter((g) => (g.languages ?? []).includes(c.language!));
|
||||
// !== null, not truthiness: "" (never filled in) is a real choice.
|
||||
if (c.safe !== null) out = out.filter((g) => safeClass(g.safe) === c.safe);
|
||||
if (c.status) out = out.filter((g) => g.status === c.status);
|
||||
@@ -96,11 +113,15 @@ export function applySort(
|
||||
export function facets(records: GameRecord[]) {
|
||||
const genre = new Map<string, number>();
|
||||
const platform = new Map<string, number>();
|
||||
// Steam's English language names, as the pipeline stores them: a facet of
|
||||
// data, never i18n keys.
|
||||
const language = new Map<string, number>();
|
||||
for (const g of records) {
|
||||
if (g.genre) genre.set(g.genre, (genre.get(g.genre) ?? 0) + 1);
|
||||
for (const p of g.platforms ?? []) platform.set(p, (platform.get(p) ?? 0) + 1);
|
||||
for (const l of new Set(g.languages ?? [])) language.set(l, (language.get(l) ?? 0) + 1);
|
||||
}
|
||||
const sorted = (m: Map<string, number>) =>
|
||||
[...m.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
||||
return { genre: sorted(genre), platform: sorted(platform) };
|
||||
return { genre: sorted(genre), platform: sorted(platform), language: sorted(language) };
|
||||
}
|
||||
@@ -14,6 +14,7 @@
|
||||
import { openPalette } from "../palette.svelte";
|
||||
import { formatNumber } from "../utils";
|
||||
import { cn } from "../utils";
|
||||
import { jumpsToGames } from "./header-search";
|
||||
|
||||
let { onOpenMenu }: { onOpenMenu: () => void } = $props();
|
||||
|
||||
@@ -44,9 +45,17 @@
|
||||
|
||||
// Typing in the header search jumps to /games, because the box filters the
|
||||
// table and staying on /charts while it silently filtered a table you cannot
|
||||
// see is the kind of thing that reads as a broken search.
|
||||
function onSearchInput() {
|
||||
if (filters.search && !location.pathname.startsWith("/games")) void goto("/games");
|
||||
// see is the kind of thing that reads as a broken search. Not mid-word under
|
||||
// an input method: that jump waits for compositionend (./header-search.ts).
|
||||
// `reset: false` keeps the box focused: the default reset moved focus to
|
||||
// <body>, and the next keystroke went nowhere. The page scrolls inside
|
||||
// <main>, which a reset never touched, so focus is all it changes.
|
||||
function jumpIfNeeded(composing: boolean) {
|
||||
if (jumpsToGames(filters.search, location.pathname, composing)) void goto("/games", { reset: false });
|
||||
}
|
||||
|
||||
function onSearchInput(e: Event) {
|
||||
jumpIfNeeded((e as InputEvent).isComposing);
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -70,6 +79,7 @@
|
||||
type="search"
|
||||
bind:value={filters.search}
|
||||
oninput={onSearchInput}
|
||||
oncompositionend={() => jumpIfNeeded(false)}
|
||||
placeholder={t("topbar.searchPlaceholder")}
|
||||
aria-label={t("topbar.searchPlaceholder")}
|
||||
class="h-9 w-full rounded-md border border-input bg-transparent pl-8 pr-3 text-sm
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { jumpsToGames } from "./header-search";
|
||||
|
||||
describe("jumpsToGames", () => {
|
||||
it("jumps from another page once there is text", () => {
|
||||
expect(jumpsToGames("apex", "/charts", false)).toBe(true);
|
||||
expect(jumpsToGames("トロイ", "/", false)).toBe(true);
|
||||
});
|
||||
|
||||
it("waits while an input method is composing", () => {
|
||||
expect(jumpsToGames("と", "/charts", true)).toBe(false);
|
||||
});
|
||||
|
||||
it("stays on /games, and on an empty box", () => {
|
||||
expect(jumpsToGames("apex", "/games", false)).toBe(false);
|
||||
expect(jumpsToGames("apex", "/games/730", false)).toBe(false);
|
||||
expect(jumpsToGames("", "/charts", false)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Whether typing in the header search takes the reader to /games, where the
|
||||
* box filters the table.
|
||||
*
|
||||
* Not while an input method is still composing. A Japanese, Chinese or Korean
|
||||
* reader's first keystroke is a kana or jamo they have not chosen yet, and the
|
||||
* jump used to come right then, mid-word; it now waits for compositionend.
|
||||
* Only the jump waits: the value itself is still written on every keystroke,
|
||||
* because Android keyboards compose Latin words too, and holding the value
|
||||
* back would stop the table filtering as an English reader types.
|
||||
*/
|
||||
export function jumpsToGames(search: string, pathname: string, composing: boolean): boolean {
|
||||
return !composing && search !== "" && !pathname.startsWith("/games");
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import Fuse from "fuse.js";
|
||||
import { fold, longEnough, searchKey } from "./search-fold";
|
||||
|
||||
describe("fold", () => {
|
||||
it("makes half-width, hiragana and katakana one script", () => {
|
||||
expect(fold("トロイ")).toBe("トロイ");
|
||||
expect(fold("とろい")).toBe("トロイ");
|
||||
expect(fold("トロイ")).toBe("トロイ");
|
||||
expect(fold("ゝゞ")).toBe("ヽヾ");
|
||||
});
|
||||
|
||||
it("keeps voiced kana voiced", () => {
|
||||
// Fuse's ignoreDiacritics would strip the dakuten: ガ would find カ.
|
||||
expect(fold("が")).toBe("ガ");
|
||||
expect(fold("が")).not.toBe(fold("か"));
|
||||
expect(fold("ガ")).toBe("ガ");
|
||||
});
|
||||
|
||||
it("reads full-width letters, digits and punctuation as ASCII", () => {
|
||||
expect(fold("4日後")).toBe("4日後");
|
||||
expect(fold("无主之地:银河")).toBe("无主之地:银河");
|
||||
expect(fold("APEX")).toBe("APEX");
|
||||
});
|
||||
|
||||
it("leaves ASCII exactly as it was, case included", () => {
|
||||
expect(fold("Apex Legends")).toBe("Apex Legends");
|
||||
expect(fold("")).toBe("");
|
||||
});
|
||||
|
||||
it("is a fixed point", () => {
|
||||
for (const s of ["トロイメライ", "とろいめらい", "4日後", "Counter"]) expect(fold(fold(s))).toBe(fold(s));
|
||||
});
|
||||
});
|
||||
|
||||
describe("searchKey and longEnough", () => {
|
||||
it("lowercases after folding", () => {
|
||||
expect(searchKey("APEX Legends")).toBe("apex legends");
|
||||
expect(searchKey("//トロイメライ・ノクターン").includes(searchKey("とろいめらい"))).toBe(true);
|
||||
});
|
||||
|
||||
it("lets one CJK character through where Latin needs two", () => {
|
||||
expect(longEnough("a", 2)).toBe(false);
|
||||
expect(longEnough("ab", 2)).toBe(true);
|
||||
expect(longEnough("星", 2)).toBe(true);
|
||||
expect(longEnough("と", 2)).toBe(true);
|
||||
expect(longEnough("한", 2)).toBe(true);
|
||||
expect(longEnough("", 2)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fold inside Fuse", () => {
|
||||
const records = [
|
||||
{ name: "//トロイメライ・ノクターン", tags: ["Visual Novel"] },
|
||||
{ name: "Apex Legends™", tags: ["Battle Royale"] },
|
||||
{ name: "无主之地:银河", tags: ["シューティング"] },
|
||||
];
|
||||
const fuse = new Fuse(records, {
|
||||
keys: [
|
||||
{ name: "name", getFn: (r) => fold(r.name) },
|
||||
{ name: "tags", getFn: (r) => r.tags.map(fold) },
|
||||
],
|
||||
threshold: 0.32,
|
||||
ignoreLocation: true,
|
||||
});
|
||||
const names = (q: string) => fuse.search(fold(q).trim()).map((r) => r.item.name);
|
||||
|
||||
it("finds a name typed in another script or width", () => {
|
||||
expect(names("とろい")).toEqual(["//トロイメライ・ノクターン"]);
|
||||
expect(names("トロイメライ")).toEqual(["//トロイメライ・ノクターン"]);
|
||||
expect(names("无主之地:银河")).toEqual(["无主之地:银河"]);
|
||||
});
|
||||
|
||||
it("folds array fields, and still finds English", () => {
|
||||
expect(names("しゅーてぃんぐ")).toEqual(["无主之地:银河"]);
|
||||
expect(names("legends")).toEqual(["Apex Legends™"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Folding for SEARCH only. What a reader types and what a game is called are
|
||||
* compared after both pass through here, so トロイ (half-width), とろい
|
||||
* (hiragana) and トロイ find the same game, and a full-width 4 or : matches
|
||||
* its ASCII twin. The catalogue holds games in every language since v4.6.0,
|
||||
* and a Japanese or Chinese keyboard types full-width forms by default.
|
||||
*
|
||||
* NFKC, then hiragana to katakana. Never applied to stored text: NFKC also
|
||||
* rewrites U+3000 and half-width kana, which the pipeline keeps as spelling.
|
||||
*
|
||||
* Not Fuse's `ignoreDiacritics`: its strip set includes the combining
|
||||
* (han)dakuten, which would make ガ equal カ.
|
||||
*/
|
||||
|
||||
/** ぁ..ゖ and ゝゞ sit exactly 0x60 below their katakana. */
|
||||
const KATAKANA_OFFSET = 0x60;
|
||||
|
||||
function isHiragana(code: number): boolean {
|
||||
return (code >= 0x3041 && code <= 0x3096) || code === 0x309d || code === 0x309e;
|
||||
}
|
||||
|
||||
function isAscii(text: string): boolean {
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if ((text.codePointAt(i) ?? 0) > 0x7f) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** The text in the form both sides of a search are compared in. Case is left
|
||||
* alone: Fuse lowercases both sides itself, and searchKey() does it for the
|
||||
* plain substring searches. */
|
||||
export function fold(text: string): string {
|
||||
// Nearly the whole catalogue is ASCII, which neither step changes.
|
||||
if (isAscii(text)) return text;
|
||||
let out = "";
|
||||
for (const ch of text.normalize("NFKC")) {
|
||||
const code = ch.codePointAt(0) ?? 0;
|
||||
out += isHiragana(code) ? String.fromCodePoint(code + KATAKANA_OFFSET) : ch;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** fold() and lowercase: the key a plain `includes()` search compares. */
|
||||
export function searchKey(text: string): string {
|
||||
return fold(text).toLowerCase();
|
||||
}
|
||||
|
||||
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
||||
|
||||
/** One kanji, kana or Hangul syllable is already a word; one Latin letter is
|
||||
* not. Where a search waits for two characters, a CJK one is enough. */
|
||||
export function longEnough(query: string, min: number): boolean {
|
||||
return query.length >= min || CJK.test(query);
|
||||
}
|
||||
@@ -118,6 +118,7 @@
|
||||
<ul class="mt-3 list-disc space-y-2 pl-5 text-sm text-muted-foreground">
|
||||
<li>{t("about.headsUpGenre")}</li>
|
||||
<li>{t("about.headsUpEnglish")}</li>
|
||||
<li>{t("about.headsUpLanguages")}</li>
|
||||
<li>{t("about.headsUpValve")}</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
import { acLevel, bucketByAntiCheat, type AcLevel } from "#lib/anti-cheat.js";
|
||||
import { appidOf } from "#lib/data-store.js";
|
||||
import { cn, formatNumber, parseIntSafe } from "#lib/utils.js";
|
||||
import { searchKey } from "#lib/search-fold.js";
|
||||
import type { GameRecord } from "#lib/schema.js";
|
||||
import QueryState from "#lib/common/QueryState.svelte";
|
||||
import PageHeader from "#lib/common/PageHeader.svelte";
|
||||
@@ -57,12 +58,13 @@
|
||||
let sortDir = $state<"asc" | "desc">("desc");
|
||||
|
||||
const shown = $derived.by(() => {
|
||||
const q = search.trim().toLowerCase();
|
||||
// A few hundred rows: folding them per keystroke costs nothing.
|
||||
const q = searchKey(search).trim();
|
||||
const list = rows.filter(
|
||||
(r) =>
|
||||
(!family || r.family === family) &&
|
||||
(!q ||
|
||||
(r.game.name ?? "").toLowerCase().includes(q) ||
|
||||
searchKey(r.game.name ?? "").includes(q) ||
|
||||
r.family.toLowerCase().includes(q) ||
|
||||
(r.game.anti_cheat ?? "").toLowerCase().includes(q)),
|
||||
);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { i18n } from "#lib/i18n.svelte.js";
|
||||
import { countBy } from "#lib/stats.js";
|
||||
import { formatNumber } from "#lib/utils.js";
|
||||
import { searchKey } from "#lib/search-fold.js";
|
||||
import QueryState from "#lib/common/QueryState.svelte";
|
||||
import PageHeader from "#lib/common/PageHeader.svelte";
|
||||
import Input from "#lib/ui/Input.svelte";
|
||||
@@ -16,9 +17,11 @@
|
||||
const studios = $derived(countBy(games.data?.records ?? [], (r) => r.developer));
|
||||
|
||||
let q = $state("");
|
||||
const shown = $derived(
|
||||
q.trim() ? studios.filter((s) => s.name.toLowerCase().includes(q.trim().toLowerCase())) : studios,
|
||||
);
|
||||
// Folded once per record set, not per keystroke: a name typed in another
|
||||
// width or kana script still matches (lib/search-fold.ts).
|
||||
const keys = $derived(studios.map((s) => searchKey(s.name)));
|
||||
const needle = $derived(searchKey(q).trim());
|
||||
const shown = $derived(needle ? studios.filter((_, i) => keys[i].includes(needle)) : studios);
|
||||
</script>
|
||||
|
||||
<Seo title={t("studios.developersTitle")} description={t("studios.developersSeo")} />
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
genre: filters.genre,
|
||||
typeGame: filters.typeGame,
|
||||
platform: filters.platform,
|
||||
language: filters.language,
|
||||
safe: filters.safe,
|
||||
status: filters.status,
|
||||
hasAntiCheat: filters.hasAntiCheat,
|
||||
@@ -166,6 +167,19 @@
|
||||
{/each}
|
||||
</select>
|
||||
|
||||
<!-- The catalogue lists games in every language. Names are Steam's English
|
||||
ones, as stored: data, not i18n keys. -->
|
||||
<select
|
||||
bind:value={filters.language}
|
||||
aria-label={t("games.filterAllLanguages")}
|
||||
class="h-8 rounded-md border border-input bg-background px-2 text-xs"
|
||||
>
|
||||
<option value={null}>{t("games.filterAllLanguages")}</option>
|
||||
{#each facet.language as [name, count] (name)}
|
||||
<option value={name}>{name} ({count})</option>
|
||||
{/each}
|
||||
</select>
|
||||
|
||||
<select
|
||||
bind:value={filters.status}
|
||||
aria-label={t("games.filterAllStatus")}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { i18n } from "#lib/i18n.svelte.js";
|
||||
import { countBy } from "#lib/stats.js";
|
||||
import { formatNumber } from "#lib/utils.js";
|
||||
import { searchKey } from "#lib/search-fold.js";
|
||||
import QueryState from "#lib/common/QueryState.svelte";
|
||||
import PageHeader from "#lib/common/PageHeader.svelte";
|
||||
import Input from "#lib/ui/Input.svelte";
|
||||
@@ -16,9 +17,11 @@
|
||||
const studios = $derived(countBy(games.data?.records ?? [], (r) => r.publisher));
|
||||
|
||||
let q = $state("");
|
||||
const shown = $derived(
|
||||
q.trim() ? studios.filter((s) => s.name.toLowerCase().includes(q.trim().toLowerCase())) : studios,
|
||||
);
|
||||
// Folded once per record set, not per keystroke: a name typed in another
|
||||
// width or kana script still matches (lib/search-fold.ts).
|
||||
const keys = $derived(studios.map((s) => searchKey(s.name)));
|
||||
const needle = $derived(searchKey(q).trim());
|
||||
const shown = $derived(needle ? studios.filter((_, i) => keys[i].includes(needle)) : studios);
|
||||
</script>
|
||||
|
||||
<Seo title={t("studios.publishersTitle")} description={t("studios.publishersSeo")} />
|
||||
|
||||
Reference in new issue
Block a user