Compare commits
11
Commits
android-v2.1.1
...
v4.2.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f4fdd51525 | ||
|
|
3eb960bf24 | ||
|
|
a43112b277 | ||
|
|
0cdbda1343 | ||
|
|
ebdd7073a1 | ||
|
|
d4e56eba3f | ||
|
|
e0a118ebfa | ||
|
|
11d7aaa615 | ||
|
|
858b8c8203 | ||
|
|
e5365b3b6e | ||
|
|
f997d79054 |
No files matched your search
@@ -27,6 +27,7 @@ on:
|
||||
- "Daily Snapshot"
|
||||
- "Top Offline Leaderboard"
|
||||
- "Refresh Store Data"
|
||||
- "Refresh All-Time Peaks"
|
||||
types: [completed]
|
||||
|
||||
# REQUIRED: declare permissions matching the reusable workflow's needs.
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Refresh All-Time Peaks
|
||||
|
||||
# all_time_peak - the most players a game has had at once - from SteamCharts'
|
||||
# recorded peak, a thirtieth of the catalogue a day (~186 pages, 2-3 s apart:
|
||||
# about ten minutes), so every game is asked once a month. Every run also
|
||||
# raises the field from our own Steam API samples, which needs no requests.
|
||||
# scripts/refresh_peaks.py has the rules: the field only ever goes up, and a
|
||||
# refusal from SteamCharts ends the run early with a warning, not a failure.
|
||||
#
|
||||
# 19:30 UTC sits between Refresh Store Data (16:00, ~40 minutes) and the Daily
|
||||
# Snapshot (23:00), where only the short Ingest runs are due. Scheduled runs
|
||||
# here start up to a few hours late, and the data-write group keeps only ONE
|
||||
# pending run, so this stays clear of the long morning jobs.
|
||||
#
|
||||
# The one-time backfill is a manual run with the box ticked: every game, about
|
||||
# four hours. Start it in the evening UTC, after Refresh Store Data.
|
||||
|
||||
on:
|
||||
schedule: [{cron: '30 19 * * *'}]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
backfill:
|
||||
description: 'Ask SteamCharts about every game, not just today''s slice (~4 h)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions: {contents: write}
|
||||
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
refresh:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: ${{ inputs.backfill && 330 || 30 }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-python@v6
|
||||
with: {python-version: '3.12'}
|
||||
- run: chmod +x ./bash/refresh_peaks.sh && ./bash/refresh_peaks.sh
|
||||
env:
|
||||
PEAKS_ALL: ${{ inputs.backfill }}
|
||||
@@ -4,6 +4,72 @@ All notable changes to this awesome noob repo will be documented here.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [v4.2.0] – 2026-10-03 (The "All-Time High" Edition)
|
||||
|
||||
Every game now shows the most players it has ever had at once, the catalogue
|
||||
the site and the apps keep for offline use is a fraction of its old size and
|
||||
no longer readable as text, and every open dependency alert is closed. Web
|
||||
app + desktop/Android bumped `2.1.1` → `2.2.0`; repo public-facing version
|
||||
`4.1.1` → `4.2.0`.
|
||||
|
||||
On the website all of this is already live. The desktop app offers 2.2.0
|
||||
through its updater once the release is published; on Android, install the
|
||||
new APK over the old one. The apps also pick up the Disclaimer's new
|
||||
paragraph on where player numbers come from, so they show the consent gate
|
||||
once, with that paragraph; the website already did.
|
||||
|
||||
### ✨ Added
|
||||
|
||||
- **All-time peak players.** Every game shows the most players it has ever had
|
||||
at once: "All-time peak" on its page, and a sortable column in the games
|
||||
table and the CSV export. The number is the peak Steam Charts has recorded,
|
||||
which for a game it tracks goes back to the launch, read about once a month
|
||||
per game; for the many small games Steam Charts does not track, it is the
|
||||
most this project's own samples have seen. It only ever goes up. The
|
||||
"(peak N)" beside the player count is gone: it was the highest of our own
|
||||
occasional samples since the last full re-fetch, not a daily or all-time
|
||||
peak, and it read like one. The Disclaimer names the new source, so
|
||||
returning visitors see the consent gate once, showing that paragraph.
|
||||
|
||||
### 🗜️ A smaller, opaque offline cache
|
||||
|
||||
- **The catalogue your browser or app keeps for offline use is now stored as
|
||||
the packed shards it downloads**, not as thousands of readable records:
|
||||
about 1.4 MB instead of roughly 9.5 MB, and opaque bytes in DevTools.
|
||||
Like the packed downloads, this is compression and obfuscation, not
|
||||
protection: the key is public, and the dataset stays readable in Git. The
|
||||
copy an earlier version saved keeps working until the data next changes,
|
||||
then it is replaced and deleted. Same storage keys, so the Privacy Policy
|
||||
and the consent gate are unchanged.
|
||||
- Opening the cached catalogue happens off the main thread now: on a
|
||||
desktop the longest freeze while loading it fell from about 20 ms to 6 ms,
|
||||
for about 45 ms more until the data is ready. The two cache keys are now
|
||||
written in one transaction, so a failed write can no longer leave a new
|
||||
index next to old data, and a cache that will not open is simply
|
||||
downloaded again.
|
||||
|
||||
### 🧰 Pipeline
|
||||
|
||||
- `scripts/refresh_peaks.py` and the daily "Refresh All-Time Peaks" workflow
|
||||
(19:30 UTC) ask Steam Charts about a thirtieth of the catalogue a day, a
|
||||
page every two to three seconds, and stop for the day if it ever refuses;
|
||||
every run also raises the field from the pipeline's own player counts. A
|
||||
manual run with `backfill` ticked asks about every game at once, in about
|
||||
four hours.
|
||||
|
||||
### 🔐 Security
|
||||
|
||||
- **Every open dependency alert is closed** (19 from Dependabot, plus the few
|
||||
`npm audit` adds); `npm audit` reports none. Only two of the packages run in
|
||||
production: `devalue`, whose parser is in the page and the apps (5.9.2 →
|
||||
5.9.4), and `urllib3` in the data pipeline (2.7.0 → 2.8.0). The rest are
|
||||
build and development tools: `undici` comes up to 7.29.1 with wrangler
|
||||
4.147.0, and `brace-expansion`, `fast-uri` and `serialize-javascript` are
|
||||
lockfile-only bumps. SvelteKit 2 still asks for `cookie` 0.6, whose fix is
|
||||
only in SvelteKit 3, so an npm override gives it 0.7.2 (same API; with the
|
||||
static adapter that code only runs while the site is built). The `sharp`
|
||||
override is gone: wrangler's own pin now holds the patched version.
|
||||
|
||||
## [v4.1.1] – 2026-10-03 (The "Pocket-Sized" Edition)
|
||||
|
||||
The site and the apps fit a phone: nothing runs off the right edge, game
|
||||
|
||||
@@ -80,12 +80,39 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
that is PUBLIC in this repository. It is obfuscation and compression (~15%
|
||||
of the JSONL), never secrecy - do not call it protection, and never commit a
|
||||
pack: Git stays the readable source. A pack unpacks to the exact committed
|
||||
bytes, so the sha256 contract above is checked on the UNPACKED bytes and
|
||||
`games-loader.ts` did not change. The plain `/api/data/data/*.jsonl` paths
|
||||
must keep serving released apps (1.4.5 unversioned, 2.0.x `?v=`), which
|
||||
cannot unpack. Packs end in `.bin` so the zone's `.jsonl` Compression Rule
|
||||
never recompresses them. No WASM: installed apps' CSP has no
|
||||
`'wasm-unsafe-eval'`.
|
||||
bytes, so the sha256 contract above is checked on the UNPACKED bytes; the
|
||||
pack itself is what the IndexedDB cache keeps (next bullet). The plain
|
||||
`/api/data/data/*.jsonl` paths must keep serving released apps (1.4.5
|
||||
unversioned, 2.0.x `?v=`), which cannot unpack. Packs end in `.bin` so the
|
||||
zone's `.jsonl` Compression Rule never recompresses them. No WASM: installed
|
||||
apps' CSP has no `'wasm-unsafe-eval'`.
|
||||
|
||||
- **The IndexedDB cache holds PACKS, never records or plain JSONL**
|
||||
(`web/src/lib/cache.ts`). Database `f2p-catalogue`: `f2p:index` (the
|
||||
DataIndex, readable - public metadata, so the generation compares without
|
||||
opening anything) and `f2p:records` (`Uint8Array[]`, each shard's pack
|
||||
exactly as `fetchShard` received it, in `index.files` order). Rules:
|
||||
- A generation is stored only if EVERY shard arrived packed; the browser
|
||||
never packs (compressing ~9.5 MB is main-thread work on a phone). A
|
||||
stored pack must carry its index entry's sha256 prefix as IV, or the
|
||||
cache is a miss.
|
||||
- The two key names are the ones `docs/PRIVACY_POLICY.md` lists. Renaming
|
||||
either is a policy edit, which reopens the consent gate. The new DATABASE
|
||||
is what keeps a tab still running app <= 2.1.x (which reads idb-keyval's
|
||||
default `keyval-store`) from handing Uint8Arrays to the page.
|
||||
- The legacy copy in `keyval-store` (parsed records) is read like any
|
||||
verified generation until the next one is written, which `delMany`s it -
|
||||
never `deleteDatabase`, which an old tab's open connection blocks. Look
|
||||
with `indexedDB.databases()` before touching it: opening a database
|
||||
creates it.
|
||||
- Records are decoded in the parser worker (`parsePack`), and only when the
|
||||
cached generation is actually used. A pack that will not open makes the
|
||||
whole generation a miss. The worker must never reply with an empty
|
||||
`error`: Chromium's failed AES-GCM decrypt is a DOMException with an
|
||||
EMPTY message, and `if (error)` once resolved a damaged pack as a shard
|
||||
with no records (`worker-pool.test.ts`, `jsonl-parser.test.ts`).
|
||||
- Obfuscation and compression (~1.4 MB stored instead of ~9.5 MB of
|
||||
objects), never protection - the key is public, like the wire packs'.
|
||||
|
||||
- **`index.json` is rebuilt from scratch on every save.** `_save_index()` in
|
||||
`scripts/core/data_store.py` writes it from a fixed set of keys, so any key
|
||||
@@ -117,6 +144,22 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
no language table or tags), compares `description` as stored (truncated),
|
||||
and bumps `last_updated` only on a real change. A new field added to it must
|
||||
obey the same rules; `scripts/tests/test_refresh_store_data.py` holds them.
|
||||
- **`all_time_peak` only ever goes UP, and `refresh_peaks.py` is what raises
|
||||
it** (daily 19:30 UTC, "Refresh All-Time Peaks"). A `crc32(appid) % 30`
|
||||
slice (`core/rotation.py`, shared with `refresh_store_data.py`) reads
|
||||
SteamCharts' recorded all-time peak - the only source older than our own
|
||||
sampling: SteamDB forbids scraping and Valve's APIs know only today. Every
|
||||
run also raises it from our samples (`peak_today`, which despite its name is
|
||||
never reset, only cleared by `refetch_all.py`, and `current_players`), and
|
||||
`apply_players()` raises it on every sample, never from a 0. SteamCharts has
|
||||
no API, terms or robots.txt, so: one page per game a month, 2-3 s apart, a
|
||||
User-Agent that names the project; 500 is an app it does not track (most of
|
||||
the long tail), 404 one it has never seen; a 403, 429 or challenge page
|
||||
stops the run with a warning, not a failure. `refetch_all.py` must never
|
||||
clear the field - it cannot be fetched again - and nothing may lower it.
|
||||
The backfill is a manual run with `backfill` ticked (~4 h, start it in the
|
||||
evening UTC). DISCLAIMER section 6 names the source, so rewording it reopens
|
||||
the consent gate. `scripts/tests/test_refresh_peaks.py` holds the rules.
|
||||
- `ingest_new.py` only reads `scripts/temp_info.jsonl`. It has exactly two
|
||||
producers now: the browser extension (`poli0981/steam-f2p-extension`, which
|
||||
pushes to the file directly) and the `/admin` approve flow (the Worker
|
||||
@@ -272,6 +315,14 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
- `echarts-wordcloud` declares a stale `echarts@^5` peer. It runs fine on
|
||||
echarts 6 (all the legacy APIs it uses are still exported), so `package.json`
|
||||
carries an `overrides` entry. Do not "fix" it by pinning echarts back to 5.
|
||||
- **The `"@sveltejs/kit@2.70.3": { "cookie": "^0.7.2" }` override closes
|
||||
GHSA-pxg6-pf52-xh8x** (kit 2 declares `cookie ^0.6.0`; only kit 3 moved on).
|
||||
Kit imports just `parse`/`serialize`, which 0.7 keeps, and under
|
||||
adapter-static that code runs only at prerender and in `vite dev`. The key
|
||||
names kit's EXACT version because npm refuses a range key for a direct
|
||||
dependency (`EOVERRIDE`). A kit bump therefore silently drops the override
|
||||
and the alert comes back: re-key it, or delete it once kit asks for
|
||||
`cookie` >= 0.7. Never carry it onto kit 3, which needs `cookie` 2.
|
||||
- **Real paths everywhere, including the packaged apps.** There is no
|
||||
HashRouter and no hash shim: `tauri::manager::get_asset()` falls back through
|
||||
`<path>.html`, `<path>/index.html`, then `index.html`, so the Tauri webview
|
||||
|
||||
@@ -148,11 +148,15 @@ scripts/
|
||||
│ ├── overrides.py # Human corrections re-applied on every write
|
||||
│ ├── fetcher.py # Batch processor, apply_details/reviews/players/scraped
|
||||
│ ├── scraper.py # Store page HTML parser (languages, DLC, tags)
|
||||
│ ├── steamcharts.py # SteamCharts all-time peak reader (polite, stops on refusal)
|
||||
│ ├── peaks.py # all_time_peak: only ever raised
|
||||
│ ├── rotation.py # crc32 daily slices shared by the refresh jobs
|
||||
│ └── health_checker.py # Game status detection with cached API data
|
||||
├── discover_new.py # Propose new games into the /admin review queue
|
||||
├── ingest_new.py # Add queued games (health check + extension merge)
|
||||
├── update_data.py # Daily: fill in incomplete records
|
||||
├── refresh_store_data.py # Daily rotation: tags/languages weekly, store details monthly
|
||||
├── refresh_peaks.py # Daily rotation: all-time peak from SteamCharts, monthly per game
|
||||
├── update_reviews.py # Reviews-only (every 2 days)
|
||||
├── mark_dead_games.py # Flag online games with no players
|
||||
├── check_dead_links.py # HEAD-request 404/410 scanner
|
||||
@@ -176,6 +180,7 @@ scripts/
|
||||
| Top Offline | 1st + 15th of month 05:00 UTC | Leaderboard for single-player F2P concurrents |
|
||||
| Update Reviews | Every 2 days 06:00 UTC | Reviews-only refresh |
|
||||
| Refresh Store Data | Daily 16:00 UTC | Rotating re-fetch: tags, languages, DLC flag weekly; name, image, description, developer/publisher, release date, platforms, Metacritic, DRM monthly |
|
||||
| Refresh All-Time Peaks | Daily 19:30 UTC | `all_time_peak` from SteamCharts, every game once a month, plus our own samples; manual run with `backfill` asks about every game |
|
||||
| Check Dead Links | Every 5 days 04:00 UTC | HEAD-request 404/410 scanner |
|
||||
| Purge Unhealthy | Weekly Mon 05:00 UTC | Full health scan + removal |
|
||||
| Mark Dead Games | Mon + Thu 04:30 UTC | Online + >1y + 0 players ≥14d → `is_dead=true` + 💀 note |
|
||||
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
pip install --quiet -r requirements.txt
|
||||
# PEAKS_ALL is the manual run's "backfill" box: "true" asks SteamCharts about
|
||||
# every game (~4 h) instead of today's thirtieth. A scheduled run leaves it
|
||||
# empty, and an unticked box sends "false" - neither may pass --all.
|
||||
args=()
|
||||
if [ "${PEAKS_ALL:-}" = "true" ]; then
|
||||
args+=(--all)
|
||||
fi
|
||||
python scripts/refresh_peaks.py ${args[@]+"${args[@]}"}
|
||||
bash "$(dirname "$0")/commit_push.sh" "Refresh all-time peaks [$(date +'%Y-%m-%d')]"
|
||||
+800
-800
File diff suppressed because it is too large.
Load diff
+800
-800
File diff suppressed because it is too large.
Load diff
+800
-800
File diff suppressed because it is too large.
Load diff
+800
-800
File diff suppressed because it is too large.
Load diff
+800
-800
File diff suppressed because it is too large.
Load diff
+800
-800
File diff suppressed because it is too large.
Load diff
+768
-768
File diff suppressed because it is too large.
Load diff
+8
-8
@@ -1,42 +1,42 @@
|
||||
{
|
||||
"max_per_file": 800,
|
||||
"total": 5568,
|
||||
"last_updated": "2026-10-02T16:52:55Z",
|
||||
"last_updated": "2026-10-03T09:29:13Z",
|
||||
"files": [
|
||||
{
|
||||
"name": "data_001.jsonl",
|
||||
"count": 800,
|
||||
"sha256": "dbec6dc21f38d1a661295db959c18bfdbba148260139eb706d768d3e1a8b627c"
|
||||
"sha256": "70b573a98bf88baea02ebe52dd3a4eba40f95c77258f33c942366ef1ccdb8e67"
|
||||
},
|
||||
{
|
||||
"name": "data_002.jsonl",
|
||||
"count": 800,
|
||||
"sha256": "c3a7e8701c029111133bd0c91bf4b4a84d4818a60afecab9c742cd7fd4430456"
|
||||
"sha256": "ef93e00a77546302fb617d3c0de133820863cfd8ac360c7783a788a3f66f6951"
|
||||
},
|
||||
{
|
||||
"name": "data_003.jsonl",
|
||||
"count": 800,
|
||||
"sha256": "17e8467f3d284490e6369534c810d04ca7d9bdf29ee57e8624fb5fa0ce69b000"
|
||||
"sha256": "fe1496cb80ddfa60967d48b0698b32b5165023dd3c083b6c8f02aaf4d61911a6"
|
||||
},
|
||||
{
|
||||
"name": "data_004.jsonl",
|
||||
"count": 800,
|
||||
"sha256": "7378758ce3b3680a84c4610b549d84a69305c90f09c36b7166589ebe6871678c"
|
||||
"sha256": "4c11ce7bbeb960658637f4e4557df2c41aa5b039a5785c68ec370ca35eeee2ba"
|
||||
},
|
||||
{
|
||||
"name": "data_005.jsonl",
|
||||
"count": 800,
|
||||
"sha256": "542ccfcc27843e7214bf69fca8cbe9a38dc712e4d28b6ac60f0051f764277024"
|
||||
"sha256": "b6a75d641afd238234fecda6965144bc31bc0f73107396d85a8b14f1a4b50bd7"
|
||||
},
|
||||
{
|
||||
"name": "data_006.jsonl",
|
||||
"count": 800,
|
||||
"sha256": "9f8bc7c8601d2051ca55e0c96cd97a844de74b80ff8e97b29a3ab89777392510"
|
||||
"sha256": "fdef56d6cd86b388647287e9427887ebb870d1df6f95c4d763f3057b18a49a81"
|
||||
},
|
||||
{
|
||||
"name": "data_007.jsonl",
|
||||
"count": 768,
|
||||
"sha256": "fe34e13dcbb005f9001423942473308b680fbeacfb63720a4fd87164fcce2ea8"
|
||||
"sha256": "4c590de21a1dec7b8e38e069c5f1f950fea2270a61ae98db243c9c3ab96e67ae"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -6,6 +6,7 @@ Big thanks to everyone and everything that made this chaotic repo possible.
|
||||
|
||||
- **Valve / Steam** — for existing and having a ton of free-to-play games that keep broke people like the maintainer entertained. The Steam logo, store, community, VAC, and all related marks are trademarks of Valve Corporation; this project is independent and not affiliated.
|
||||
- **GitHub** — for free hosting (data, Pages, Actions, Releases), free CI for the Python pipeline + the web app deploy + the desktop release matrix builds, and for letting unemployed devs pretend they're productive.
|
||||
- **Steam Charts** ([steamcharts.com](https://steamcharts.com/)) — for the all-time peak player counts. The pipeline reads each game's recorded peak from its public page, about once a month per game and a few seconds apart. Not affiliated with this project.
|
||||
- **The Steam community & various review aggregators** — for reviews, tags, and metadata that the pipeline scrapes off public store pages. Where copy-paste happens it stays attributed.
|
||||
- **Caffeine & instant noodles** — the real MVPs keeping the maintainer awake during update sessions.
|
||||
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ These are the known classes of error you should expect when relying on the data:
|
||||
`anti_cheat`, `is_kernel_ac`, and `anti_cheat_note` are populated by matching known AC names (VAC, EAC, BattlEye, Vanguard, …) against the Steam page text. If a game ships an AC under a vendor-renamed brand or only mentions it in a launcher EULA, the field will be `-`. Cross-check before relying on this for kernel-driver risk decisions.
|
||||
|
||||
6. **Player counts are sampled, not realtime.**
|
||||
`current_players` and `peak_today` come from periodic Steam Web API calls and lag by minutes-to-hours.
|
||||
`current_players` and `peak_today` come from periodic Steam Web API calls and lag by minutes-to-hours. `all_time_peak` is the higher of the all-time peak [Steam Charts](https://steamcharts.com/) recorded for the game, read about once a month, and the highest of those samples; it only ever goes up. Steam Charts samples periodically too, so a peak can sit a little under what Steam really reached, and it does not track every game: for many small ones the figure is only the most this project has seen, not a peak since launch.
|
||||
|
||||
7. **Malware / safety flags are not antivirus output.**
|
||||
The `safe` field is a manual `y / n / ?` set by the maintainer based on personal experience or third-party reports. Run your own antivirus.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Exact pins, not ranges: the data pipeline should install the same versions
|
||||
# every run, and Dependabot only raises PRs for pinned dependencies.
|
||||
requests==2.34.2
|
||||
urllib3==2.7.0
|
||||
urllib3==2.8.0
|
||||
@@ -341,6 +341,8 @@ _SKELETON_TEMPLATE = {
|
||||
"genre": "", "type_game": "", "has_paid_dlc": False,
|
||||
"developer": [], "publisher": [], "release_date": "",
|
||||
"reviews": "N/A", "current_players": "N/A", "peak_today": "N/A",
|
||||
# Only ever raised, never cleared: see core/peaks.py.
|
||||
"all_time_peak": "N/A",
|
||||
"metacritic": "N/A",
|
||||
"anti_cheat": "-", "anti_cheat_note": "", "is_kernel_ac": None,
|
||||
"platforms": [], "languages": [], "language_details": [], "tags": [],
|
||||
|
||||
@@ -16,6 +16,7 @@ from .constants import (
|
||||
)
|
||||
from .steam_client import get_client
|
||||
from .data_store import extract_appid, now_iso, is_info_complete, is_empty
|
||||
from .peaks import raise_peak
|
||||
from .scraper import scrape_store_page
|
||||
|
||||
|
||||
@@ -157,6 +158,9 @@ def apply_reviews(game: dict, summary: dict) -> dict:
|
||||
|
||||
def apply_players(game: dict, count: int) -> dict:
|
||||
game["current_players"] = f"{count:,}"
|
||||
# A sample of 0 says nothing about a peak; anything else may beat it.
|
||||
if count > 0:
|
||||
raise_peak(game, count)
|
||||
old = game.get("peak_today", "N/A")
|
||||
if old in ("N/A", "Error", ""):
|
||||
game["peak_today"] = f"{count:,}"
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
all_time_peak: the most players a game has had at once, as far as anyone
|
||||
recorded.
|
||||
|
||||
Two sources raise it, and nothing ever lowers it:
|
||||
- SteamCharts' recorded all-time peak (refresh_peaks.py). It goes back to
|
||||
when SteamCharts started tracking the game, which for anything that ever
|
||||
had a crowd is its launch.
|
||||
- Our own Steam Web API samples: apply_players() raises it whenever a sample
|
||||
beats it, and peak_today - the highest sample since the last refetch, never
|
||||
reset daily despite its name - is a floor too.
|
||||
|
||||
A formatted string like current_players ("1,818,368"); "N/A" until either
|
||||
source has a number. refetch_all.py must never clear it: unlike the other
|
||||
player fields it cannot be fetched again.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
FIELD = "all_time_peak"
|
||||
|
||||
|
||||
def parse_count(value) -> Optional[int]:
|
||||
"""A stored player count ("1,818,368", 1818368) as an int; None for "N/A",
|
||||
"Error", "" and anything else that is not a count."""
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value if value >= 0 else None
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
digits = value.replace(",", "").strip()
|
||||
return int(digits) if digits.isdigit() else None
|
||||
|
||||
|
||||
def format_count(count: int) -> str:
|
||||
return f"{count:,}"
|
||||
|
||||
|
||||
def raise_peak(game: dict, count: Optional[int]) -> bool:
|
||||
"""Set all_time_peak to `count` if that is higher than what is stored.
|
||||
Returns True when it changed. A count of 0 is accepted here (SteamCharts
|
||||
does record games whose peak was 0); callers holding only our own samples
|
||||
skip 0, which says nothing about a peak."""
|
||||
if count is None or count < 0:
|
||||
return False
|
||||
stored = parse_count(game.get(FIELD))
|
||||
if stored is not None and count <= stored:
|
||||
return False
|
||||
game[FIELD] = format_count(count)
|
||||
return True
|
||||
|
||||
|
||||
def observed_peak(game: dict) -> Optional[int]:
|
||||
"""The highest count our own samples recorded for this game, or None when
|
||||
they never saw a single player."""
|
||||
samples = (parse_count(game.get("peak_today")), parse_count(game.get("current_players")))
|
||||
return max((s for s in samples if s), default=None)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""
|
||||
The daily rotation the refresh jobs share: each game is visited on one fixed
|
||||
day of a cycle, so a daily run touches 1/cycle of the catalogue.
|
||||
|
||||
A game's slice is crc32(appid) % cycle, NOT appid % cycle: Steam appids are
|
||||
overwhelmingly multiples of 10, so `% 30` would leave 27 of the 30 monthly
|
||||
slices empty and pile the whole catalogue into the other three.
|
||||
"""
|
||||
import zlib
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from .data_store import extract_appid
|
||||
|
||||
|
||||
def slice_of(appid: str, cycle: int) -> int:
|
||||
"""The day of the cycle on which this game is refreshed. Stable forever:
|
||||
crc32 is fixed, so a game keeps its weekday and its day of the month."""
|
||||
return zlib.crc32(appid.encode("ascii")) % cycle
|
||||
|
||||
|
||||
def day_number(today: Optional[date] = None) -> int:
|
||||
"""Days since 1970-01-01, UTC. Consecutive runs walk through the slices."""
|
||||
today = today or datetime.now(timezone.utc).date()
|
||||
return (today - date(1970, 1, 1)).days
|
||||
|
||||
|
||||
def due(game: dict, cycle: int, day: int) -> bool:
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
return bool(appid) and slice_of(appid, cycle) == day % cycle
|
||||
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
SteamCharts (steamcharts.com): the only public record of a game's all-time
|
||||
peak that goes back further than this project's own sampling. SteamDB has the
|
||||
same numbers and forbids scraping; Valve's APIs only know today.
|
||||
|
||||
It has no API, no terms of use and no robots.txt (a 404 when checked on
|
||||
2026-10-03), and it is served through Cloudflare. So this reads one page per
|
||||
game, at most once a month per game (refresh_peaks.py), 2-3 seconds apart,
|
||||
with a User-Agent that names the project, and gives up for the day at the
|
||||
first sign of being unwelcome: a 403, a 429 or a Cloudflare challenge page.
|
||||
|
||||
What its pages answer, as measured on 2026-10-03:
|
||||
200 with an "all-time peak" stat the number
|
||||
500 an app it does not track - most of the
|
||||
long tail, games that never had a crowd
|
||||
404 an appid it has never seen (new games)
|
||||
"""
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
import requests
|
||||
|
||||
URL = "https://steamcharts.com/app/{appid}"
|
||||
USER_AGENT = "SteamF2PTracker/2.2 (+https://github.com/poli0981/free-steam-games-list)"
|
||||
DELAY_MIN = 2.0
|
||||
DELAY_MAX = 3.0
|
||||
|
||||
_PEAK = re.compile(
|
||||
r'<span class="num">\s*([0-9][0-9,]*)\s*</span>\s*<br\s*/?>\s*all-time peak',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# What Cloudflare serves instead of the page when it wants a browser.
|
||||
_CHALLENGE_MARKERS = ("cf-chl", "challenge-platform", "<title>Just a moment...</title>")
|
||||
|
||||
|
||||
def parse_all_time_peak(html: str) -> Optional[int]:
|
||||
"""The "all-time peak" stat of an app page, or None if the page has none."""
|
||||
m = _PEAK.search(html or "")
|
||||
return int(m.group(1).replace(",", "")) if m else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
"""status: "ok" (peak set), "untracked" (500/404: SteamCharts has no data),
|
||||
"blocked" (403/429/challenge: stop for the day) or "error" (network trouble,
|
||||
another status, or a 200 page without the stat - a changed layout)."""
|
||||
status: str
|
||||
peak: Optional[int] = None
|
||||
|
||||
|
||||
class SteamChartsClient:
|
||||
def __init__(
|
||||
self,
|
||||
session: Optional[requests.Session] = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self._session = session or requests.Session()
|
||||
self._session.headers.update({"User-Agent": USER_AGENT, "Accept-Language": "en"})
|
||||
self._sleep = sleep
|
||||
self._clock = clock
|
||||
self._last: Optional[float] = None
|
||||
|
||||
def _throttle(self) -> None:
|
||||
if self._last is not None:
|
||||
wait = random.uniform(DELAY_MIN, DELAY_MAX) - (self._clock() - self._last)
|
||||
if wait > 0:
|
||||
self._sleep(wait)
|
||||
self._last = self._clock()
|
||||
|
||||
def fetch(self, appid: str) -> Result:
|
||||
self._throttle()
|
||||
try:
|
||||
resp = self._session.get(URL.format(appid=appid), timeout=20)
|
||||
except requests.RequestException:
|
||||
return Result("error")
|
||||
code = resp.status_code
|
||||
if code in (403, 429):
|
||||
return Result("blocked")
|
||||
if code in (404, 500):
|
||||
return Result("untracked")
|
||||
if code != 200:
|
||||
return Result("error")
|
||||
text = resp.text
|
||||
peak = parse_all_time_peak(text)
|
||||
if peak is not None:
|
||||
return Result("ok", peak)
|
||||
# Only a page WITHOUT the stat can be a challenge: Cloudflare may also
|
||||
# inject its challenge-platform script into ordinary pages.
|
||||
if any(marker in text for marker in _CHALLENGE_MARKERS):
|
||||
return Result("blocked")
|
||||
return Result("error")
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Refresh all_time_peak - the most players each game has had at once - from
|
||||
SteamCharts, a thirtieth of the catalogue a day.
|
||||
|
||||
core/peaks.py explains the field; core/steamcharts.py the source and why it is
|
||||
read so carefully. Each run does two things:
|
||||
|
||||
1. For EVERY game, raise all_time_peak to the highest of our own samples
|
||||
(peak_today and current_players). No requests; this is what gives the
|
||||
long tail SteamCharts does not track a number at all, and it seeds the
|
||||
field for the whole catalogue on the first run.
|
||||
2. For the games whose slice is due (crc32(appid) % 30, see core/rotation.py:
|
||||
every game once a month), read SteamCharts' recorded all-time peak and
|
||||
raise the field to it.
|
||||
|
||||
The rules:
|
||||
- all_time_peak only ever goes UP. A failed request, an untracked app or a
|
||||
lower number leaves the stored value alone.
|
||||
- Nothing else is written: not the MANUAL_FIELDS, not notes, and not
|
||||
last_updated, which player counts have never moved either.
|
||||
- SteamCharts answers 500 for apps it does not track and 404 for ones it has
|
||||
never seen; both are normal and simply skipped until next month.
|
||||
- A 403, a 429 or a Cloudflare challenge stops the SteamCharts part of the
|
||||
run (everything already raised is kept and saved), and so do ten errors in
|
||||
a row. The run still succeeds, with a warning: a refusal is SteamCharts'
|
||||
call to make, not a pipeline failure.
|
||||
- Everything is written by save_main().
|
||||
|
||||
Usage (from the repository root):
|
||||
python scripts/refresh_peaks.py today's slice
|
||||
python scripts/refresh_peaks.py --all every game (the backfill, ~4 h)
|
||||
python scripts/refresh_peaks.py --day 20730 the slice of a given day
|
||||
python scripts/refresh_peaks.py --limit 5 --dry-run
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.data_store import extract_appid, load_main, migrate_record, save_main
|
||||
from core.peaks import observed_peak, raise_peak
|
||||
from core.rotation import day_number, due
|
||||
from core.steamcharts import SteamChartsClient
|
||||
|
||||
PEAK_CYCLE_DAYS = 30
|
||||
MAX_CONSECUTIVE_ERRORS = 10
|
||||
|
||||
|
||||
def raise_from_samples(games: list[dict]) -> int:
|
||||
"""Step 1. Returns how many games' all_time_peak went up."""
|
||||
return sum(1 for g in games if raise_peak(g, observed_peak(g)))
|
||||
|
||||
|
||||
def raise_from_steamcharts(targets: list[dict], client, log=print) -> Counter:
|
||||
"""Step 2. Returns a tally of the outcomes: "raised", "unchanged",
|
||||
"untracked", "error", and "skipped" for games never asked because the run
|
||||
stopped early."""
|
||||
tally: Counter = Counter()
|
||||
errors_in_a_row = 0
|
||||
for i, game in enumerate(targets):
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if not appid:
|
||||
continue
|
||||
result = client.fetch(appid)
|
||||
if result.status == "blocked":
|
||||
tally["skipped"] += len(targets) - i
|
||||
log(f"::warning::SteamCharts refused the request for app {appid}; "
|
||||
f"stopping for today with {len(targets) - i} games left.")
|
||||
break
|
||||
if result.status == "error":
|
||||
tally["error"] += 1
|
||||
errors_in_a_row += 1
|
||||
if errors_in_a_row >= MAX_CONSECUTIVE_ERRORS:
|
||||
tally["skipped"] += len(targets) - i - 1
|
||||
log(f"::warning::{MAX_CONSECUTIVE_ERRORS} SteamCharts errors in a row "
|
||||
f"(a changed page layout?); stopping for today.")
|
||||
break
|
||||
continue
|
||||
errors_in_a_row = 0
|
||||
if result.status == "untracked":
|
||||
tally["untracked"] += 1
|
||||
continue
|
||||
before = game.get("all_time_peak")
|
||||
if raise_peak(game, result.peak):
|
||||
tally["raised"] += 1
|
||||
log(f" {appid} {game.get('name', '')[:40]}: {before} -> {game['all_time_peak']}")
|
||||
else:
|
||||
tally["unchanged"] += 1
|
||||
return tally
|
||||
|
||||
|
||||
def main(argv: Optional[list[str]] = None) -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip())
|
||||
ap.add_argument("--all", action="store_true", help="every game, not just today's slice (the backfill)")
|
||||
ap.add_argument("--day", type=int, help="day number to take the slice of (default: today, UTC)")
|
||||
ap.add_argument("--limit", type=int, help="ask SteamCharts about at most this many games")
|
||||
ap.add_argument("--dry-run", action="store_true", help="fetch and report, write nothing")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
games = load_main()
|
||||
for g in games:
|
||||
migrate_record(g)
|
||||
|
||||
from_samples = raise_from_samples(games)
|
||||
print(f"{len(games)} games; all_time_peak raised from our own samples for {from_samples}")
|
||||
|
||||
day = day_number() if args.day is None else args.day
|
||||
targets = games if args.all else [g for g in games if due(g, PEAK_CYCLE_DAYS, day)]
|
||||
if args.limit is not None:
|
||||
targets = targets[: args.limit]
|
||||
scope = "every game" if args.all else f"slice {day % PEAK_CYCLE_DAYS}/{PEAK_CYCLE_DAYS}"
|
||||
print(f"SteamCharts: {len(targets)} games ({scope})")
|
||||
|
||||
tally = raise_from_steamcharts(targets, SteamChartsClient()) if targets else Counter()
|
||||
summary = ", ".join(f"{k} {v}" for k, v in tally.most_common()) or "nothing asked"
|
||||
print(f"\nSteamCharts: {summary}")
|
||||
|
||||
if args.dry_run:
|
||||
print("Dry run: nothing written.")
|
||||
return
|
||||
save_main(games)
|
||||
print(f"✓ Saved {len(games)} games")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -48,9 +48,7 @@ Usage (from the repository root):
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
@@ -59,6 +57,8 @@ from core.data_store import (
|
||||
extract_appid, load_main, migrate_record, now_iso, save_main, truncate_description,
|
||||
)
|
||||
from core.fetcher import process_batch
|
||||
# The rotation (crc32 slices) is shared with refresh_peaks.py.
|
||||
from core.rotation import day_number, due
|
||||
from core.scraper import scrape_store_page
|
||||
from core.steam_client import get_client
|
||||
|
||||
@@ -72,25 +72,6 @@ DETAILS_FIELDS = (
|
||||
)
|
||||
|
||||
|
||||
# ──────────── Rotation ────────────
|
||||
|
||||
def slice_of(appid: str, cycle: int) -> int:
|
||||
"""The day of the cycle on which this game is refreshed. Stable forever:
|
||||
crc32 is fixed, so a game keeps its weekday and its day of the month."""
|
||||
return zlib.crc32(appid.encode("ascii")) % cycle
|
||||
|
||||
|
||||
def day_number(today: Optional[date] = None) -> int:
|
||||
"""Days since 1970-01-01, UTC. Consecutive runs walk through the slices."""
|
||||
today = today or datetime.now(timezone.utc).date()
|
||||
return (today - date(1970, 1, 1)).days
|
||||
|
||||
|
||||
def due(game: dict, cycle: int, day: int) -> bool:
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
return bool(appid) and slice_of(appid, cycle) == day % cycle
|
||||
|
||||
|
||||
# ──────────── Applying fresh values ────────────
|
||||
|
||||
def _replace(game: dict, field: str, value, changed: list[str]) -> None:
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
all_time_peak is a number this project can never fetch again if it loses one,
|
||||
so these pin the rules that keep it safe: it only ever goes up, a refusal or an
|
||||
error from SteamCharts changes nothing, our own samples count but a sample of
|
||||
0 does not, and nothing but all_time_peak is ever written.
|
||||
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
import refetch_all
|
||||
import refresh_peaks as rp
|
||||
from core import steamcharts
|
||||
from core.data_store import make_skeleton
|
||||
from core.fetcher import apply_players
|
||||
from core.peaks import observed_peak, parse_count, raise_peak
|
||||
from core.steamcharts import Result, SteamChartsClient, parse_all_time_peak
|
||||
|
||||
# The markup of a SteamCharts app page, as served on 2026-10-03 (tabs and all).
|
||||
STATS = (
|
||||
'<div id="app-heading" class="content">\n'
|
||||
'\t<div class="app-stat"> \n\t\t<span class="num">76135</span>\n'
|
||||
'\t\t<br>playing <abbr class="timeago" title="2026-10-02T23:02:03Z"></abbr>\n\t</div>\n'
|
||||
'\t<div class="app-stat"> \n\t\t<span class="num">89465</span>\n\t\t<br>24-hour peak\n\t</div>\n'
|
||||
'\t<div class="app-stat"> \n\t\t<span class="num">253225</span>\n\t\t<br>all-time peak\n\t</div>\n'
|
||||
'</div>'
|
||||
)
|
||||
|
||||
|
||||
def _game(appid: str = "440", **fields) -> dict:
|
||||
base = {
|
||||
"link": f"https://store.steampowered.com/app/{appid}/",
|
||||
"name": "Team Fortress 2",
|
||||
"genre": "FPS",
|
||||
"type_game": "online",
|
||||
"notes": "Not reviewed yet",
|
||||
"current_players": "61,531",
|
||||
"peak_today": "61,531",
|
||||
"all_time_peak": "N/A",
|
||||
"last_updated": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
base.update(fields)
|
||||
return base
|
||||
|
||||
|
||||
# ──────────── Parsing the page ────────────
|
||||
|
||||
def test_reads_the_all_time_peak_not_the_other_stats():
|
||||
assert parse_all_time_peak(STATS) == 253225
|
||||
|
||||
|
||||
def test_reads_a_peak_written_with_thousands_separators():
|
||||
assert parse_all_time_peak('<span class="num">1,818,368</span><br>all-time peak') == 1818368
|
||||
|
||||
|
||||
def test_a_page_without_the_stat_has_no_peak():
|
||||
assert parse_all_time_peak("<html><title>Steam Charts</title></html>") is None
|
||||
assert parse_all_time_peak("") is None
|
||||
|
||||
|
||||
# ──────────── What each answer means ────────────
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code: int, text: str = ""):
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, answer):
|
||||
self.answer = answer
|
||||
self.headers: dict = {}
|
||||
self.urls: list[str] = []
|
||||
|
||||
def get(self, url, timeout):
|
||||
self.urls.append(url)
|
||||
if isinstance(self.answer, Exception):
|
||||
raise self.answer
|
||||
return self.answer
|
||||
|
||||
|
||||
def _fetch(answer) -> Result:
|
||||
return SteamChartsClient(session=_Session(answer), sleep=lambda s: None).fetch("440")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("answer, expected", [
|
||||
(_Resp(200, STATS), Result("ok", 253225)),
|
||||
(_Resp(500), Result("untracked")),
|
||||
(_Resp(404), Result("untracked")),
|
||||
(_Resp(403), Result("blocked")),
|
||||
(_Resp(429), Result("blocked")),
|
||||
(_Resp(200, "<html><title>Just a moment...</title><script>cf-chl</script></html>"), Result("blocked")),
|
||||
(_Resp(200, "<html><body>a page we do not recognise</body></html>"), Result("error")),
|
||||
(_Resp(502), Result("error")),
|
||||
(requests.ConnectionError("down"), Result("error")),
|
||||
])
|
||||
def test_each_answer_maps_to_one_outcome(answer, expected):
|
||||
assert _fetch(answer) == expected
|
||||
|
||||
|
||||
def test_a_real_page_that_also_carries_cloudflare_script_is_still_read():
|
||||
page = STATS + '<script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script>'
|
||||
assert _fetch(_Resp(200, page)) == Result("ok", 253225)
|
||||
|
||||
|
||||
def test_requests_are_at_least_two_seconds_apart_and_name_the_project():
|
||||
now = [100.0]
|
||||
slept: list[float] = []
|
||||
|
||||
def sleep(s):
|
||||
slept.append(s)
|
||||
now[0] += s
|
||||
|
||||
session = _Session(_Resp(500))
|
||||
client = SteamChartsClient(session=session, sleep=sleep, clock=lambda: now[0])
|
||||
client.fetch("1")
|
||||
client.fetch("2")
|
||||
assert slept and slept[0] >= steamcharts.DELAY_MIN
|
||||
assert "free-steam-games-list" in session.headers["User-Agent"]
|
||||
assert session.urls == ["https://steamcharts.com/app/1", "https://steamcharts.com/app/2"]
|
||||
|
||||
|
||||
# ──────────── The field only goes up ────────────
|
||||
|
||||
def test_parse_count_reads_only_counts():
|
||||
assert parse_count("1,818,368") == 1818368
|
||||
assert parse_count(42) == 42
|
||||
for junk in ("N/A", "Error", "", None, True, -3, "12a"):
|
||||
assert parse_count(junk) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stored, count, after", [
|
||||
("N/A", 1000, "1,000"),
|
||||
("N/A", 0, "0"), # SteamCharts does record games that peaked at 0
|
||||
("1,000", 2000, "2,000"),
|
||||
("1,000", 1000, "1,000"),
|
||||
("1,000", 999, "1,000"),
|
||||
("1,000", None, "1,000"),
|
||||
("Error", 5, "5"),
|
||||
])
|
||||
def test_raise_peak_never_lowers(stored, count, after):
|
||||
game = {"all_time_peak": stored}
|
||||
raise_peak(game, count)
|
||||
assert game["all_time_peak"] == after
|
||||
|
||||
|
||||
def test_our_samples_count_but_a_zero_does_not():
|
||||
assert observed_peak({"peak_today": "8,052", "current_players": "3,452"}) == 8052
|
||||
assert observed_peak({"peak_today": "0", "current_players": "0"}) is None
|
||||
assert observed_peak({"peak_today": "N/A", "current_players": "N/A"}) is None
|
||||
|
||||
|
||||
def test_a_player_sample_raises_the_peak_but_never_lowers_it():
|
||||
game = _game(all_time_peak="N/A", peak_today="N/A")
|
||||
apply_players(game, 5000)
|
||||
assert game["all_time_peak"] == "5,000"
|
||||
apply_players(game, 20)
|
||||
assert game["all_time_peak"] == "5,000"
|
||||
|
||||
|
||||
def test_a_sample_of_zero_does_not_invent_a_peak():
|
||||
game = _game(all_time_peak="N/A", peak_today="N/A")
|
||||
apply_players(game, 0)
|
||||
assert game["all_time_peak"] == "N/A"
|
||||
|
||||
|
||||
# ──────────── A run ────────────
|
||||
|
||||
class _Client:
|
||||
def __init__(self, results: dict):
|
||||
self.results = results
|
||||
self.asked: list[str] = []
|
||||
|
||||
def fetch(self, appid):
|
||||
self.asked.append(appid)
|
||||
return self.results[appid]
|
||||
|
||||
|
||||
def test_samples_seed_every_game_without_a_request():
|
||||
games = [
|
||||
_game("1", current_players="3,452", peak_today="8,052"),
|
||||
_game("2", current_players="0", peak_today="0"),
|
||||
]
|
||||
assert rp.raise_from_samples(games) == 1
|
||||
assert games[0]["all_time_peak"] == "8,052"
|
||||
assert games[1]["all_time_peak"] == "N/A"
|
||||
|
||||
|
||||
def test_steamcharts_raises_and_skips_what_it_does_not_know():
|
||||
games = [_game("1"), _game("2"), _game("3", all_time_peak="300,000")]
|
||||
client = _Client({"1": Result("ok", 253225), "2": Result("untracked"), "3": Result("ok", 264860)})
|
||||
tally = rp.raise_from_steamcharts(games, client, log=lambda *a: None)
|
||||
assert [g["all_time_peak"] for g in games] == ["253,225", "N/A", "300,000"]
|
||||
assert tally == {"raised": 1, "untracked": 1, "unchanged": 1}
|
||||
|
||||
|
||||
def test_a_refusal_stops_the_run_and_keeps_what_was_raised():
|
||||
games = [_game("1"), _game("2"), _game("3")]
|
||||
client = _Client({"1": Result("ok", 10), "2": Result("blocked"), "3": Result("ok", 30)})
|
||||
tally = rp.raise_from_steamcharts(games, client, log=lambda *a: None)
|
||||
assert client.asked == ["1", "2"]
|
||||
assert games[0]["all_time_peak"] == "10"
|
||||
assert tally["skipped"] == 2
|
||||
|
||||
|
||||
def test_ten_errors_in_a_row_stop_the_run():
|
||||
games = [_game(str(i)) for i in range(15)]
|
||||
client = _Client({str(i): Result("error") for i in range(15)})
|
||||
tally = rp.raise_from_steamcharts(games, client, log=lambda *a: None)
|
||||
assert len(client.asked) == rp.MAX_CONSECUTIVE_ERRORS
|
||||
assert tally["error"] == rp.MAX_CONSECUTIVE_ERRORS
|
||||
assert tally["skipped"] == 15 - rp.MAX_CONSECUTIVE_ERRORS
|
||||
|
||||
|
||||
def test_nothing_but_all_time_peak_is_written():
|
||||
game = _game("1")
|
||||
before = dict(game)
|
||||
rp.raise_from_samples([game])
|
||||
rp.raise_from_steamcharts([game], _Client({"1": Result("ok", 999_999)}), log=lambda *a: None)
|
||||
changed = {k for k in game if game[k] != before.get(k)}
|
||||
assert changed == {"all_time_peak"}
|
||||
|
||||
|
||||
def test_every_game_is_asked_once_a_month():
|
||||
games = [_game(str(appid)) for appid in range(10, 30_010, 10)]
|
||||
days = [sum(1 for g in games if rp.due(g, rp.PEAK_CYCLE_DAYS, day)) for day in range(30)]
|
||||
assert sum(days) == len(games)
|
||||
assert min(days) > 0
|
||||
|
||||
|
||||
# ──────────── It survives the other jobs ────────────
|
||||
|
||||
def test_new_records_start_without_a_peak():
|
||||
assert make_skeleton("https://store.steampowered.com/app/1/")["all_time_peak"] == "N/A"
|
||||
|
||||
|
||||
def test_refetch_all_never_clears_it():
|
||||
# The other player fields are cleared and fetched again; this one cannot be.
|
||||
assert "all_time_peak" not in refetch_all.CLEARABLE
|
||||
assert "all_time_peak" not in refetch_all.CONDITIONAL
|
||||
@@ -10,6 +10,7 @@ import pytest
|
||||
|
||||
import refresh_store_data as rsd
|
||||
from core.constants import MANUAL_FIELDS, DESCRIPTION_MAX
|
||||
from core.rotation import slice_of
|
||||
|
||||
|
||||
def _game(appid: str = "730", **fields) -> dict:
|
||||
@@ -76,7 +77,7 @@ def test_slices_are_even_although_appids_are_multiples_of_ten():
|
||||
for cycle in (rsd.PAGE_CYCLE_DAYS, rsd.DETAILS_CYCLE_DAYS):
|
||||
sizes = [0] * cycle
|
||||
for a in appids:
|
||||
sizes[rsd.slice_of(a, cycle)] += 1
|
||||
sizes[slice_of(a, cycle)] += 1
|
||||
mean = len(appids) / cycle
|
||||
assert min(sizes) > 0.7 * mean and max(sizes) < 1.3 * mean, (cycle, sizes)
|
||||
|
||||
|
||||
+2
-1
@@ -81,7 +81,8 @@ Worker:
|
||||
`index.json` lists a SHA-256 for every shard. The app requests shards as
|
||||
`?v=<sha256>` (packed or plain), which the Worker serves content-addressed and immutable (or 503s
|
||||
while GitHub's CDN still has the previous bytes), and hashes what it receives
|
||||
before caching it. IndexedDB holds the last generation that verified; the app
|
||||
before caching it. IndexedDB holds the last generation that verified, as the
|
||||
packs it arrived in (opaque `Uint8Array`s, `src/lib/cache.ts`); the app
|
||||
re-checks the index when the tab regains focus and every ten minutes while it is
|
||||
visible, and falls back to that cache offline.
|
||||
|
||||
|
||||
Generated
+63
-63
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "free-steam-games-web",
|
||||
"version": "2.1.1",
|
||||
"version": "2.2.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "free-steam-games-web",
|
||||
"version": "2.1.1",
|
||||
"version": "2.2.0",
|
||||
"dependencies": {
|
||||
"@fontsource-variable/bricolage-grotesque": "^5.3.0",
|
||||
"@fontsource-variable/ibm-plex-sans": "^5.3.0",
|
||||
@@ -51,7 +51,7 @@
|
||||
"vite": "^8.0.16",
|
||||
"vite-plugin-pwa": "^1.3.0",
|
||||
"vitest": "^5.0.0",
|
||||
"wrangler": "^4.130.0"
|
||||
"wrangler": "^4.147.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@apideck/better-ajv-errors": {
|
||||
@@ -1624,9 +1624,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/unenv-preset": {
|
||||
"version": "2.16.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz",
|
||||
"integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==",
|
||||
"version": "2.16.2",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.2.tgz",
|
||||
"integrity": "sha512-JBP1+Z7ZSNG/d4mRP+y8VC5dka3tZVMLEZRvS+rzQ4DGV1EoxRFQckcJTTkXbHSQiTj0DtNI01Zwb/V2fX0mvQ==",
|
||||
"dev": true,
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"peerDependencies": {
|
||||
@@ -1640,9 +1640,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-darwin-64": {
|
||||
"version": "1.20260908.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260908.1.tgz",
|
||||
"integrity": "sha512-t3juyCXFn12OklBL0S7UC98py3nLEmuioBOUOazeyeVsLUu8fv+pfJrR+XzwSH2Uh6rCXZNogRh+LtV0YpzMcQ==",
|
||||
"version": "1.20261001.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261001.1.tgz",
|
||||
"integrity": "sha512-4cgSgDf28JSw/P5Dj5GCS59hzVqS5XnmGAWNkvYHLI6ODU9idGaMMNEuhJXDkEG/lsABmlVlnCgsdh2VbKWepw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1657,9 +1657,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-darwin-arm64": {
|
||||
"version": "1.20260908.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260908.1.tgz",
|
||||
"integrity": "sha512-I1nwA4qm/fUNSKhPSO76YA6JAhcA0KU63yFcYHN6AiDowGbDyfjDPH9KCVopT5rI1LY4GfbdAi5b9gODqZsAkQ==",
|
||||
"version": "1.20261001.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261001.1.tgz",
|
||||
"integrity": "sha512-8ulAWruEVouNmEIsQsy9WSSCS9zkLu93W2MTwp5esiFyoPp05BNSVFIFsYSPi1pkFmBBd7fsnwMpbLkaJLaVPQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1674,9 +1674,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-linux-64": {
|
||||
"version": "1.20260908.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260908.1.tgz",
|
||||
"integrity": "sha512-s/h5uSW1UC6dGeVKSqrPLpTu+vo0fKJZCNEokW1Ol1qcCB2oN6WCRHhoyzo4Y69oONAXRgLfLBYaHWe7z51Vnw==",
|
||||
"version": "1.20261001.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261001.1.tgz",
|
||||
"integrity": "sha512-kZbTZJGrhsMOdqZ2BIybjaBRLZjYsRLWj7mcNw/6Y3hodOhp5MrNzcz4iWGwNVWwyIV+7Pl+/LX5VcgnRqdHOg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1691,9 +1691,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-linux-arm64": {
|
||||
"version": "1.20260908.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260908.1.tgz",
|
||||
"integrity": "sha512-PP/nUKl0R6colwfocggL8dctO/dFMqMft12unzFUkoAJr0aXQeT+ia0JuNmOUWXe6EfvyCSmAbijEsTKQ5t/ew==",
|
||||
"version": "1.20261001.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261001.1.tgz",
|
||||
"integrity": "sha512-oOk3Zj6k/8oP0FJgZBWDn7+BqbsqIMEMaV95pulHPVbwVu4wYoLfQq2hp0vkbCNsCFLqZb7cqixXdrwD54ZIow==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1708,9 +1708,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-windows-64": {
|
||||
"version": "1.20260908.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260908.1.tgz",
|
||||
"integrity": "sha512-jkaS5EKKTvzAdIlbMfOYrHoTucWVRwYBwIig+xRsjXQv5BXTLA2Llg+iM8djlKtk0CjkztZhXmuxuqoqWKZmFw==",
|
||||
"version": "1.20261001.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261001.1.tgz",
|
||||
"integrity": "sha512-uRxm5W4VyBkoSaoP1BfOuH0873tE+vxsknF4sab6/5YIRvIAufChq3QDp+zlAn67PuGPGcn7W8QraA0t4ucmOQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -4474,9 +4474,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@sveltejs/kit/node_modules/cookie": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
|
||||
"integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
|
||||
"version": "0.7.2",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
|
||||
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -5660,9 +5660,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "5.0.9",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
|
||||
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
|
||||
"version": "5.0.12",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz",
|
||||
"integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -6192,9 +6192,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/devalue": {
|
||||
"version": "5.9.2",
|
||||
"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.2.tgz",
|
||||
"integrity": "sha512-po4PAY5c53tw5XMocSnf8A/5OHhbbUftpr93aEN6BBoAdntUmK7vu7wOATqvt7cXO7m1Cl4gMVn6p7n6n4mj0w==",
|
||||
"version": "5.9.4",
|
||||
"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz",
|
||||
"integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/devlop": {
|
||||
@@ -6620,9 +6620,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.7",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz",
|
||||
"integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==",
|
||||
"version": "3.1.8",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
|
||||
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -6682,9 +6682,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/filelist/node_modules/brace-expansion": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
|
||||
"integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
|
||||
"version": "2.1.7",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz",
|
||||
"integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -9066,16 +9066,16 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/miniflare": {
|
||||
"version": "5.20260908.0-alpha",
|
||||
"resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260908.0-alpha.tgz",
|
||||
"integrity": "sha512-BHIknb0u6vLIvb+R8PsUAzgoWWk3OlW85DrV2SG0EydfSpIba28YT0rH6xZThjnSwDxzNuW3FDRNSWvbiI/DVw==",
|
||||
"version": "5.20261001.0-alpha",
|
||||
"resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261001.0-alpha.tgz",
|
||||
"integrity": "sha512-GaimS5mSIOMyvd16ga+e1/QkI8cmp3z35QPHFex0DktqNQf2RVZQwMPQXdyoUreUiFP/0Gpe2MoQInTyMAD5xA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@cspotcode/source-map-support": "0.8.1",
|
||||
"sharp": "0.35.2",
|
||||
"undici": "7.29.0",
|
||||
"workerd": "1.20260908.1",
|
||||
"sharp": "0.35.4",
|
||||
"undici": "7.29.1",
|
||||
"workerd": "1.20261001.1",
|
||||
"ws": "8.21.0",
|
||||
"youch": "4.1.0-beta.10"
|
||||
},
|
||||
@@ -10028,9 +10028,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/serialize-javascript": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.1.1.tgz",
|
||||
"integrity": "sha512-k3CMsaIvvdSwm8oLB4MXSl0wH2/cwlH7xGcnRd2DaeRmBkbzYmyT8j0tsX60DwD1eRwHTpNpH8ljKu9oUT1MeQ==",
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.1.2.tgz",
|
||||
"integrity": "sha512-GL2BWwVa6JydKO6l/ljVgjAZF4QJ3S7dWDWi53s5GZT8PJzD2fNxi67HANQGKAqPrwEpL5ba7gUBGi0Ls/sEoQ==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
@@ -10990,9 +10990,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.29.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
|
||||
"integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
|
||||
"version": "7.29.1",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz",
|
||||
"integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -11823,9 +11823,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/workerd": {
|
||||
"version": "1.20260908.1",
|
||||
"resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260908.1.tgz",
|
||||
"integrity": "sha512-rYhpW6NWHD++p34ej+VXWzi5Pdnmd7ncdbB/ux4s+i3mf7IeOpW3O4roqClQ+Z8ernvyMCknBLCb76z1rA+a7Q==",
|
||||
"version": "1.20261001.1",
|
||||
"resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261001.1.tgz",
|
||||
"integrity": "sha512-d/SIYHFO0PT/wiFZg8in4NpRIxYuFwslX1HdylOtWkBIIUmSpkGFhK820cV84XACFylwJ48xuRoWW/8DWDPsPQ==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "Apache-2.0",
|
||||
@@ -11836,28 +11836,28 @@
|
||||
"node": ">=16"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@cloudflare/workerd-darwin-64": "1.20260908.1",
|
||||
"@cloudflare/workerd-darwin-arm64": "1.20260908.1",
|
||||
"@cloudflare/workerd-linux-64": "1.20260908.1",
|
||||
"@cloudflare/workerd-linux-arm64": "1.20260908.1",
|
||||
"@cloudflare/workerd-windows-64": "1.20260908.1"
|
||||
"@cloudflare/workerd-darwin-64": "1.20261001.1",
|
||||
"@cloudflare/workerd-darwin-arm64": "1.20261001.1",
|
||||
"@cloudflare/workerd-linux-64": "1.20261001.1",
|
||||
"@cloudflare/workerd-linux-arm64": "1.20261001.1",
|
||||
"@cloudflare/workerd-windows-64": "1.20261001.1"
|
||||
}
|
||||
},
|
||||
"node_modules/wrangler": {
|
||||
"version": "4.130.0",
|
||||
"resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.130.0.tgz",
|
||||
"integrity": "sha512-fzNjnTzyZl31PGJOFXbiLeZqEIToQ9KOzkkvGdQz4wlB+BoHnl3BRwCR5xg8AqYyzVunvvMHlMzvlbThQ7rrAA==",
|
||||
"version": "4.147.0",
|
||||
"resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.147.0.tgz",
|
||||
"integrity": "sha512-pQYRoiq8PTAxphaG69z8+GC1DkSGd19EDZehQ8zxjo/Ko3mRB6Qs1mTrd8ZuKAarLklIjTqr1lUdCK9r4q2hUg==",
|
||||
"dev": true,
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"dependencies": {
|
||||
"@cloudflare/kv-asset-handler": "0.5.0",
|
||||
"@cloudflare/unenv-preset": "2.16.1",
|
||||
"@cloudflare/unenv-preset": "2.16.2",
|
||||
"blake3-wasm": "2.1.5",
|
||||
"esbuild": "0.28.1",
|
||||
"miniflare": "5.20260908.0-alpha",
|
||||
"miniflare": "5.20261001.0-alpha",
|
||||
"path-to-regexp": "6.3.0",
|
||||
"unenv": "2.0.0-rc.24",
|
||||
"workerd": "1.20260908.1"
|
||||
"workerd": "1.20261001.1"
|
||||
},
|
||||
"bin": {
|
||||
"cf-wrangler": "bin/cf-wrangler.js",
|
||||
@@ -11871,7 +11871,7 @@
|
||||
"fsevents": "2.3.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@cloudflare/workers-types": "^5.20260908.1"
|
||||
"@cloudflare/workers-types": "^5.20261001.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@cloudflare/workers-types": {
|
||||
|
||||
+5
-3
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "free-steam-games-web",
|
||||
"private": true,
|
||||
"version": "2.1.1",
|
||||
"version": "2.2.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
@@ -74,12 +74,14 @@
|
||||
"vite": "^8.0.16",
|
||||
"vite-plugin-pwa": "^1.3.0",
|
||||
"vitest": "^5.0.0",
|
||||
"wrangler": "^4.130.0"
|
||||
"wrangler": "^4.147.0"
|
||||
},
|
||||
"overrides": {
|
||||
"echarts-wordcloud": {
|
||||
"echarts": "$echarts"
|
||||
},
|
||||
"sharp": "0.35.4"
|
||||
"@sveltejs/kit@2.70.3": {
|
||||
"cookie": "^0.7.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PACK_MAGIC, packBytes, sha256, unpackBytes } from "./data-pack";
|
||||
import { PACK_MAGIC, packBytes, packIvHex, sha256, unpackBytes } from "./data-pack";
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
@@ -55,4 +55,16 @@ describe("data-pack", () => {
|
||||
const { packed } = await pack("");
|
||||
expect(await unpackBytes(packed.buffer as ArrayBuffer)).toEqual(new Uint8Array(0));
|
||||
});
|
||||
|
||||
it("names the shard a pack holds by its IV: the sha256 prefix", async () => {
|
||||
const { plain, packed } = await pack(SHARD);
|
||||
const hex = [...(await sha256(plain))].map((b) => b.toString(16).padStart(2, "0")).join("");
|
||||
expect(packIvHex(packed)).toBe(hex.slice(0, 24));
|
||||
|
||||
const wrongMagic = packed.slice();
|
||||
wrongMagic[3] = 0x02;
|
||||
expect(packIvHex(wrongMagic)).toBeNull();
|
||||
expect(packIvHex(enc.encode('{"link":"x"}\n'))).toBeNull();
|
||||
expect(packIvHex(packed.subarray(0, 20))).toBeNull();
|
||||
});
|
||||
});
|
||||
+18
-3
@@ -41,6 +41,23 @@ export function packsSupported(): boolean {
|
||||
return typeof DecompressionStream === "function" && typeof crypto !== "undefined" && Boolean(crypto.subtle);
|
||||
}
|
||||
|
||||
function isPack(bytes: Uint8Array): boolean {
|
||||
return bytes.length >= HEADER_BYTES + TAG_BYTES && PACK_MAGIC.every((b, i) => bytes[i] === b);
|
||||
}
|
||||
|
||||
/**
|
||||
* A format-1 pack's IV as lowercase hex, or null when `packed` is not one.
|
||||
* The IV is the first 12 bytes of the plain bytes' SHA-256, so it names the
|
||||
* shard a pack holds without opening it: lib/cache.ts checks every stored pack
|
||||
* against its index entry's sha256 this way.
|
||||
*/
|
||||
export function packIvHex(packed: Uint8Array): string | null {
|
||||
if (!isPack(packed)) return null;
|
||||
let hex = "";
|
||||
for (const b of packed.subarray(PACK_MAGIC.length, HEADER_BYTES)) hex += b.toString(16).padStart(2, "0");
|
||||
return hex;
|
||||
}
|
||||
|
||||
async function key(usage: "encrypt" | "decrypt"): Promise<CryptoKey> {
|
||||
return crypto.subtle.importKey("raw", PACK_KEY, "AES-GCM", false, [usage]);
|
||||
}
|
||||
@@ -78,9 +95,7 @@ export async function packBytes(plain: Uint8Array, digest: Uint8Array): Promise<
|
||||
/** Unpack to the original bytes. Throws on anything that is not an intact pack. */
|
||||
export async function unpackBytes(packed: ArrayBuffer | Uint8Array): Promise<Uint8Array<ArrayBuffer>> {
|
||||
const bytes = packed instanceof Uint8Array ? packed : new Uint8Array(packed);
|
||||
if (bytes.length < HEADER_BYTES + TAG_BYTES || PACK_MAGIC.some((b, i) => bytes[i] !== b)) {
|
||||
throw new Error("data-pack: not a format-1 pack");
|
||||
}
|
||||
if (!isPack(bytes)) throw new Error("data-pack: not a format-1 pack");
|
||||
const iv = bytes.slice(PACK_MAGIC.length, HEADER_BYTES);
|
||||
const compressed = new Uint8Array(
|
||||
await crypto.subtle.decrypt(
|
||||
|
||||
Generated
+1
-1
@@ -961,7 +961,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "f2p-tracker"
|
||||
version = "2.1.1"
|
||||
version = "2.2.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "f2p-tracker"
|
||||
version = "2.1.1"
|
||||
version = "2.2.0"
|
||||
description = "Desktop wrapper around the Steam F2P Tracker web app"
|
||||
authors = ["poli0981"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "F2P Tracker",
|
||||
"version": "2.1.1",
|
||||
"version": "2.2.0",
|
||||
"identifier": "io.github.poli0981.f2p-tracker",
|
||||
"build": {
|
||||
"beforeBuildCommand": "npm run build:desktop",
|
||||
|
||||
@@ -399,6 +399,7 @@
|
||||
"labelPlatforms": "Platforms",
|
||||
"labelReviews": "Reviews",
|
||||
"labelPlayers": "Players",
|
||||
"labelAllTimePeak": "All-time peak",
|
||||
"labelAntiCheat": "Anti-cheat",
|
||||
"labelMetacritic": "Metacritic",
|
||||
"labelDrm": "DRM",
|
||||
@@ -408,7 +409,6 @@
|
||||
"labelTags": "Tags",
|
||||
"labelLanguages": "Languages ({{count}})",
|
||||
"labelNotes": "Notes",
|
||||
"playersPeak": "(peak {{peak}})",
|
||||
"descriptionSource": "Blurb by the publisher, via Steam — shortened.",
|
||||
"removedOn": "Removed from the catalogue on {{date}}: {{reason}}"
|
||||
},
|
||||
|
||||
@@ -399,6 +399,7 @@
|
||||
"labelPlatforms": "Nền tảng",
|
||||
"labelReviews": "Đánh giá",
|
||||
"labelPlayers": "Người chơi",
|
||||
"labelAllTimePeak": "Đỉnh mọi thời điểm",
|
||||
"labelAntiCheat": "Anti-cheat",
|
||||
"labelMetacritic": "Metacritic",
|
||||
"labelDrm": "DRM",
|
||||
@@ -408,7 +409,6 @@
|
||||
"labelTags": "Tag",
|
||||
"labelLanguages": "Ngôn ngữ ({{count}})",
|
||||
"labelNotes": "Ghi chú",
|
||||
"playersPeak": "(đỉnh {{peak}})",
|
||||
"descriptionSource": "Mô tả do nhà phát hành viết, lấy từ Steam — đã rút gọn.",
|
||||
"removedOn": "Đã bị gỡ khỏi danh mục ngày {{date}}: {{reason}}"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { packBytes, sha256 } from "../../shared/data-pack";
|
||||
import type { DataIndex, GameRecord } from "./schema";
|
||||
|
||||
/**
|
||||
* What lib/cache.ts promises: the catalogue sits in IndexedDB as the packs it
|
||||
* arrived in - opaque bytes, never records or plain JSONL - in one consistent
|
||||
* write; a stored pack must be the shard its index entry names; and the
|
||||
* readable copy older app versions left behind is read until the next
|
||||
* generation, which deletes it, without ever creating that database.
|
||||
*
|
||||
* idb-keyval is replaced by an in-memory one. Like IndexedDB, it creates a
|
||||
* database the first time anything opens it.
|
||||
*/
|
||||
const idb = vi.hoisted(() => {
|
||||
const dbs = new Map<string, Map<unknown, unknown>>();
|
||||
const open = (name: string) => {
|
||||
if (!dbs.has(name)) dbs.set(name, new Map());
|
||||
return dbs.get(name)!;
|
||||
};
|
||||
return { dbs, open, setMany: [] as [unknown, unknown][][] };
|
||||
});
|
||||
|
||||
vi.mock("idb-keyval", () => ({
|
||||
createStore: (db: string) => db,
|
||||
getMany: async (keys: unknown[], store: string) => keys.map((k) => idb.open(store).get(k)),
|
||||
setMany: async (entries: [unknown, unknown][], store: string) => {
|
||||
idb.setMany.push(entries);
|
||||
for (const [k, v] of entries) idb.open(store).set(k, v);
|
||||
},
|
||||
delMany: async (keys: unknown[], store: string) => {
|
||||
for (const k of keys) idb.open(store).delete(k);
|
||||
},
|
||||
}));
|
||||
|
||||
const NEW_DB = "f2p-catalogue";
|
||||
const LEGACY_DB = "keyval-store";
|
||||
|
||||
let cache: typeof import("./cache");
|
||||
|
||||
beforeEach(async () => {
|
||||
idb.dbs.clear();
|
||||
idb.setMany.length = 0;
|
||||
vi.stubGlobal("indexedDB", {
|
||||
databases: async () => [...idb.dbs.keys()].map((name) => ({ name, version: 1 })),
|
||||
});
|
||||
// The module remembers its store handles; every test starts from none.
|
||||
vi.resetModules();
|
||||
cache = await import("./cache");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
async function generation(stamp: string, shards: string[][]) {
|
||||
const packs: Uint8Array[] = [];
|
||||
const files: DataIndex["files"] = [];
|
||||
for (const [i, names] of shards.entries()) {
|
||||
const text = names.map((n) => JSON.stringify({ link: `https://store.steampowered.com/app/${n}/`, name: `Game ${n}` })).join("\n") + "\n";
|
||||
const plain = enc.encode(text);
|
||||
const digest = await sha256(plain);
|
||||
packs.push(await packBytes(plain, digest));
|
||||
files.push({
|
||||
name: `data_00${i + 1}.jsonl`,
|
||||
count: names.length,
|
||||
sha256: [...digest].map((b) => b.toString(16).padStart(2, "0")).join(""),
|
||||
});
|
||||
}
|
||||
const index: DataIndex = { max_per_file: 800, total: shards.flat().length, last_updated: stamp, files };
|
||||
return { index, packs };
|
||||
}
|
||||
|
||||
function seedLegacy(index: DataIndex, records: GameRecord[]) {
|
||||
idb.open(LEGACY_DB).set("f2p:records", records).set("f2p:index", index);
|
||||
}
|
||||
|
||||
describe("cache", () => {
|
||||
it("stores the packs as they arrived: Uint8Arrays with no game data readable", async () => {
|
||||
const { index, packs } = await generation("t1", [["730", "570"], ["440"]]);
|
||||
await cache.writeCache(index, packs);
|
||||
|
||||
const stored = idb.dbs.get(NEW_DB)!;
|
||||
const records = stored.get("f2p:records") as Uint8Array[];
|
||||
expect(records).toHaveLength(2);
|
||||
for (const p of records) {
|
||||
expect(p).toBeInstanceOf(Uint8Array);
|
||||
const text = new TextDecoder().decode(p);
|
||||
expect(text).not.toContain("Game ");
|
||||
expect(text).not.toContain("steampowered");
|
||||
}
|
||||
expect(stored.get("f2p:index")).toEqual(index);
|
||||
});
|
||||
|
||||
it("writes both keys in one transaction, the packs first", async () => {
|
||||
const { index, packs } = await generation("t1", [["730"]]);
|
||||
await cache.writeCache(index, packs);
|
||||
expect(idb.setMany).toHaveLength(1);
|
||||
expect(idb.setMany[0].map(([k]) => k)).toEqual(["f2p:records", "f2p:index"]);
|
||||
});
|
||||
|
||||
it("reads back exactly what it wrote", async () => {
|
||||
const { index, packs } = await generation("t1", [["730"], ["570"]]);
|
||||
await cache.writeCache(index, packs);
|
||||
expect(await cache.readCache()).toEqual({ kind: "packs", index, packs });
|
||||
});
|
||||
|
||||
it("is a miss when a stored pack is not the shard its index entry names", async () => {
|
||||
const { index, packs } = await generation("t1", [["730"], ["570"]]);
|
||||
await cache.writeCache(index, packs);
|
||||
const stored = idb.dbs.get(NEW_DB)!;
|
||||
|
||||
// The same packs under the wrong entries.
|
||||
stored.set("f2p:records", [packs[1], packs[0]]);
|
||||
expect(await cache.readCache()).toBeNull();
|
||||
|
||||
// One pack short.
|
||||
stored.set("f2p:records", [packs[0]]);
|
||||
expect(await cache.readCache()).toBeNull();
|
||||
|
||||
// Not a pack at all: plain JSONL, as no version ever stored here.
|
||||
stored.set("f2p:records", [enc.encode('{"link":"x"}\n'), packs[1]]);
|
||||
expect(await cache.readCache()).toBeNull();
|
||||
});
|
||||
|
||||
it("refuses to store a generation in which any shard came over the plain path", async () => {
|
||||
const { index, packs } = await generation("t1", [["730"], ["570"]]);
|
||||
await cache.writeCache(index, [packs[0], null]);
|
||||
expect(idb.setMany).toHaveLength(0);
|
||||
expect(idb.dbs.get(NEW_DB)?.get("f2p:records")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses to store packs where they could not be opened again", async () => {
|
||||
vi.stubGlobal("DecompressionStream", undefined);
|
||||
const { index, packs } = await generation("t1", [["730"]]);
|
||||
await cache.writeCache(index, packs);
|
||||
expect(idb.setMany).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("reads the legacy copy only while the new store is empty, and deletes it on the next write", async () => {
|
||||
const old = await generation("t0", [["730"]]);
|
||||
const legacyRecords = [{ name: "Game 730" } as GameRecord];
|
||||
seedLegacy(old.index, legacyRecords);
|
||||
expect(await cache.readCache()).toEqual({ kind: "legacy", index: old.index, records: legacyRecords });
|
||||
|
||||
const next = await generation("t1", [["730", "570"]]);
|
||||
await cache.writeCache(next.index, next.packs);
|
||||
expect(idb.dbs.get(LEGACY_DB)!.size).toBe(0);
|
||||
expect((await cache.readCache())?.kind).toBe("packs");
|
||||
});
|
||||
|
||||
it("deletes the legacy copy even when there is nothing it may store", async () => {
|
||||
const old = await generation("t0", [["730"]]);
|
||||
seedLegacy(old.index, [{ name: "Game 730" } as GameRecord]);
|
||||
const next = await generation("t1", [["570"]]);
|
||||
await cache.writeCache(next.index, [null]);
|
||||
expect(idb.dbs.get(LEGACY_DB)!.size).toBe(0);
|
||||
});
|
||||
|
||||
it("never creates the legacy database for a visitor who has none", async () => {
|
||||
expect(await cache.readCache()).toBeNull();
|
||||
const { index, packs } = await generation("t1", [["730"]]);
|
||||
await cache.writeCache(index, packs);
|
||||
await cache.clearCache();
|
||||
expect(idb.dbs.has(LEGACY_DB)).toBe(false);
|
||||
});
|
||||
|
||||
it("clearCache empties both stores", async () => {
|
||||
const old = await generation("t0", [["730"]]);
|
||||
seedLegacy(old.index, [{ name: "Game 730" } as GameRecord]);
|
||||
const { index, packs } = await generation("t1", [["570"]]);
|
||||
idb.open(NEW_DB).set("f2p:records", packs).set("f2p:index", index);
|
||||
|
||||
await cache.clearCache();
|
||||
expect(idb.dbs.get(NEW_DB)!.size).toBe(0);
|
||||
expect(idb.dbs.get(LEGACY_DB)!.size).toBe(0);
|
||||
expect(await cache.readCache()).toBeNull();
|
||||
});
|
||||
});
|
||||
+123
-15
@@ -3,43 +3,151 @@
|
||||
*
|
||||
* What counts as the same generation, and when one may be written at all, is
|
||||
* games-loader.ts's decision - this file only stores it.
|
||||
*
|
||||
* WHAT IS STORED is each shard's pack exactly as it arrived from
|
||||
* /api/data/p1/ (shared/data-pack.ts) - opaque bytes in a Uint8Array, never
|
||||
* parsed records and never the plain JSONL. Like the packs on the wire this is
|
||||
* obfuscation and compression (~15% of the JSONL), NOT protection: the key is
|
||||
* public, and the dataset is meant to be read from Git. A generation in which
|
||||
* any shard came over the plain path is not stored at all; the browser never
|
||||
* packs anything itself (compressing ~9.5 MB on the main thread, several times
|
||||
* a day, would cost a phone a second each time).
|
||||
*
|
||||
* f2p:index DataIndex, as fetched - public metadata, kept readable so the
|
||||
* generation can be compared without opening anything
|
||||
* f2p:records Uint8Array[], one pack per index.files entry, in its order
|
||||
*
|
||||
* The packs live in a database of their own, `f2p-catalogue`. Up to app 2.1.x
|
||||
* the same two keys held PARSED records in idb-keyval's default database
|
||||
* (`keyval-store`), and a tab still running that code would hand Uint8Arrays to
|
||||
* the page if it found them there. That legacy copy is still read - it is a
|
||||
* verified generation like any other - until the next generation is written,
|
||||
* which deletes it.
|
||||
*/
|
||||
import { get, set, del } from "idb-keyval";
|
||||
import { createStore, delMany, getMany, setMany, type UseStore } from "idb-keyval";
|
||||
import { packIvHex, packsSupported } from "../../shared/data-pack";
|
||||
import type { DataIndex, GameRecord } from "./schema";
|
||||
|
||||
/** The names docs/PRIVACY_POLICY.md lists. Renaming either means editing that
|
||||
* document, which reopens the consent gate for everyone. */
|
||||
const KEY_RECORDS = "f2p:records";
|
||||
const KEY_INDEX = "f2p:index";
|
||||
const KEYS = [KEY_RECORDS, KEY_INDEX];
|
||||
|
||||
export interface CachedBundle {
|
||||
index: DataIndex;
|
||||
records: GameRecord[];
|
||||
const LEGACY_DB = "keyval-store";
|
||||
|
||||
export type CachedBundle =
|
||||
| { kind: "packs"; index: DataIndex; packs: Uint8Array[] }
|
||||
/** Written by app 2.1.x and older; see the header. */
|
||||
| { kind: "legacy"; index: DataIndex; records: GameRecord[] };
|
||||
|
||||
let packStore: UseStore | undefined;
|
||||
let legacyStore: UseStore | undefined;
|
||||
|
||||
function packs(): UseStore {
|
||||
// Lazily: creating a store opens nothing, but this module is imported
|
||||
// during prerender too, where there is no IndexedDB at all.
|
||||
return (packStore ??= createStore("f2p-catalogue", "keyval"));
|
||||
}
|
||||
|
||||
/**
|
||||
* The legacy database, or null when it does not exist. Opening a database
|
||||
* creates it, so where the browser can list its databases this looks first:
|
||||
* a new visitor never gets an empty `keyval-store` from a check for old data.
|
||||
*/
|
||||
async function legacy(): Promise<UseStore | null> {
|
||||
if (legacyStore) return legacyStore;
|
||||
const idb = globalThis.indexedDB;
|
||||
if (!idb) return null;
|
||||
if (typeof idb.databases === "function") {
|
||||
const dbs = await idb.databases();
|
||||
if (!dbs.some((d) => d.name === LEGACY_DB)) return null;
|
||||
}
|
||||
return (legacyStore = createStore(LEGACY_DB, "keyval"));
|
||||
}
|
||||
|
||||
function isIndex(v: unknown): v is DataIndex {
|
||||
if (typeof v !== "object" || v === null) return false;
|
||||
const i = v as DataIndex;
|
||||
return (
|
||||
typeof i.last_updated === "string" &&
|
||||
Array.isArray(i.files) &&
|
||||
i.files.every((f) => typeof f?.name === "string" && typeof f.count === "number")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* One pack per index entry, each a pack of the shard that entry names: its IV
|
||||
* is the shard's sha256 prefix (where the entry records one).
|
||||
*/
|
||||
function packsFor(index: DataIndex, v: unknown): v is Uint8Array[] {
|
||||
return (
|
||||
Array.isArray(v) &&
|
||||
v.length === index.files.length &&
|
||||
v.every((p, i) => {
|
||||
if (!(p instanceof Uint8Array)) return false;
|
||||
const iv = packIvHex(p);
|
||||
const sha = index.files[i].sha256;
|
||||
return iv !== null && (!sha || sha.slice(0, iv.length) === iv);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export async function readCache(): Promise<CachedBundle | null> {
|
||||
try {
|
||||
const [records, index] = await Promise.all([
|
||||
get<GameRecord[]>(KEY_RECORDS),
|
||||
get<DataIndex>(KEY_INDEX),
|
||||
]);
|
||||
if (records && index) return { records, index };
|
||||
// Both keys in one transaction: one consistent snapshot, even while
|
||||
// another tab is writing the next generation.
|
||||
const [records, index] = await getMany<unknown>(KEYS, packs());
|
||||
if (isIndex(index) && packsFor(index, records)) return { kind: "packs", index, packs: records };
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
try {
|
||||
const store = await legacy();
|
||||
if (!store) return null;
|
||||
const [records, index] = await getMany<unknown>(KEYS, store);
|
||||
if (isIndex(index) && Array.isArray(records)) {
|
||||
return { kind: "legacy", index, records: records as GameRecord[] };
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function writeCache(bundle: CachedBundle): Promise<void> {
|
||||
/**
|
||||
* Store a verified generation - if every shard of it arrived packed. Either
|
||||
* way the legacy copy goes: it is an older generation in a readable form.
|
||||
*/
|
||||
export async function writeCache(index: DataIndex, shards: (Uint8Array | null)[]): Promise<void> {
|
||||
// First: it may be what frees the room for the write below.
|
||||
await dropLegacy();
|
||||
if (!packsSupported() || !packsFor(index, shards)) return;
|
||||
try {
|
||||
await Promise.all([
|
||||
set(KEY_RECORDS, bundle.records),
|
||||
set(KEY_INDEX, bundle.index),
|
||||
]);
|
||||
// One transaction, records first: if the first put already fails, the
|
||||
// index is never written next to the previous generation's packs.
|
||||
await setMany(
|
||||
[
|
||||
[KEY_RECORDS, shards],
|
||||
[KEY_INDEX, index],
|
||||
],
|
||||
packs(),
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn("[cache] write failed:", err);
|
||||
}
|
||||
}
|
||||
|
||||
export async function clearCache(): Promise<void> {
|
||||
await Promise.all([del(KEY_RECORDS), del(KEY_INDEX)]);
|
||||
await delMany(KEYS, packs());
|
||||
await dropLegacy();
|
||||
}
|
||||
|
||||
async function dropLegacy(): Promise<void> {
|
||||
try {
|
||||
const store = await legacy();
|
||||
if (store) await delMany(KEYS, store);
|
||||
} catch {
|
||||
// Best effort; the next write tries again.
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ const CSV_COLS: { key: keyof GameRecord; header: string }[] = [
|
||||
{ key: "reviews", header: "reviews" },
|
||||
{ key: "current_players", header: "current_players" },
|
||||
{ key: "peak_today", header: "peak_today" },
|
||||
{ key: "all_time_peak", header: "all_time_peak" },
|
||||
{ key: "metacritic", header: "metacritic" },
|
||||
{ key: "anti_cheat", header: "anti_cheat" },
|
||||
{ key: "anti_cheat_note", header: "anti_cheat_note" },
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { fetchRemovedText, fetchShard } from "./fetcher";
|
||||
import { decodeShard, fetchRemovedText, fetchShard } from "./fetcher";
|
||||
import { ShardNotReadyError } from "./games-loader";
|
||||
import { packBytes, sha256 } from "../../shared/data-pack";
|
||||
|
||||
@@ -38,16 +38,21 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("fetchShard", () => {
|
||||
it("fetches the pack by hash and returns the unpacked bytes", async () => {
|
||||
fetchMock.mockResolvedValue(new Response(await packed(BODY)));
|
||||
it("fetches the pack by hash and returns the unpacked bytes, and the pack", async () => {
|
||||
const pack = await packed(BODY);
|
||||
fetchMock.mockResolvedValue(new Response(pack));
|
||||
const out = await fetchShard(entry(), true);
|
||||
expect(text(out)).toBe(BODY);
|
||||
expect(text(out.plain)).toBe(BODY);
|
||||
// Kept as it arrived: this is what the IndexedDB cache stores.
|
||||
expect(out.pack).toEqual(pack);
|
||||
expect(urls()).toEqual([`/api/data/p1/data_001.bin?v=${"a".repeat(64)}`]);
|
||||
});
|
||||
|
||||
it("asks for the unversioned pack without the browser cache", async () => {
|
||||
fetchMock.mockResolvedValue(new Response(await packed(BODY)));
|
||||
await fetchShard(entry(), false);
|
||||
const pack = await packed(BODY);
|
||||
fetchMock.mockResolvedValue(new Response(pack));
|
||||
const out = await fetchShard(entry(), false);
|
||||
expect(out.pack).toEqual(pack);
|
||||
expect(urls()).toEqual(["/api/data/p1/data_001.bin"]);
|
||||
expect(fetchMock.mock.calls[0][1]).toMatchObject({ cache: "no-store" });
|
||||
});
|
||||
@@ -63,14 +68,16 @@ describe("fetchShard", () => {
|
||||
.mockResolvedValueOnce(new Response("not found", { status: 404 }))
|
||||
.mockResolvedValueOnce(new Response(BODY));
|
||||
const out = await fetchShard(entry(), true);
|
||||
expect(text(out)).toBe(BODY);
|
||||
expect(text(out.plain)).toBe(BODY);
|
||||
expect(out.pack).toBeNull();
|
||||
expect(urls()[1]).toBe(`/api/data/data/data_001.jsonl?v=${"a".repeat(64)}`);
|
||||
});
|
||||
|
||||
it("falls back when the answer is not a pack", async () => {
|
||||
fetchMock.mockResolvedValueOnce(new Response(BODY)).mockResolvedValueOnce(new Response(BODY));
|
||||
const out = await fetchShard(entry(), true);
|
||||
expect(text(out)).toBe(BODY);
|
||||
expect(text(out.plain)).toBe(BODY);
|
||||
expect(out.pack).toBeNull();
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
@@ -84,11 +91,26 @@ describe("fetchShard", () => {
|
||||
vi.stubGlobal("DecompressionStream", undefined);
|
||||
fetchMock.mockResolvedValue(new Response(BODY));
|
||||
const out = await fetchShard(entry(), true);
|
||||
expect(text(out)).toBe(BODY);
|
||||
expect(text(out.plain)).toBe(BODY);
|
||||
expect(out.pack).toBeNull();
|
||||
expect(urls()).toEqual([`/api/data/data/data_001.jsonl?v=${"a".repeat(64)}`]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeShard", () => {
|
||||
it("parses plain bytes and opens a pack first", async () => {
|
||||
const plain = enc.encode(BODY);
|
||||
expect((await decodeShard(plain.buffer as ArrayBuffer, false)).map((r) => r.name)).toEqual(["Counter-Strike 2"]);
|
||||
expect((await decodeShard(await packed(BODY), true)).map((r) => r.name)).toEqual(["Counter-Strike 2"]);
|
||||
});
|
||||
|
||||
it("rejects a pack that will not open", async () => {
|
||||
const bad = (await packed(BODY)).slice();
|
||||
bad[bad.length - 1] ^= 0xff;
|
||||
await expect(decodeShard(bad, true)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("fetchRemovedText", () => {
|
||||
const REMOVED = '{"link":"https://store.steampowered.com/app/1/","removed_at":"2026-01-01"}\n';
|
||||
|
||||
|
||||
+24
-12
@@ -16,7 +16,7 @@ import {
|
||||
type GameRecord,
|
||||
type ShardManifestEntry,
|
||||
} from "./schema";
|
||||
import { ShardNotReadyError } from "./games-loader";
|
||||
import { ShardNotReadyError, type FetchedShard } from "./games-loader";
|
||||
import { packsSupported, unpackBytes } from "../../shared/data-pack";
|
||||
import { migrateRecord } from "./data-store";
|
||||
import { API_ORIGIN } from "./site";
|
||||
@@ -45,7 +45,7 @@ export async function fetchIndex(signal?: AbortSignal): Promise<DataIndex> {
|
||||
}
|
||||
|
||||
/**
|
||||
* One shard's bytes.
|
||||
* One shard.
|
||||
*
|
||||
* `versioned` asks for it by the hash its index entry records. That response is
|
||||
* immutable, so it may come from any cache - no `no-store` - and the Worker
|
||||
@@ -54,15 +54,17 @@ export async function fetchIndex(signal?: AbortSignal): Promise<DataIndex> {
|
||||
* without hashes and for a first visit during that 503 window.
|
||||
*
|
||||
* Both are fetched PACKED (/api/data/p1/*.bin, shared/data-pack.ts) when this
|
||||
* runtime can unpack, and returned unpacked: the caller hashes and parses the
|
||||
* same bytes as ever. The plain path is the fallback for a runtime without
|
||||
* DecompressionStream or Web Crypto, or a pack that will not open.
|
||||
* runtime can unpack, and returned unpacked as `plain`: the caller hashes and
|
||||
* parses the same bytes as ever. The pack itself comes back too, as `pack`,
|
||||
* because that is what the IndexedDB cache stores (lib/cache.ts). The plain
|
||||
* path - the fallback for a runtime without DecompressionStream or Web Crypto,
|
||||
* or a pack that will not open - has no pack to give.
|
||||
*/
|
||||
export async function fetchShard(
|
||||
entry: ShardManifestEntry,
|
||||
versioned: boolean,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArrayBuffer> {
|
||||
): Promise<FetchedShard> {
|
||||
if (packsSupported()) {
|
||||
try {
|
||||
return await fetchPacked(packUrl(entry.name), entry, versioned, signal);
|
||||
@@ -79,7 +81,7 @@ export async function fetchShard(
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to fetch ${entry.name}: ${res.status}`);
|
||||
}
|
||||
return await res.arrayBuffer();
|
||||
return { plain: await res.arrayBuffer(), pack: null };
|
||||
}
|
||||
|
||||
/** scripts/removed_games.jsonl as text, packed when possible (see fetchShard). */
|
||||
@@ -114,16 +116,17 @@ async function fetchPacked(
|
||||
entry: ShardManifestEntry,
|
||||
versioned: boolean,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArrayBuffer> {
|
||||
): Promise<FetchedShard> {
|
||||
const res = versioned && entry.sha256
|
||||
? await fetch(`${url}?v=${entry.sha256}`, { signal })
|
||||
: await fetch(url, { signal, cache: "no-store" });
|
||||
if (versioned && res.status === 503) throw new ShardNotReadyError(entry.name);
|
||||
if (!res.ok) throw new PackUnavailableError(`HTTP ${res.status}`);
|
||||
return unpack(await res.arrayBuffer());
|
||||
const pack = new Uint8Array(await res.arrayBuffer());
|
||||
return { plain: await unpack(pack), pack };
|
||||
}
|
||||
|
||||
async function unpack(packed: ArrayBuffer): Promise<ArrayBuffer> {
|
||||
async function unpack(packed: ArrayBuffer | Uint8Array): Promise<ArrayBuffer> {
|
||||
try {
|
||||
const bytes = await unpackBytes(packed);
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
||||
@@ -132,8 +135,17 @@ async function unpack(packed: ArrayBuffer): Promise<ArrayBuffer> {
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse JSONL text → records array (synchronous; called from worker). */
|
||||
export function parseJsonl(text: string): GameRecord[] {
|
||||
/**
|
||||
* Shard bytes → records. `packed`: the bytes are a pack, as the IndexedDB
|
||||
* cache holds them, and are opened first. Runs in the parser worker
|
||||
* (workers/jsonl-parser.ts) and in its main-thread fallback (worker-pool.ts).
|
||||
*/
|
||||
export async function decodeShard(bytes: ArrayBuffer | Uint8Array, packed: boolean): Promise<GameRecord[]> {
|
||||
return parseJsonl(new TextDecoder().decode(packed ? await unpackBytes(bytes) : bytes));
|
||||
}
|
||||
|
||||
/** Parse JSONL text → records array. */
|
||||
function parseJsonl(text: string): GameRecord[] {
|
||||
const out: GameRecord[] = [];
|
||||
const lines = text.split("\n");
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
|
||||
@@ -40,30 +40,41 @@ async function index(stamp: string, shards: string[][], hashed = true): Promise<
|
||||
return { index: { max_per_file: 800, total: shards.flat().length, last_updated: stamp, files }, bodies };
|
||||
}
|
||||
|
||||
/** What the harness's IndexedDB holds: a generation, and whether it will open. */
|
||||
type Cached = Generation & { corrupt?: boolean };
|
||||
|
||||
interface Harness {
|
||||
deps: LoaderDeps;
|
||||
cache: { value: Generation | null };
|
||||
fetchShard: ReturnType<typeof vi.fn>;
|
||||
writeCache: ReturnType<typeof vi.fn>;
|
||||
/** Calls to the cached generation's records(): each one is a decode. */
|
||||
decode: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
/** A stand-in pack: the loader never opens one, it only hands it to the cache. */
|
||||
const packOf = (name: string) => enc.encode(`pack:${name}`);
|
||||
|
||||
function harness(opts: {
|
||||
index: DataIndex | Error;
|
||||
/** What the network returns for each shard name, versioned or not. */
|
||||
bodies: Map<string, string>;
|
||||
/** Shards that 503 when requested by hash. */
|
||||
notReady?: Set<string>;
|
||||
cached?: Generation | null;
|
||||
/** Shards that arrive over the plain path, so without a pack. */
|
||||
plainPath?: Set<string>;
|
||||
cached?: Cached | null;
|
||||
noCrypto?: boolean;
|
||||
}): Harness {
|
||||
const cache = { value: opts.cached ?? null };
|
||||
const fetchShard = vi.fn(async (entry: ShardManifestEntry, versioned: boolean) => {
|
||||
if (versioned && opts.notReady?.has(entry.name)) throw new ShardNotReadyError(entry.name);
|
||||
return enc.encode(opts.bodies.get(entry.name) ?? "").buffer as ArrayBuffer;
|
||||
});
|
||||
const writeCache = vi.fn(async (g: Generation) => {
|
||||
cache.value = g;
|
||||
return {
|
||||
plain: enc.encode(opts.bodies.get(entry.name) ?? "").buffer as ArrayBuffer,
|
||||
pack: opts.plainPath?.has(entry.name) ? null : packOf(entry.name),
|
||||
};
|
||||
});
|
||||
const writeCache = vi.fn(async (_index: DataIndex, _packs: (Uint8Array | null)[]) => {});
|
||||
const decode = vi.fn(async (c: Cached) => (c.corrupt ? null : c.records));
|
||||
const cached = opts.cached ?? null;
|
||||
const deps: LoaderDeps = {
|
||||
fetchIndex: async () => {
|
||||
if (opts.index instanceof Error) throw opts.index;
|
||||
@@ -81,10 +92,10 @@ function harness(opts: {
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((l) => JSON.parse(l) as GameRecord),
|
||||
readCache: async () => cache.value,
|
||||
readCache: async () => cached && { index: cached.index, records: () => decode(cached) },
|
||||
writeCache,
|
||||
};
|
||||
return { deps, cache, fetchShard, writeCache };
|
||||
return { deps, fetchShard, writeCache, decode };
|
||||
}
|
||||
|
||||
const signal = () => new AbortController().signal;
|
||||
@@ -112,7 +123,7 @@ describe("sameGeneration / isVersioned", () => {
|
||||
});
|
||||
|
||||
describe("createLoader", () => {
|
||||
it("verifies, parses and caches a new generation", async () => {
|
||||
it("verifies, parses and caches a new generation - as the packs it arrived in", async () => {
|
||||
const { index: idx, bodies } = await index("t1", [["1", "2"], ["3"]]);
|
||||
const h = harness({ index: idx, bodies });
|
||||
const r = await createLoader(h.deps)(signal());
|
||||
@@ -120,6 +131,14 @@ describe("createLoader", () => {
|
||||
expect(r).toMatchObject({ offline: false, unverified: false, retryInMs: null });
|
||||
expect(h.fetchShard.mock.calls.every((c) => c[1] === true)).toBe(true);
|
||||
expect(h.writeCache).toHaveBeenCalledTimes(1);
|
||||
expect(h.writeCache).toHaveBeenCalledWith(idx, [packOf("data_001.jsonl"), packOf("data_002.jsonl")]);
|
||||
});
|
||||
|
||||
it("offers a shard that came over the plain path to the cache as null", async () => {
|
||||
const { index: idx, bodies } = await index("t1", [["1"], ["2"]]);
|
||||
const h = harness({ index: idx, bodies, plainPath: new Set(["data_002.jsonl"]) });
|
||||
await createLoader(h.deps)(signal());
|
||||
expect(h.writeCache).toHaveBeenCalledWith(idx, [packOf("data_001.jsonl"), null]);
|
||||
});
|
||||
|
||||
it("fetches no shard when the cached generation is current", async () => {
|
||||
@@ -129,6 +148,25 @@ describe("createLoader", () => {
|
||||
const r = await createLoader(h.deps)(signal());
|
||||
expect(r.records).toBe(cached.records);
|
||||
expect(h.fetchShard).not.toHaveBeenCalled();
|
||||
expect(h.decode).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not open the cache just to find it is an older generation", async () => {
|
||||
const old = await index("t1", [["1"]]);
|
||||
const next = await index("t2", [["2"]]);
|
||||
const h = harness({ index: next.index, bodies: next.bodies, cached: { index: old.index, records: [{ name: "1" } as GameRecord] } });
|
||||
const r = await createLoader(h.deps)(signal());
|
||||
expect(r.records.map((x) => x.name)).toEqual(["2"]);
|
||||
expect(h.decode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("treats a cache that will not open as a miss", async () => {
|
||||
const { index: idx, bodies } = await index("t1", [["1"]]);
|
||||
const h = harness({ index: idx, bodies, cached: { index: idx, records: [], corrupt: true } });
|
||||
const r = await createLoader(h.deps)(signal());
|
||||
expect(r.records.map((x) => x.name)).toEqual(["1"]);
|
||||
expect(h.fetchShard).toHaveBeenCalledTimes(1);
|
||||
expect(h.writeCache).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("fetches no shard when the in-memory generation is current", async () => {
|
||||
@@ -162,6 +200,17 @@ describe("createLoader", () => {
|
||||
expect(h.writeCache).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("with nothing in memory, shows the cached generation while the shard is not ready", async () => {
|
||||
const old = await index("t1", [["1"]]);
|
||||
const next = await index("t2", [["2"]]);
|
||||
const cached = { index: old.index, records: [{ name: "1" } as GameRecord] };
|
||||
const h = harness({ index: next.index, bodies: next.bodies, notReady: new Set(["data_001.jsonl"]), cached });
|
||||
const r = await createLoader(h.deps)(signal());
|
||||
expect(r.records).toBe(cached.records);
|
||||
expect(r).toMatchObject({ unverified: false, retryInMs: 60_000 });
|
||||
expect(h.writeCache).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("with nothing held, shows unversioned shards but does not cache them", async () => {
|
||||
const next = await index("t2", [["2"]]);
|
||||
const h = harness({ index: next.index, bodies: next.bodies, notReady: new Set(["data_001.jsonl"]) });
|
||||
@@ -187,13 +236,19 @@ describe("createLoader", () => {
|
||||
await expect(createLoader(h.deps)(signal())).rejects.toThrow("Failed to fetch");
|
||||
});
|
||||
|
||||
it("offline with a cache that will not open rethrows", async () => {
|
||||
const old = await index("t1", [["1"]]);
|
||||
const h = harness({ index: new TypeError("Failed to fetch"), bodies: new Map(), cached: { index: old.index, records: [], corrupt: true } });
|
||||
await expect(createLoader(h.deps)(signal())).rejects.toThrow("Failed to fetch");
|
||||
});
|
||||
|
||||
it("a legacy index loads and caches as before", async () => {
|
||||
const { index: idx, bodies } = await index("t1", [["1"]], false);
|
||||
const h = harness({ index: idx, bodies });
|
||||
const r = await createLoader(h.deps)(signal());
|
||||
expect(r.records.map((x) => x.name)).toEqual(["1"]);
|
||||
expect(h.fetchShard.mock.calls.every((c) => c[1] === false)).toBe(true);
|
||||
expect(h.writeCache).toHaveBeenCalledTimes(1);
|
||||
expect(h.writeCache).toHaveBeenCalledWith(idx, [packOf("data_001.jsonl")]);
|
||||
});
|
||||
|
||||
it("without Web Crypto, trusts the Worker's own verification of ?v=", async () => {
|
||||
|
||||
+55
-19
@@ -16,6 +16,10 @@
|
||||
* worker), so right after a data commit a client could receive the NEW index
|
||||
* with an OLD shard - and the previous loader wrote that pair to IndexedDB under
|
||||
* the new `last_updated`, where it stayed until the next data change.
|
||||
*
|
||||
* WHAT IS CACHED is each shard's pack as it arrived (shared/data-pack.ts),
|
||||
* never parsed records: see lib/cache.ts. So a cached generation is decoded
|
||||
* only when it is actually used, never just to compare its index.
|
||||
*/
|
||||
import type { DataIndex, GameRecord, ShardManifestEntry } from "./schema";
|
||||
|
||||
@@ -24,15 +28,35 @@ export interface Generation {
|
||||
records: GameRecord[];
|
||||
}
|
||||
|
||||
/** One shard as fetched. */
|
||||
export interface FetchedShard {
|
||||
/** The committed bytes: what is hashed and parsed. */
|
||||
plain: ArrayBuffer;
|
||||
/** The pack they arrived in, or null when they came over the plain path. */
|
||||
pack: Uint8Array | null;
|
||||
}
|
||||
|
||||
/** A cached generation whose records are decoded only when asked for. */
|
||||
export interface CachedGeneration {
|
||||
index: DataIndex;
|
||||
/** null: the cache would not decode, which makes it a miss. */
|
||||
records(): Promise<GameRecord[] | null>;
|
||||
}
|
||||
|
||||
export interface LoaderDeps {
|
||||
fetchIndex(signal: AbortSignal): Promise<DataIndex>;
|
||||
/** Shard bytes. `versioned` asks for them by hash (`?v=<sha256>`). */
|
||||
fetchShard(entry: ShardManifestEntry, versioned: boolean, signal: AbortSignal): Promise<ArrayBuffer>;
|
||||
/** One shard. `versioned` asks for it by hash (`?v=<sha256>`). */
|
||||
fetchShard(entry: ShardManifestEntry, versioned: boolean, signal: AbortSignal): Promise<FetchedShard>;
|
||||
/** Lowercase hex sha256, or null where Web Crypto is unavailable. */
|
||||
hash(bytes: ArrayBuffer): Promise<string | null>;
|
||||
parse(bytes: ArrayBuffer): Promise<GameRecord[]>;
|
||||
readCache(): Promise<Generation | null>;
|
||||
writeCache(generation: Generation): Promise<void>;
|
||||
readCache(): Promise<CachedGeneration | null>;
|
||||
/**
|
||||
* Offered every generation this loader would cache, with each shard's pack
|
||||
* (null where it came over the plain path). The cache decides what it can
|
||||
* store.
|
||||
*/
|
||||
writeCache(index: DataIndex, packs: (Uint8Array | null)[]): Promise<void>;
|
||||
}
|
||||
|
||||
export interface LoadResult extends Generation {
|
||||
@@ -75,6 +99,16 @@ export function retryDelay(attempt: number): number {
|
||||
}
|
||||
|
||||
export function createLoader(deps: LoaderDeps) {
|
||||
/** A cached generation's records, or null when there is none or it will not decode. */
|
||||
async function decode(cached: CachedGeneration | null): Promise<Generation | null> {
|
||||
const records = cached ? await cached.records() : null;
|
||||
return cached && records ? { index: cached.index, records } : null;
|
||||
}
|
||||
|
||||
async function parseAll(shards: FetchedShard[]): Promise<GameRecord[]> {
|
||||
return (await Promise.all(shards.map((s) => deps.parse(s.plain)))).flat();
|
||||
}
|
||||
|
||||
return async function load(
|
||||
signal: AbortSignal,
|
||||
current?: Generation,
|
||||
@@ -89,7 +123,7 @@ export function createLoader(deps: LoaderDeps) {
|
||||
if (signal.aborted) throw err;
|
||||
// Offline, or the Worker is unreachable. Whatever generation is held is
|
||||
// still correct data - it just may not be the newest - so show it.
|
||||
const held = current ?? (await deps.readCache());
|
||||
const held = current ?? (await decode(await deps.readCache()));
|
||||
if (held) return { ...held, offline: true, unverified: false, retryInMs: null };
|
||||
throw err;
|
||||
}
|
||||
@@ -97,43 +131,45 @@ export function createLoader(deps: LoaderDeps) {
|
||||
// Nothing changed: no shard is fetched at all.
|
||||
if (current && sameGeneration(current.index, index)) return ok(current);
|
||||
const cached = await deps.readCache();
|
||||
if (cached && sameGeneration(cached.index, index)) return ok(cached);
|
||||
if (cached && sameGeneration(cached.index, index)) {
|
||||
const hit = await decode(cached);
|
||||
if (hit) return ok(hit);
|
||||
}
|
||||
|
||||
if (!isVersioned(index)) {
|
||||
// An index written before hashes existed. Behave as the old loader did.
|
||||
const bytes = await Promise.all(index.files.map((f) => deps.fetchShard(f, false, signal)));
|
||||
const fresh = { index, records: (await Promise.all(bytes.map((b) => deps.parse(b)))).flat() };
|
||||
void deps.writeCache(fresh).catch(() => {});
|
||||
const shards = await Promise.all(index.files.map((f) => deps.fetchShard(f, false, signal)));
|
||||
const fresh = { index, records: await parseAll(shards) };
|
||||
void deps.writeCache(index, shards.map((s) => s.pack)).catch(() => {});
|
||||
return ok(fresh);
|
||||
}
|
||||
|
||||
let verified: ArrayBuffer[] | null = null;
|
||||
let verified: FetchedShard[] | null = null;
|
||||
try {
|
||||
const bytes = await Promise.all(index.files.map((f) => deps.fetchShard(f, true, signal)));
|
||||
const hashes = await Promise.all(bytes.map((b) => deps.hash(b)));
|
||||
const shards = await Promise.all(index.files.map((f) => deps.fetchShard(f, true, signal)));
|
||||
const hashes = await Promise.all(shards.map((s) => deps.hash(s.plain)));
|
||||
// null = no Web Crypto here. The Worker only ever answers `?v=` with
|
||||
// bytes that hash to v, so that path is still verified, once.
|
||||
if (hashes.every((h, i) => h === null || h === index.files[i].sha256)) verified = bytes;
|
||||
if (hashes.every((h, i) => h === null || h === index.files[i].sha256)) verified = shards;
|
||||
} catch (err) {
|
||||
if (signal.aborted) throw err;
|
||||
if (!(err instanceof ShardNotReadyError)) throw err;
|
||||
}
|
||||
|
||||
if (verified) {
|
||||
const fresh = { index, records: (await Promise.all(verified.map((b) => deps.parse(b)))).flat() };
|
||||
void deps.writeCache(fresh).catch(() => {});
|
||||
const fresh = { index, records: await parseAll(verified) };
|
||||
void deps.writeCache(index, verified.map((s) => s.pack)).catch(() => {});
|
||||
return ok(fresh);
|
||||
}
|
||||
|
||||
// The upstream has not caught up with this commit yet.
|
||||
const retryInMs = retryDelay(attempt);
|
||||
const held = current ?? cached;
|
||||
const held = current ?? (await decode(cached));
|
||||
if (held) return { ...held, offline: false, unverified: false, retryInMs };
|
||||
|
||||
// Nothing held at all (a first visit in that window). Show the unversioned
|
||||
// shards so the page is not empty, but never cache them.
|
||||
const bytes = await Promise.all(index.files.map((f) => deps.fetchShard(f, false, signal)));
|
||||
const records = (await Promise.all(bytes.map((b) => deps.parse(b)))).flat();
|
||||
return { index, records, offline: false, unverified: true, retryInMs };
|
||||
const shards = await Promise.all(index.files.map((f) => deps.fetchShard(f, false, signal)));
|
||||
return { index, records: await parseAll(shards), offline: false, unverified: true, retryInMs };
|
||||
};
|
||||
}
|
||||
@@ -7,12 +7,13 @@
|
||||
* worker, and decides WHEN to look for newer data.
|
||||
*
|
||||
* The JSONL is parsed off the main thread in a worker, because parsing ~3,650
|
||||
* records blocks it for long enough to drop frames.
|
||||
* records blocks it for long enough to drop frames. The same worker opens the
|
||||
* packs the IndexedDB cache holds (lib/cache.ts).
|
||||
*/
|
||||
import { fetchIndex, fetchRemovedText, fetchShard } from "./fetcher";
|
||||
import { parseShard } from "./worker-pool";
|
||||
import { readCache, writeCache } from "./cache";
|
||||
import { createLoader, type Generation } from "./games-loader";
|
||||
import { parsePack, parseShard } from "./worker-pool";
|
||||
import { readCache, writeCache, type CachedBundle } from "./cache";
|
||||
import { createLoader, type CachedGeneration, type Generation } from "./games-loader";
|
||||
import { buildIndex, extractAppid } from "./data-store";
|
||||
import { Resource } from "./resource.svelte";
|
||||
import { appReady } from "./app-ready";
|
||||
@@ -42,14 +43,38 @@ async function sha256Hex(bytes: ArrayBuffer): Promise<string | null> {
|
||||
return out;
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
/**
|
||||
* What the loader sees of the cache: the index at once, the records only when
|
||||
* it asks. Packs are opened in the worker, once; any that will not open make
|
||||
* the whole generation a miss. A legacy bundle (parsed records, app 2.1.x and
|
||||
* older) is used as it stands until the next generation replaces it.
|
||||
*/
|
||||
function cachedGeneration(bundle: CachedBundle | null): CachedGeneration | null {
|
||||
if (!bundle) return null;
|
||||
if (bundle.kind === "legacy") {
|
||||
const { records } = bundle;
|
||||
return { index: bundle.index, records: async () => records };
|
||||
}
|
||||
let decoded: Promise<GameRecord[] | null> | undefined;
|
||||
return {
|
||||
index: bundle.index,
|
||||
records: () =>
|
||||
(decoded ??= Promise.all(bundle.packs.map((p) => parsePack(p))).then(
|
||||
(parts) => parts.flat(),
|
||||
(err: unknown) => {
|
||||
console.warn("[cache] the cached generation would not open:", err);
|
||||
return null;
|
||||
},
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
const load = createLoader({
|
||||
fetchIndex: (signal) => fetchIndex(signal),
|
||||
fetchShard: (entry, versioned, signal) => fetchShard(entry, versioned, signal),
|
||||
hash: sha256Hex,
|
||||
parse: (bytes) => parseShard(decoder.decode(bytes)),
|
||||
readCache,
|
||||
parse: parseShard,
|
||||
readCache: async () => cachedGeneration(await readCache()),
|
||||
writeCache,
|
||||
});
|
||||
|
||||
|
||||
@@ -178,6 +178,8 @@
|
||||
{/if}
|
||||
{:else if col.key === "current_players"}
|
||||
<span class="font-mono tnum">{formatNumber(g.current_players)}</span>
|
||||
{:else if col.key === "all_time_peak"}
|
||||
<span class="font-mono tnum">{formatNumber(g.all_time_peak)}</span>
|
||||
{:else if col.key === "anti_cheat"}
|
||||
{#if g.anti_cheat && g.anti_cheat !== "-"}
|
||||
<Badge variant={g.is_kernel_ac ? "destructive" : "warning"}>{g.anti_cheat}</Badge>
|
||||
|
||||
@@ -22,6 +22,7 @@ export type ColumnKey =
|
||||
| "type_game"
|
||||
| "reviews"
|
||||
| "current_players"
|
||||
| "all_time_peak"
|
||||
| "anti_cheat"
|
||||
| "platforms"
|
||||
| "release_date"
|
||||
@@ -79,6 +80,15 @@ export const COLS: ColDef[] = [
|
||||
align: "right",
|
||||
sortValue: (g) => parseIntSafe(g.current_players),
|
||||
},
|
||||
{
|
||||
key: "all_time_peak",
|
||||
label: "detail.labelAllTimePeak",
|
||||
width: 120,
|
||||
sortable: true,
|
||||
align: "right",
|
||||
// -1 for "N/A", as for reviews: unknown peaks cluster at one end.
|
||||
sortValue: (g) => (g.all_time_peak && g.all_time_peak !== "N/A" ? parseIntSafe(g.all_time_peak) : -1),
|
||||
},
|
||||
{
|
||||
key: "anti_cheat",
|
||||
label: "detail.labelAntiCheat",
|
||||
|
||||
@@ -103,6 +103,9 @@ export interface GameRecord {
|
||||
reviews: string;
|
||||
current_players: string;
|
||||
peak_today: string;
|
||||
/** Only ever raised: SteamCharts' recorded peak or our own highest sample
|
||||
* (scripts/core/peaks.py). "N/A" until either has a number. */
|
||||
all_time_peak: string;
|
||||
metacritic: string;
|
||||
anti_cheat: string;
|
||||
anti_cheat_note: string;
|
||||
@@ -154,6 +157,7 @@ export const SKELETON_TEMPLATE: GameRecord = {
|
||||
reviews: "N/A",
|
||||
current_players: "N/A",
|
||||
peak_today: "N/A",
|
||||
all_time_peak: "N/A",
|
||||
metacritic: "N/A",
|
||||
anti_cheat: "-",
|
||||
anti_cheat_note: "",
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { parsePack, parseShard } from "./worker-pool";
|
||||
import { packBytes, sha256 } from "../../shared/data-pack";
|
||||
|
||||
/**
|
||||
* Node has no Worker, so this drives the main-thread fallback - the same code
|
||||
* the worker runs (decodeShard), and what every shard still queued falls back
|
||||
* to if the worker dies.
|
||||
*/
|
||||
|
||||
const BODY =
|
||||
'{"link":"https://store.steampowered.com/app/730/","name":"Counter-Strike 2"}\n' +
|
||||
'{"link":"https://store.steampowered.com/app/570/","name":"Dota 2"}\n';
|
||||
const plain = new TextEncoder().encode(BODY);
|
||||
|
||||
describe("worker-pool", () => {
|
||||
it("parses a fetched shard's plain bytes", async () => {
|
||||
const records = await parseShard(plain.slice().buffer as ArrayBuffer);
|
||||
expect(records.map((r) => r.name)).toEqual(["Counter-Strike 2", "Dota 2"]);
|
||||
});
|
||||
|
||||
it("opens and parses a cached pack", async () => {
|
||||
const pack = await packBytes(plain, await sha256(plain));
|
||||
const records = await parsePack(pack);
|
||||
expect(records.map((r) => r.name)).toEqual(["Counter-Strike 2", "Dota 2"]);
|
||||
});
|
||||
|
||||
it("rejects a pack that will not open, so the cache counts as a miss", async () => {
|
||||
const pack = (await packBytes(plain, await sha256(plain))).slice();
|
||||
pack[20] ^= 0xff;
|
||||
await expect(parsePack(pack)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("worker-pool with a worker", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/** A stand-in Worker that answers every message with `reply`. */
|
||||
async function poolAnswering(reply: (id: number) => object) {
|
||||
class FakeWorker {
|
||||
onmessage: ((e: { data: object }) => void) | null = null;
|
||||
onerror: ((e: { message: string }) => void) | null = null;
|
||||
postMessage(msg: { id: number }) {
|
||||
setTimeout(() => this.onmessage?.({ data: reply(msg.id) }));
|
||||
}
|
||||
terminate() {}
|
||||
}
|
||||
vi.stubGlobal("Worker", FakeWorker);
|
||||
vi.resetModules(); // the pool keeps its worker in module state
|
||||
return import("./worker-pool");
|
||||
}
|
||||
|
||||
it("rejects when the worker reports an error with an EMPTY message", async () => {
|
||||
// What Chromium's failed AES-GCM decrypt looks like after error.message.
|
||||
// Read as success, it resolved a damaged pack as a shard with no records.
|
||||
const pool = await poolAnswering((id) => ({ id, error: "" }));
|
||||
await expect(pool.parsePack(new Uint8Array(40))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("resolves with the worker's records", async () => {
|
||||
const pool = await poolAnswering((id) => ({ id, records: [{ name: "Dota 2" }] }));
|
||||
expect(await pool.parseShard(plain.slice().buffer as ArrayBuffer)).toEqual([{ name: "Dota 2" }]);
|
||||
});
|
||||
});
|
||||
+39
-24
@@ -1,19 +1,25 @@
|
||||
/**
|
||||
* Tiny single-worker JSONL parser proxy.
|
||||
* Tiny single-worker shard parser proxy.
|
||||
*
|
||||
* One worker parses the shards (five of them at ~800 records each) off the main
|
||||
* thread, one message per shard. If the worker itself fails, every shard still
|
||||
* waiting on it is parsed on the main thread instead - see onerror.
|
||||
* One worker turns shard bytes into records off the main thread, one message
|
||||
* per shard: the plain JSONL bytes of a fresh download, or a pack straight
|
||||
* from the IndexedDB cache, which it opens first (decodeShard in fetcher.ts).
|
||||
* If the worker itself fails, every shard still waiting on it is parsed on the
|
||||
* main thread instead - see onerror.
|
||||
*/
|
||||
import type { GameRecord } from "./schema";
|
||||
import { parseJsonl } from "./fetcher";
|
||||
import { decodeShard } from "./fetcher";
|
||||
|
||||
interface Job {
|
||||
bytes: ArrayBuffer | Uint8Array;
|
||||
packed: boolean;
|
||||
resolve: (r: GameRecord[]) => void;
|
||||
reject: (e: Error) => void;
|
||||
}
|
||||
|
||||
let worker: Worker | null = null;
|
||||
let nextId = 1;
|
||||
const pending = new Map<
|
||||
number,
|
||||
{ text: string; resolve: (r: GameRecord[]) => void; reject: (e: Error) => void }
|
||||
>();
|
||||
const pending = new Map<number, Job>();
|
||||
|
||||
function getWorker(): Worker | null {
|
||||
if (typeof Worker === "undefined") return null;
|
||||
@@ -30,7 +36,11 @@ function getWorker(): Worker | null {
|
||||
const p = pending.get(id);
|
||||
if (!p) return;
|
||||
pending.delete(id);
|
||||
if (error) p.reject(new Error(error));
|
||||
// Not `if (error)`: a failed AES-GCM decrypt is a DOMException whose
|
||||
// message is EMPTY in Chromium, and "" read as success resolved a
|
||||
// damaged pack as a shard with no records - the cache then served the
|
||||
// catalogue a shard short instead of counting as a miss.
|
||||
if (typeof error === "string") p.reject(new Error(error || "the parser worker failed"));
|
||||
else p.resolve(records ?? []);
|
||||
};
|
||||
worker.onerror = (ev) => {
|
||||
@@ -42,13 +52,7 @@ function getWorker(): Worker | null {
|
||||
// load hung on "Loading…" forever.
|
||||
const orphans = [...pending.values()];
|
||||
pending.clear();
|
||||
for (const p of orphans) {
|
||||
try {
|
||||
p.resolve(parseJsonl(p.text));
|
||||
} catch (err) {
|
||||
p.reject(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
for (const p of orphans) decodeShard(p.bytes, p.packed).then(p.resolve, p.reject);
|
||||
};
|
||||
return worker;
|
||||
} catch {
|
||||
@@ -56,15 +60,26 @@ function getWorker(): Worker | null {
|
||||
}
|
||||
}
|
||||
|
||||
export function parseShard(text: string): Promise<GameRecord[]> {
|
||||
function parse(bytes: ArrayBuffer | Uint8Array, packed: boolean): Promise<GameRecord[]> {
|
||||
const w = getWorker();
|
||||
if (!w) {
|
||||
// Fallback: parse on main thread.
|
||||
return Promise.resolve(parseJsonl(text));
|
||||
}
|
||||
// Fallback: parse on the main thread.
|
||||
if (!w) return decodeShard(bytes, packed);
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { text, resolve, reject });
|
||||
w.postMessage({ id, text });
|
||||
pending.set(id, { bytes, packed, resolve, reject });
|
||||
// Copied, not transferred: a transferred buffer is emptied on this side,
|
||||
// and both onerror (which re-parses from it) and the loader (which hands
|
||||
// the packs to the cache after parsing) still need it.
|
||||
w.postMessage({ id, bytes, packed });
|
||||
});
|
||||
}
|
||||
|
||||
/** A shard's plain JSONL bytes, as fetched. */
|
||||
export function parseShard(bytes: ArrayBuffer): Promise<GameRecord[]> {
|
||||
return parse(bytes, false);
|
||||
}
|
||||
|
||||
/** A shard's pack, as the IndexedDB cache holds it. Rejects if it will not open. */
|
||||
export function parsePack(pack: Uint8Array): Promise<GameRecord[]> {
|
||||
return parse(pack, true);
|
||||
}
|
||||
@@ -97,11 +97,10 @@
|
||||
|
||||
const rows = $derived.by((): Row[] => {
|
||||
if (!view) return [];
|
||||
const players = live?.current_players
|
||||
? live.peak_today
|
||||
? `${formatNumber(live.current_players)} ${t("detail.playersPeak", { peak: formatNumber(live.peak_today) })}`
|
||||
: formatNumber(live.current_players)
|
||||
: undefined;
|
||||
const players = live?.current_players ? formatNumber(live.current_players) : undefined;
|
||||
// Not peak_today, which the page used to show as "(peak N)": that is only the
|
||||
// highest of our sparse samples since the last refetch, never reset daily.
|
||||
const peak = live?.all_time_peak && live.all_time_peak !== "N/A" ? formatNumber(live.all_time_peak) : undefined;
|
||||
const all: Row[] = [
|
||||
{ label: "detail.labelGenre", text: view.genre },
|
||||
{ label: "detail.labelType", text: view.type_game },
|
||||
@@ -110,6 +109,7 @@
|
||||
{ label: "detail.labelReleased", text: view.release_date },
|
||||
{ label: "detail.labelPlatforms", text: view.platforms.join(", ") },
|
||||
{ label: "detail.labelPlayers", text: players },
|
||||
{ label: "detail.labelAllTimePeak", text: peak },
|
||||
{ label: "detail.labelAntiCheat", text: view.anti_cheat },
|
||||
{ label: "detail.labelMetacritic", text: live?.metacritic },
|
||||
{ label: "detail.labelDrm", text: view.drm_notes },
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
/**
|
||||
* The parser worker's replies. Every failure must reach the page as a
|
||||
* NON-EMPTY `error`: a failed AES-GCM decrypt is a DOMException whose message
|
||||
* is empty in Chromium, and an empty error once let a damaged cached pack
|
||||
* through as a shard with no records.
|
||||
*/
|
||||
|
||||
const decodeShard = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../lib/fetcher", () => ({ decodeShard }));
|
||||
|
||||
let replies: unknown[];
|
||||
let fakeSelf: { onmessage: ((e: { data: unknown }) => Promise<void>) | null; postMessage: (m: unknown) => void };
|
||||
|
||||
beforeEach(async () => {
|
||||
replies = [];
|
||||
fakeSelf = { onmessage: null, postMessage: (m) => replies.push(m) };
|
||||
vi.stubGlobal("self", fakeSelf);
|
||||
vi.resetModules();
|
||||
await import("./jsonl-parser");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
decodeShard.mockReset();
|
||||
});
|
||||
|
||||
describe("jsonl-parser worker", () => {
|
||||
it("replies with the records", async () => {
|
||||
decodeShard.mockResolvedValue([{ name: "Dota 2" }]);
|
||||
await fakeSelf.onmessage!({ data: { id: 7, bytes: new Uint8Array(4), packed: true } });
|
||||
expect(decodeShard).toHaveBeenCalledWith(new Uint8Array(4), true);
|
||||
expect(replies).toEqual([{ id: 7, records: [{ name: "Dota 2" }] }]);
|
||||
});
|
||||
|
||||
it("names a failure even when its message is empty", async () => {
|
||||
decodeShard.mockRejectedValue(new DOMException("", "OperationError"));
|
||||
await fakeSelf.onmessage!({ data: { id: 8, bytes: new Uint8Array(4), packed: true } });
|
||||
expect(replies).toEqual([{ id: 8, error: "OperationError" }]);
|
||||
});
|
||||
|
||||
it("refuses a malformed message instead of throwing", async () => {
|
||||
await fakeSelf.onmessage!({ data: { id: 9, text: "the old { id, text } shape" } });
|
||||
expect(replies).toHaveLength(1);
|
||||
expect(replies[0]).toMatchObject({ id: 9 });
|
||||
expect((replies[0] as { error: string }).error).toMatch(/malformed/);
|
||||
expect(decodeShard).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,15 @@
|
||||
/// <reference lib="webworker" />
|
||||
/**
|
||||
* Web Worker: parse JSONL text off the main thread.
|
||||
* Receives { id, text }, returns { id, records } or { id, error }.
|
||||
* Web Worker: turn shard bytes into records off the main thread.
|
||||
* Receives { id, bytes, packed }, returns { id, records } or { id, error }.
|
||||
* `packed` bytes are a pack from the IndexedDB cache and are opened first.
|
||||
*/
|
||||
import { parseJsonl } from "../lib/fetcher";
|
||||
import { decodeShard } from "../lib/fetcher";
|
||||
|
||||
interface InMessage {
|
||||
id: number;
|
||||
text: string;
|
||||
bytes: ArrayBuffer | Uint8Array;
|
||||
packed: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -20,33 +22,37 @@ interface InMessage {
|
||||
* rejected promise instead of an unhandled throw inside the worker.
|
||||
*/
|
||||
function isInMessage(d: unknown): d is InMessage {
|
||||
if (typeof d !== "object" || d === null) return false;
|
||||
const m = d as InMessage;
|
||||
return (
|
||||
typeof d === "object" &&
|
||||
d !== null &&
|
||||
typeof (d as InMessage).id === "number" &&
|
||||
typeof (d as InMessage).text === "string"
|
||||
typeof m.id === "number" &&
|
||||
(m.bytes instanceof ArrayBuffer || m.bytes instanceof Uint8Array) &&
|
||||
typeof m.packed === "boolean"
|
||||
);
|
||||
}
|
||||
|
||||
self.onmessage = (e: MessageEvent<unknown>) => {
|
||||
self.onmessage = async (e: MessageEvent<unknown>) => {
|
||||
if (!isInMessage(e.data)) {
|
||||
const id = (e.data as { id?: unknown } | null)?.id;
|
||||
(self as unknown as Worker).postMessage({
|
||||
id: typeof id === "number" ? id : -1,
|
||||
error: "jsonl-parser: malformed message (expected { id: number, text: string })",
|
||||
error: "jsonl-parser: malformed message (expected { id: number, bytes: ArrayBuffer | Uint8Array, packed: boolean })",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { id, text } = e.data;
|
||||
const { id, bytes, packed } = e.data;
|
||||
try {
|
||||
const records = parseJsonl(text);
|
||||
const records = await decodeShard(bytes, packed);
|
||||
(self as unknown as Worker).postMessage({ id, records });
|
||||
} catch (error) {
|
||||
(self as unknown as Worker).postMessage({
|
||||
id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
(self as unknown as Worker).postMessage({ id, error: describe(error) });
|
||||
}
|
||||
};
|
||||
|
||||
/** Never empty: a failed decrypt is a DOMException with an empty message in Chromium. */
|
||||
function describe(error: unknown): string {
|
||||
if (error instanceof Error) return error.message || error.name || "Error";
|
||||
return String(error) || "unknown error";
|
||||
}
|
||||
|
||||
export {};
|
||||
+3
-2
@@ -12,8 +12,9 @@
|
||||
// declare `self`, `onmessage` and friends with different types.
|
||||
// - TypeScript resolves `cookie` to the HOISTED 1.1.1 (pulled in by
|
||||
// wrangler -> miniflare -> youch) while @sveltejs/kit has its own
|
||||
// nested cookie@0.6.0 and uses that at runtime. The type error is a
|
||||
// resolution artifact, not a real version conflict.
|
||||
// nested cookie@0.7.x (package.json `overrides`) and uses that at
|
||||
// runtime. The type error is a resolution artifact, not a real version
|
||||
// conflict.
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
|
||||
Reference in new issue
Block a user